summaryrefslogtreecommitdiffstats
path: root/Lib/test/test_xml_etree_c.py
blob: 78a6a9241c9b97c50f8af7357975ec381bb3e99d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# xml.etree test for cElementTree

from test import test_support

cET = test_support.import_module('xml.etree.cElementTree')


# cElementTree specific tests

def sanity():
    """
    Import sanity.

    >>> from xml.etree import cElementTree
    """


def test_main():
    from test import test_xml_etree, test_xml_etree_c

    # Run the tests specific to the C implementation
    test_support.run_doctest(test_xml_etree_c, verbosity=True)

    # Assign the C implementation before running the doctests
    # Patch the __name__, to prevent confusion with the pure Python test
    pyET = test_xml_etree.ET
    py__name__ = test_xml_etree.__name__
    test_xml_etree.ET = cET
    if __name__ != '__main__':
        test_xml_etree.__name__ = __name__
    try:
        # Run the same test suite as xml.etree.ElementTree
        test_xml_etree.test_main(module_name='xml.etree.cElementTree')
    finally:
        test_xml_etree.ET = pyET
        test_xml_etree.__name__ = py__name__

if __name__ == '__main__':
    test_main()
ef='/oss-git/cpython.git/diff/Doc/c-api/arg.rst?h=benjamin-clang&id=f5bdfda2709ab5492ae0b306d5ff706777e6cdb5&id2=aa2d085d2951a7b5c7c81d30a2fdd333a2fa7145'>Doc/c-api/arg.rst2
-rw-r--r--Doc/c-api/concrete.rst22
-rw-r--r--Doc/c-api/dict.rst18
-rw-r--r--Doc/c-api/exceptions.rst67
-rw-r--r--Doc/c-api/file.rst2
-rw-r--r--Doc/c-api/import.rst8
-rw-r--r--Doc/c-api/init.rst50
-rw-r--r--Doc/c-api/long.rst2
-rw-r--r--Doc/c-api/mapping.rst6
-rw-r--r--Doc/c-api/memory.rst171
-rw-r--r--Doc/c-api/module.rst11
-rw-r--r--Doc/c-api/object.rst27
-rw-r--r--Doc/c-api/tuple.rst8
-rw-r--r--Doc/c-api/type.rst10
-rw-r--r--Doc/c-api/typeobj.rst49
-rw-r--r--Doc/c-api/unicode.rst11
-rw-r--r--Doc/c-api/veryhigh.rst52
-rw-r--r--Doc/conf.py6
-rw-r--r--Doc/contents.rst12
-rw-r--r--Doc/data/refcounts.dat33
-rw-r--r--Doc/distributing/index.rst143
-rw-r--r--Doc/distutils/apiref.rst13
-rw-r--r--Doc/distutils/builtdist.rst3
-rw-r--r--Doc/distutils/index.rst6
-rw-r--r--Doc/distutils/sourcedist.rst20
-rw-r--r--Doc/extending/embedding.rst8
-rw-r--r--Doc/extending/index.rst42
-rw-r--r--Doc/extending/newtypes.rst24
-rw-r--r--Doc/faq/extending.rst4
-rw-r--r--Doc/faq/library.rst8
-rw-r--r--Doc/faq/programming.rst10
-rw-r--r--Doc/glossary.rst36
-rw-r--r--Doc/howto/clinic.rst1686
-rw-r--r--Doc/howto/functional.rst144
-rw-r--r--Doc/howto/index.rst1
-rw-r--r--Doc/howto/logging-cookbook.rst8
-rw-r--r--Doc/howto/logging.rst4
-rw-r--r--Doc/howto/regex.rst24
-rw-r--r--Doc/howto/urllib2.rst2
-rw-r--r--Doc/includes/email-alternative-new-api.py56
-rw-r--r--Doc/includes/email-dir.py86
-rw-r--r--Doc/includes/email-read-alternative-new-api.py74
-rw-r--r--Doc/includes/email-unpack.py45
-rw-r--r--Doc/includes/mp_benchmarks.py239
-rw-r--r--Doc/includes/mp_newtype.py14
-rw-r--r--Doc/includes/mp_pool.py337
-rw-r--r--Doc/includes/mp_synchronize.py278
-rw-r--r--Doc/includes/mp_webserver.py70
-rw-r--r--Doc/includes/mp_workers.py13
-rw-r--r--Doc/includes/typestruct.h7
-rw-r--r--Doc/install/index.rst6
-rw-r--r--Doc/installing/index.rst210
-rw-r--r--Doc/library/2to3.rst37
-rw-r--r--Doc/library/abc.rst38
-rw-r--r--Doc/library/aifc.rst19
-rw-r--r--Doc/library/argparse.rst23
-rw-r--r--Doc/library/asynchat.rst5
-rw-r--r--Doc/library/asyncio-dev.rst251
-rw-r--r--Doc/library/asyncio-eventloop.rst668
-rw-r--r--Doc/library/asyncio-protocol.rst502
-rw-r--r--Doc/library/asyncio-stream.rst278
-rw-r--r--Doc/library/asyncio-subprocess.rst270
-rw-r--r--Doc/library/asyncio-sync.rst410
-rw-r--r--Doc/library/asyncio-task.rst545
-rw-r--r--Doc/library/asyncio.rst59
-rw-r--r--Doc/library/asyncore.rst5
-rw-r--r--Doc/library/audioop.rst29
-rw-r--r--Doc/library/base64.rst104
-rw-r--r--Doc/library/bz2.rst15
-rw-r--r--Doc/library/cgi.rst13
-rw-r--r--Doc/library/code.rst5
-rw-r--r--Doc/library/codecs.rst156
-rw-r--r--Doc/library/collections.abc.rst2
-rw-r--r--Doc/library/collections.rst26
-rw-r--r--Doc/library/compileall.rst3
-rw-r--r--Doc/library/concurrency.rst1
-rw-r--r--Doc/library/concurrent.futures.rst3
-rw-r--r--Doc/library/contextlib.rst227
-rw-r--r--Doc/library/curses.rst2
-rw-r--r--Doc/library/datatypes.rst1
-rw-r--r--Doc/library/datetime.rst10
-rw-r--r--Doc/library/dbm.rst38
-rw-r--r--Doc/library/debug.rst1
-rw-r--r--Doc/library/development.rst1
-rw-r--r--Doc/library/dis.rst253
-rw-r--r--Doc/library/distribution.rst14
-rw-r--r--Doc/library/distutils.rst19
-rw-r--r--Doc/library/doctest.rst28
-rw-r--r--Doc/library/email-examples.rst30
-rw-r--r--Doc/library/email.contentmanager.rst430
-rw-r--r--Doc/library/email.message.rst87
-rw-r--r--Doc/library/email.policy.rst45
-rw-r--r--Doc/library/email.rst1
-rw-r--r--Doc/library/ensurepip.rst131
-rw-r--r--Doc/library/enum.rst749
-rw-r--r--Doc/library/exceptions.rst10
-rw-r--r--Doc/library/faulthandler.rst35
-rw-r--r--Doc/library/filecmp.rst32
-rw-r--r--Doc/library/fileinput.rst3
-rw-r--r--Doc/library/filesys.rst1
-rw-r--r--Doc/library/fnmatch.rst2
-rw-r--r--Doc/library/formatter.rst5
-rw-r--r--Doc/library/ftplib.rst55
-rw-r--r--Doc/library/functions.rst77
-rw-r--r--Doc/library/functools.rst179
-rw-r--r--Doc/library/gc.rst48
-rw-r--r--Doc/library/glob.rst15
-rw-r--r--Doc/library/gzip.rst15
-rw-r--r--Doc/library/hashlib.rst65
-rw-r--r--Doc/library/hmac.rst43
-rw-r--r--Doc/library/html.entities.rst1
-rw-r--r--Doc/library/html.parser.rst39
-rw-r--r--Doc/library/html.rst11
-rw-r--r--Doc/library/http.client.rst62
-rw-r--r--Doc/library/http.server.rst37
-rw-r--r--Doc/library/idle.rst345
-rw-r--r--Doc/library/imaplib.rst30
-rw-r--r--Doc/library/imp.rst65
-rw-r--r--Doc/library/importlib.rst693
-rw-r--r--Doc/library/index.rst1
-rw-r--r--Doc/library/inspect.rst73
-rw-r--r--Doc/library/io.rst57
-rw-r--r--Doc/library/ipaddress.rst30
-rw-r--r--Doc/library/ipc.rst5
-rw-r--r--Doc/library/json.rst72
-rw-r--r--Doc/library/logging.config.rst54
-rw-r--r--Doc/library/logging.handlers.rst14
-rw-r--r--Doc/library/lzma.rst15
-rw-r--r--Doc/library/marshal.rst9
-rw-r--r--Doc/library/multiprocessing.rst320
-rw-r--r--Doc/library/netrc.rst2
-rw-r--r--Doc/library/nntplib.rst19
-rw-r--r--Doc/library/numeric.rst1
-rw-r--r--Doc/library/operator.rst11
-rw-r--r--Doc/library/optparse.rst4
-rw-r--r--Doc/library/os.path.rst38
-rw-r--r--Doc/library/os.rst272
-rw-r--r--Doc/library/pathlib-inheritance.pngbin0 -> 11280 bytes-rw-r--r--Doc/library/pathlib-inheritance.svg4
-rw-r--r--Doc/library/pathlib.rst919
-rw-r--r--Doc/library/pdb.rst14
-rw-r--r--Doc/library/pickle.rst171
-rw-r--r--Doc/library/pkgutil.rst17
-rw-r--r--Doc/library/plistlib.rst184
-rw-r--r--Doc/library/poplib.rst51
-rw-r--r--Doc/library/pprint.rst254
-rw-r--r--Doc/library/pty.rst3
-rw-r--r--Doc/library/py_compile.rst16
-rw-r--r--Doc/library/pydoc.rst5
-rw-r--r--Doc/library/pyexpat.rst9
-rw-r--r--Doc/library/python.rst1
-rw-r--r--Doc/library/re.rst48
-rw-r--r--Doc/library/readline.rst20
-rw-r--r--Doc/library/resource.rst95
-rw-r--r--Doc/library/rlcompleter.rst16
-rw-r--r--Doc/library/runpy.rst69
-rw-r--r--Doc/library/select.rst66
-rw-r--r--Doc/library/selectors.rst257
-rw-r--r--Doc/library/shelve.rst17
-rw-r--r--Doc/library/shutil.rst22
-rw-r--r--Doc/library/site.rst27
-rw-r--r--Doc/library/smtpd.rst19
-rw-r--r--Doc/library/smtplib.rst39
-rw-r--r--Doc/library/socket.rst102
-rw-r--r--Doc/library/spwd.rst10
-rw-r--r--Doc/library/sqlite3.rst14
-rw-r--r--Doc/library/ssl.rst485
-rw-r--r--Doc/library/stat.rst51
-rw-r--r--Doc/library/statistics.rst418
-rw-r--r--Doc/library/stdtypes.rst6
-rw-r--r--Doc/library/struct.rst20
-rw-r--r--Doc/library/subprocess.rst45
-rw-r--r--Doc/library/sunau.rst14
-rw-r--r--Doc/library/sys.rst52
-rw-r--r--Doc/library/sysconfig.rst2
-rw-r--r--Doc/library/tarfile.rst61
-rw-r--r--Doc/library/test.rst22
-rw-r--r--Doc/library/textwrap.rst73
-rw-r--r--Doc/library/threading.rst9
-rw-r--r--Doc/library/timeit.rst2
-rw-r--r--Doc/library/tkinter.rst28
-rw-r--r--Doc/library/traceback.rst9
-rw-r--r--Doc/library/tracemalloc.rst634
-rw-r--r--Doc/library/tulip_coro.diabin0 -> 4461 bytes-rw-r--r--Doc/library/tulip_coro.pngbin0 -> 45565 bytes-rw-r--r--Doc/library/types.rst53
-rw-r--r--Doc/library/undoc.rst2
-rw-r--r--Doc/library/unicodedata.rst8
-rw-r--r--Doc/library/unittest.mock-examples.rst18
-rw-r--r--Doc/library/unittest.mock.rst31
-rw-r--r--Doc/library/unittest.rst206
-rw-r--r--Doc/library/urllib.error.rst7
-rw-r--r--Doc/library/urllib.request.rst139
-rw-r--r--Doc/library/venv.rst24
-rw-r--r--Doc/library/wave.rst62
-rw-r--r--Doc/library/weakref.rst257
-rw-r--r--Doc/library/xml.etree.elementtree.rst177
-rw-r--r--Doc/library/zipfile.rst58
-rw-r--r--Doc/license.rst76
-rw-r--r--Doc/make.bat19
-rw-r--r--Doc/reference/compound_stmts.rst17
-rw-r--r--Doc/reference/datamodel.rst53
-rw-r--r--Doc/reference/import.rst587
-rw-r--r--Doc/tools/sphinx-build.py28
-rw-r--r--Doc/tools/sphinxext/download.html4
-rw-r--r--Doc/tools/sphinxext/indexcontent.html8
-rw-r--r--Doc/tools/sphinxext/indexsidebar.html2
-rw-r--r--Doc/tools/sphinxext/layout.html2
-rw-r--r--Doc/tools/sphinxext/pyspecific.py2
-rw-r--r--Doc/tools/sphinxext/susp-ignored.csv38
-rw-r--r--Doc/tutorial/interactive.rst153
-rw-r--r--Doc/tutorial/interpreter.rst18
-rw-r--r--Doc/tutorial/modules.rst45
-rw-r--r--Doc/tutorial/stdlib.rst2
-rw-r--r--Doc/tutorial/stdlib2.rst2
-rw-r--r--Doc/tutorial/whatnow.rst4
-rw-r--r--Doc/using/cmdline.rst95
-rw-r--r--Doc/using/mac.rst6
-rw-r--r--Doc/using/unix.rst2
-rw-r--r--Doc/using/venv-create.inc49
-rw-r--r--Doc/using/windows.rst10
-rw-r--r--Doc/whatsnew/3.4.rst2506
-rw-r--r--Doc/whatsnew/index.rst1
-rw-r--r--Include/Python-ast.h31
-rw-r--r--Include/Python.h1
-rw-r--r--Include/abstract.h48
-rw-r--r--Include/asdl.h15
-rw-r--r--Include/ast.h5
-rw-r--r--Include/bytearrayobject.h11
-rw-r--r--Include/bytesobject.h10
-rw-r--r--Include/compile.h18
-rw-r--r--Include/dictobject.h8
-rw-r--r--Include/fileobject.h5
-rw-r--r--Include/fileutils.h19
-rw-r--r--Include/frameobject.h6
-rw-r--r--Include/genobject.h2
-rw-r--r--Include/grammar.h8
-rw-r--r--Include/import.h24
-rw-r--r--Include/longobject.h19
-rw-r--r--Include/marshal.h5
-rw-r--r--Include/modsupport.h3
-rw-r--r--Include/moduleobject.h1
-rw-r--r--Include/object.h116
-rw-r--r--Include/objimpl.h104
-rw-r--r--Include/opcode.h2
-rw-r--r--Include/osdefs.h8
-rw-r--r--Include/parsetok.h49
-rw-r--r--Include/patchlevel.h6
-rw-r--r--Include/pydebug.h1
-rw-r--r--Include/pyerrors.h44
-rw-r--r--Include/pyhash.h149
-rw-r--r--Include/pymacro.h6
-rw-r--r--Include/pymem.h99
-rw-r--r--Include/pyport.h71
-rw-r--r--Include/pystate.h50
-rw-r--r--Include/pythonrun.h47
-rw-r--r--Include/pytime.h17
-rw-r--r--Include/setobject.h9
-rw-r--r--Include/sliceobject.h3
-rw-r--r--Include/structseq.h2
-rw-r--r--Include/symtable.h10
-rw-r--r--Include/sysmodule.h5
-rw-r--r--Include/token.h2
-rw-r--r--Include/unicodeobject.h91
-rw-r--r--Include/warnings.h17
-rw-r--r--LICENSE4
-rw-r--r--Lib/_bootlocale.py34
-rw-r--r--Lib/_collections_abc.py734
-rw-r--r--Lib/_dummy_thread.py4
-rw-r--r--Lib/_osx_support.py4
-rw-r--r--Lib/_pyio.py209
-rw-r--r--Lib/_sitebuiltins.py99
-rw-r--r--Lib/_strptime.py4
-rw-r--r--Lib/abc.py20
-rw-r--r--Lib/aifc.py41
-rw-r--r--Lib/argparse.py31
-rw-r--r--Lib/ast.py6
-rw-r--r--Lib/asynchat.py9
-rw-r--r--Lib/asyncio/__init__.py45
-rw-r--r--Lib/asyncio/base_events.py828
-rw-r--r--Lib/asyncio/base_subprocess.py159
-rw-r--r--Lib/asyncio/constants.py7
-rw-r--r--Lib/asyncio/events.py485
-rw-r--r--Lib/asyncio/futures.py371
-rw-r--r--Lib/asyncio/locks.py466
-rw-r--r--Lib/asyncio/log.py7
-rw-r--r--Lib/asyncio/proactor_events.py455
-rw-r--r--Lib/asyncio/protocols.py129
-rw-r--r--Lib/asyncio/queues.py288
-rw-r--r--Lib/asyncio/selector_events.py846
-rw-r--r--Lib/asyncio/streams.py468
-rw-r--r--Lib/asyncio/subprocess.py195
-rw-r--r--Lib/asyncio/tasks.py682
-rw-r--r--Lib/asyncio/test_utils.py374
-rw-r--r--Lib/asyncio/transports.py295
-rw-r--r--Lib/asyncio/unix_events.py838
-rw-r--r--Lib/asyncio/windows_events.py489
-rw-r--r--Lib/asyncio/windows_utils.py205
-rw-r--r--Lib/asyncore.py29
-rwxr-xr-xLib/base64.py370
-rw-r--r--Lib/bdb.py36
-rw-r--r--Lib/bz2.py50
-rwxr-xr-xLib/cProfile.py53
-rwxr-xr-xLib/cgi.py13
-rw-r--r--Lib/chunk.py6
-rw-r--r--Lib/code.py2
-rw-r--r--Lib/collections/__init__.py28
-rw-r--r--Lib/collections/abc.py730
-rw-r--r--Lib/colorsys.py18
-rw-r--r--Lib/compileall.py20
-rw-r--r--Lib/concurrent/futures/_base.py9
-rw-r--r--Lib/concurrent/futures/process.py11
-rw-r--r--Lib/concurrent/futures/thread.py2
-rw-r--r--Lib/configparser.py2
-rw-r--r--Lib/contextlib.py71
-rw-r--r--Lib/copyreg.py6
-rw-r--r--Lib/ctypes/__init__.py26
-rw-r--r--Lib/ctypes/test/__init__.py2
-rw-r--r--Lib/ctypes/test/test_checkretval.py2
-rw-r--r--Lib/ctypes/test/test_internals.py19
-rw-r--r--Lib/ctypes/test/test_macholib.py63
-rw-r--r--Lib/ctypes/test/test_win32.py2
-rw-r--r--Lib/ctypes/util.py22
-rw-r--r--Lib/datetime.py395
-rw-r--r--Lib/dbm/__init__.py16
-rw-r--r--Lib/dbm/dumb.py16
-rw-r--r--Lib/decimal.py5
-rw-r--r--Lib/difflib.py36
-rw-r--r--Lib/dis.py396
-rw-r--r--Lib/distutils/__init__.py2
-rw-r--r--Lib/distutils/archive_util.py71
-rw-r--r--Lib/distutils/ccompiler.py5
-rw-r--r--Lib/distutils/cmd.py6
-rw-r--r--Lib/distutils/command/bdist.py16
-rw-r--r--Lib/distutils/command/bdist_dumb.py19
-rw-r--r--Lib/distutils/command/build_ext.py26
-rw-r--r--Lib/distutils/command/build_py.py10
-rw-r--r--Lib/distutils/command/build_scripts.py2
-rw-r--r--Lib/distutils/command/install.py15
-rw-r--r--Lib/distutils/command/install_lib.py6
-rw-r--r--Lib/distutils/command/sdist.py9
-rw-r--r--Lib/distutils/command/upload.py2
-rw-r--r--Lib/distutils/config.py9
-rw-r--r--Lib/distutils/core.py7
-rw-r--r--Lib/distutils/cygwinccompiler.py28
-rw-r--r--Lib/distutils/dir_util.py4
-rw-r--r--Lib/distutils/dist.py35
-rw-r--r--Lib/distutils/emxccompiler.py315
-rw-r--r--Lib/distutils/errors.py4
-rw-r--r--Lib/distutils/file_util.py16
-rw-r--r--Lib/distutils/msvc9compiler.py2
-rw-r--r--Lib/distutils/spawn.py28
-rw-r--r--Lib/distutils/sysconfig.py38
-rw-r--r--Lib/distutils/tests/test_archive_util.py61
-rw-r--r--Lib/distutils/tests/test_bdist_dumb.py5
-rw-r--r--Lib/distutils/tests/test_build_py.py10
-rw-r--r--Lib/distutils/tests/test_dist.py29
-rw-r--r--Lib/distutils/tests/test_install.py6
-rw-r--r--Lib/distutils/tests/test_install_lib.py8
-rw-r--r--Lib/distutils/tests/test_sdist.py54
-rw-r--r--Lib/distutils/tests/test_sysconfig.py48
-rw-r--r--Lib/distutils/tests/test_upload.py9
-rw-r--r--Lib/distutils/tests/test_util.py2
-rw-r--r--Lib/distutils/util.py13
-rw-r--r--Lib/doctest.py76
-rw-r--r--Lib/email/_header_value_parser.py39
-rw-r--r--Lib/email/contentmanager.py249
-rw-r--r--Lib/email/encoders.py17
-rw-r--r--Lib/email/feedparser.py52
-rw-r--r--Lib/email/generator.py4
-rw-r--r--Lib/email/header.py2
-rw-r--r--Lib/email/iterators.py6
-rw-r--r--Lib/email/message.py269
-rw-r--r--Lib/email/mime/text.py1
-rw-r--r--Lib/email/parser.py7
-rw-r--r--Lib/email/policy.py13
-rw-r--r--Lib/email/quoprimime.py1
-rw-r--r--Lib/email/utils.py30
-rw-r--r--Lib/encodings/aliases.py47
-rw-r--r--Lib/encodings/cp037.py1
-rw-r--r--Lib/encodings/cp1125.py698
-rw-r--r--Lib/encodings/cp273.py307
-rw-r--r--Lib/encodings/cp500.py1
-rw-r--r--Lib/encodings/iso8859_1.py1
-rw-r--r--Lib/ensurepip/__init__.py210
-rw-r--r--Lib/ensurepip/__main__.py4
-rw-r--r--Lib/ensurepip/_bundled/pip-1.5.4-py2.py3-none-any.whlbin0 -> 1169272 bytes-rw-r--r--Lib/ensurepip/_bundled/setuptools-2.1-py2.py3-none-any.whlbin0 -> 540293 bytes-rw-r--r--Lib/ensurepip/_uninstall.py30
-rw-r--r--Lib/enum.py525
-rw-r--r--Lib/filecmp.py25
-rw-r--r--Lib/fileinput.py16
-rw-r--r--Lib/formatter.py3
-rw-r--r--Lib/fractions.py14
-rw-r--r--Lib/ftplib.py188
-rw-r--r--Lib/functools.py422
-rw-r--r--Lib/genericpath.py34
-rw-r--r--Lib/gettext.py14
-rw-r--r--Lib/glob.py21
-rw-r--r--Lib/gzip.py115
-rw-r--r--Lib/hashlib.py97
-rw-r--r--Lib/hmac.py31
-rw-r--r--Lib/html/__init__.py123
-rw-r--r--Lib/html/parser.py111
-rw-r--r--Lib/http/client.py61
-rw-r--r--Lib/http/cookiejar.py31
-rw-r--r--Lib/http/server.py44
-rw-r--r--Lib/idlelib/Bindings.py32
-rw-r--r--Lib/idlelib/Debugger.py2
-rw-r--r--Lib/idlelib/EditorWindow.py27
-rw-r--r--Lib/idlelib/FileList.py2
-rw-r--r--Lib/idlelib/GrepDialog.py1
-rw-r--r--Lib/idlelib/IOBinding.py2
-rw-r--r--Lib/idlelib/MultiCall.py36
-rw-r--r--Lib/idlelib/NEWS.txt68
-rw-r--r--Lib/idlelib/PathBrowser.py2
-rwxr-xr-xLib/idlelib/PyShell.py16
-rw-r--r--Lib/idlelib/ScriptBinding.py4
-rw-r--r--Lib/idlelib/TreeWidget.py4
-rw-r--r--Lib/idlelib/ZoomHeight.py2
-rw-r--r--Lib/idlelib/configDialog.py2
-rw-r--r--Lib/idlelib/configHandler.py16
-rw-r--r--Lib/idlelib/help.txt556
-rw-r--r--Lib/idlelib/idle_test/test_calltips.py7
-rw-r--r--Lib/idlelib/idlever.py2
-rw-r--r--Lib/idlelib/keybindingDialog.py5
-rw-r--r--Lib/idlelib/macosxSupport.py136
-rw-r--r--Lib/idlelib/rpc.py8
-rw-r--r--Lib/idlelib/run.py4
-rw-r--r--Lib/imaplib.py30
-rw-r--r--Lib/imghdr.py2
-rw-r--r--Lib/imp.py224
-rw-r--r--Lib/importlib/__init__.py94
-rw-r--r--Lib/importlib/_bootstrap.py1748
-rw-r--r--Lib/importlib/abc.py367
-rw-r--r--Lib/importlib/machinery.py1
-rw-r--r--Lib/importlib/util.py187
-rw-r--r--Lib/inspect.py1034
-rw-r--r--Lib/io.py2
-rw-r--r--Lib/ipaddress.py93
-rw-r--r--Lib/json/__init__.py30
-rw-r--r--Lib/json/decoder.py28
-rw-r--r--Lib/json/encoder.py56
-rw-r--r--Lib/json/scanner.py4
-rw-r--r--Lib/json/tool.py3
-rwxr-xr-xLib/keyword.py23
-rw-r--r--Lib/lib2to3/btm_utils.py6
-rw-r--r--Lib/lib2to3/fixer_util.py23
-rw-r--r--Lib/lib2to3/fixes/fix_asserts.py34
-rw-r--r--Lib/lib2to3/fixes/fix_intern.py21
-rw-r--r--Lib/lib2to3/fixes/fix_reload.py28
-rw-r--r--Lib/lib2to3/main.py4
-rw-r--r--Lib/lib2to3/pgen2/conv.py4
-rw-r--r--Lib/lib2to3/pgen2/driver.py2
-rw-r--r--Lib/lib2to3/pgen2/grammar.py10
-rw-r--r--Lib/lib2to3/pytree.py9
-rw-r--r--Lib/lib2to3/refactor.py6
-rwxr-xr-xLib/lib2to3/tests/pytree_idempotency.py2
-rw-r--r--Lib/lib2to3/tests/test_fixers.py109
-rw-r--r--Lib/linecache.py8
-rw-r--r--Lib/locale.py487
-rw-r--r--Lib/logging/__init__.py90
-rw-r--r--Lib/logging/config.py183
-rw-r--r--Lib/logging/handlers.py127
-rw-r--r--Lib/lzma.py137
-rw-r--r--Lib/macpath.py2
-rw-r--r--Lib/mailbox.py84
-rw-r--r--Lib/mailcap.py2
-rw-r--r--Lib/mimetypes.py2
-rw-r--r--Lib/modulefinder.py19
-rw-r--r--Lib/multiprocessing/__init__.py251
-rw-r--r--Lib/multiprocessing/connection.py150
-rw-r--r--Lib/multiprocessing/context.py348
-rw-r--r--Lib/multiprocessing/dummy/__init__.py31
-rw-r--r--Lib/multiprocessing/dummy/connection.py27
-rw-r--r--Lib/multiprocessing/forking.py474
-rw-r--r--Lib/multiprocessing/forkserver.py267
-rw-r--r--Lib/multiprocessing/heap.py54
-rw-r--r--Lib/multiprocessing/managers.py62
-rw-r--r--Lib/multiprocessing/pool.py143
-rw-r--r--Lib/multiprocessing/popen_fork.py83
-rw-r--r--Lib/multiprocessing/popen_forkserver.py69
-rw-r--r--Lib/multiprocessing/popen_spawn_posix.py69
-rw-r--r--Lib/multiprocessing/popen_spawn_win32.py99
-rw-r--r--Lib/multiprocessing/process.py83
-rw-r--r--Lib/multiprocessing/queues.py106
-rw-r--r--Lib/multiprocessing/reduction.py362
-rw-r--r--Lib/multiprocessing/resource_sharer.py158
-rw-r--r--Lib/multiprocessing/semaphore_tracker.py143
-rw-r--r--Lib/multiprocessing/sharedctypes.py40
-rw-r--r--Lib/multiprocessing/spawn.py287
-rw-r--r--Lib/multiprocessing/synchronize.py111
-rw-r--r--Lib/multiprocessing/util.py54
-rw-r--r--Lib/netrc.py5
-rw-r--r--Lib/nntplib.py18
-rw-r--r--Lib/ntpath.py60
-rw-r--r--Lib/nturl2path.py4
-rw-r--r--Lib/opcode.py17
-rw-r--r--Lib/operator.py411
-rw-r--r--Lib/os.py99
-rw-r--r--Lib/os2emxpath.py158
-rw-r--r--Lib/pathlib.py1288
-rwxr-xr-xLib/pdb.py47
-rw-r--r--Lib/pickle.py862
-rw-r--r--Lib/pickletools.py871
-rw-r--r--Lib/pkgutil.py157
-rw-r--r--Lib/plat-os2emx/IN.py82
-rw-r--r--Lib/plat-os2emx/SOCKET.py106
-rw-r--r--Lib/plat-os2emx/_emx_link.py79
-rw-r--r--Lib/plat-os2emx/grp.py182
-rw-r--r--Lib/plat-os2emx/pwd.py208
-rwxr-xr-xLib/plat-os2emx/regen7
-rwxr-xr-xLib/platform.py311
-rw-r--r--Lib/plistlib.py1111
-rw-r--r--Lib/poplib.py102
-rw-r--r--Lib/posixpath.py57
-rw-r--r--Lib/pprint.py124
-rwxr-xr-xLib/profile.py63
-rw-r--r--Lib/pstats.py12
-rw-r--r--Lib/pty.py22
-rw-r--r--Lib/py_compile.py77
-rw-r--r--Lib/pyclbr.py11
-rwxr-xr-xLib/pydoc.py244
-rw-r--r--Lib/pydoc_data/topics.py20
-rwxr-xr-xLib/quopri.py2
-rw-r--r--Lib/random.py16
-rw-r--r--Lib/re.py29
-rw-r--r--Lib/rlcompleter.py5
-rw-r--r--Lib/runpy.py113
-rw-r--r--Lib/sched.py11
-rw-r--r--Lib/selectors.py525
-rw-r--r--Lib/shelve.py9
-rw-r--r--Lib/shutil.py39
-rw-r--r--Lib/site.py213
-rwxr-xr-xLib/smtpd.py21
-rwxr-xr-xLib/smtplib.py33
-rw-r--r--Lib/sndhdr.py17
-rw-r--r--Lib/socket.py116
-rw-r--r--Lib/socketserver.py9
-rw-r--r--Lib/sqlite3/test/dbapi.py18
-rw-r--r--Lib/sre_compile.py272
-rw-r--r--Lib/sre_constants.py8
-rw-r--r--Lib/sre_parse.py77
-rw-r--r--Lib/ssl.py401
-rw-r--r--Lib/stat.py6
-rw-r--r--Lib/statistics.py595
-rw-r--r--Lib/struct.py1
-rw-r--r--Lib/subprocess.py372
-rw-r--r--Lib/sunau.py40
-rwxr-xr-xLib/symbol.py4
-rw-r--r--Lib/symtable.py3
-rw-r--r--Lib/sysconfig.py46
-rwxr-xr-xLib/tabnanny.py2
-rwxr-xr-xLib/tarfile.py149
-rw-r--r--Lib/telnetlib.py275
-rw-r--r--Lib/tempfile.py107
-rw-r--r--Lib/test/__main__.py14
-rw-r--r--Lib/test/_test_multiprocessing.py (renamed from Lib/test/test_multiprocessing.py)602
-rw-r--r--Lib/test/audiodata/pluck-pcm24.aubin0 -> 19866 bytes-rw-r--r--Lib/test/audiotests.py119
-rw-r--r--Lib/test/badsyntax_future10.py3
-rw-r--r--Lib/test/buffer_tests.py16
-rw-r--r--Lib/test/bytecode_helper.py41
-rw-r--r--Lib/test/coding20731.py8
-rw-r--r--Lib/test/datetimetester.py4
-rw-r--r--Lib/test/final_a.py19
-rw-r--r--Lib/test/final_b.py19
-rw-r--r--Lib/test/fork_wait.py2
-rw-r--r--Lib/test/keycert3.pem73
-rw-r--r--Lib/test/keycert4.pem73
-rw-r--r--Lib/test/leakers/test_gestalt.py14
-rw-r--r--Lib/test/lock_tests.py20
-rw-r--r--Lib/test/make_ssl_certs.py118
-rw-r--r--Lib/test/mock_socket.py8
-rw-r--r--Lib/test/mp_fork_bomb.py5
-rw-r--r--Lib/test/multibytecodec_support.py2
-rw-r--r--Lib/test/pickletester.py659
-rw-r--r--Lib/test/pycacert.pem78
-rw-r--r--Lib/test/pycakey.pem28
-rwxr-xr-xLib/test/regrtest.py1153
-rw-r--r--Lib/test/revocation.crl11
-rw-r--r--Lib/test/script_helper.py42
-rw-r--r--Lib/test/sortperf.py6
-rw-r--r--Lib/test/ssl_servers.py10
-rw-r--r--Lib/test/ssltests.py22
-rw-r--r--Lib/test/string_tests.py30
-rw-r--r--Lib/test/subprocessdata/fd_status.py26
-rw-r--r--Lib/test/support/__init__.py253
-rw-r--r--Lib/test/test___all__.py30
-rw-r--r--Lib/test/test__opcode.py23
-rw-r--r--Lib/test/test_abc.py16
-rw-r--r--Lib/test/test_aifc.py26
-rw-r--r--Lib/test/test_argparse.py36
-rw-r--r--Lib/test/test_array.py16
-rw-r--r--Lib/test/test_ast.py100
-rw-r--r--Lib/test/test_asynchat.py35
-rw-r--r--Lib/test/test_asyncio/__init__.py29
-rw-r--r--Lib/test/test_asyncio/__main__.py5
-rw-r--r--Lib/test/test_asyncio/echo.py8
-rw-r--r--Lib/test/test_asyncio/echo2.py6
-rw-r--r--Lib/test/test_asyncio/echo3.py11
-rw-r--r--Lib/test/test_asyncio/keycert3.pem73
-rw-r--r--Lib/test/test_asyncio/pycacert.pem78
-rw-r--r--Lib/test/test_asyncio/ssl_cert.pem15
-rw-r--r--Lib/test/test_asyncio/ssl_key.pem16
-rw-r--r--Lib/test/test_asyncio/test_base_events.py949
-rw-r--r--Lib/test/test_asyncio/test_events.py2044
-rw-r--r--Lib/test/test_asyncio/test_futures.py351
-rw-r--r--Lib/test/test_asyncio/test_locks.py870
-rw-r--r--Lib/test/test_asyncio/test_proactor_events.py490
-rw-r--r--Lib/test/test_asyncio/test_queues.py466
-rw-r--r--Lib/test/test_asyncio/test_selector_events.py1690
-rw-r--r--Lib/test/test_asyncio/test_streams.py588
-rw-r--r--Lib/test/test_asyncio/test_subprocess.py184
-rw-r--r--Lib/test/test_asyncio/test_tasks.py1677
-rw-r--r--Lib/test/test_asyncio/test_transports.py88
-rw-r--r--Lib/test/test_asyncio/test_unix_events.py1580
-rw-r--r--Lib/test/test_asyncio/test_windows_events.py134
-rw-r--r--Lib/test/test_asyncio/test_windows_utils.py160
-rw-r--r--Lib/test/test_asyncio/tests.txt14
-rw-r--r--Lib/test/test_asyncore.py25
-rw-r--r--Lib/test/test_atexit.py41
-rw-r--r--Lib/test/test_audioop.py243
-rw-r--r--Lib/test/test_base64.py362
-rw-r--r--Lib/test/test_bisect.py10
-rw-r--r--Lib/test/test_buffer.py6
-rw-r--r--Lib/test/test_builtin.py128
-rw-r--r--Lib/test/test_bytes.py34
-rw-r--r--Lib/test/test_bz2.py218
-rw-r--r--Lib/test/test_capi.py161
-rw-r--r--Lib/test/test_cmd_line.py100
-rw-r--r--Lib/test/test_cmd_line_script.py47
-rw-r--r--Lib/test/test_code_module.py14
-rw-r--r--Lib/test/test_codeccallbacks.py4
-rw-r--r--Lib/test/test_codecs.py389
-rw-r--r--Lib/test/test_coding.py71
-rw-r--r--Lib/test/test_collections.py128
-rw-r--r--Lib/test/test_colorsys.py32
-rw-r--r--Lib/test/test_compileall.py54
-rw-r--r--Lib/test/test_complex.py3
-rw-r--r--Lib/test/test_concurrent_futures.py22
-rw-r--r--Lib/test/test_contextlib.py141
-rw-r--r--Lib/test/test_cprofile.py3
-rw-r--r--Lib/test/test_crypt.py6
-rw-r--r--Lib/test/test_csv.py30
-rw-r--r--Lib/test/test_curses.py4
-rw-r--r--Lib/test/test_dbm.py2
-rw-r--r--Lib/test/test_dbm_dumb.py13
-rw-r--r--Lib/test/test_dbm_gnu.py11
-rw-r--r--Lib/test/test_dbm_ndbm.py13
-rw-r--r--Lib/test/test_deque.py4
-rw-r--r--Lib/test/test_descr.py671
-rw-r--r--Lib/test/test_devpoll.py37
-rw-r--r--Lib/test/test_dis.py489
-rw-r--r--Lib/test/test_doctest.py317
-rw-r--r--Lib/test/test_dynamicclassattribute.py304
-rw-r--r--Lib/test/test_email/__init__.py26
-rw-r--r--Lib/test/test_email/test_contentmanager.py796
-rw-r--r--Lib/test/test_email/test_email.py50
-rw-r--r--Lib/test/test_email/test_headerregistry.py14
-rw-r--r--Lib/test/test_email/test_message.py758
-rw-r--r--Lib/test/test_email/test_policy.py10
-rw-r--r--Lib/test/test_email/torture_test.py2
-rw-r--r--Lib/test/test_ensurepip.py345
-rw-r--r--Lib/test/test_enum.py1594
-rw-r--r--Lib/test/test_enumerate.py10
-rw-r--r--Lib/test/test_epoll.py72
-rw-r--r--Lib/test/test_exceptions.py23
-rw-r--r--Lib/test/test_faulthandler.py77
-rw-r--r--Lib/test/test_fcntl.py39
-rw-r--r--Lib/test/test_file.py6
-rw-r--r--Lib/test/test_filecmp.py23
-rw-r--r--Lib/test/test_fileinput.py36
-rw-r--r--Lib/test/test_fileio.py20
-rw-r--r--Lib/test/test_finalization.py522
-rw-r--r--Lib/test/test_float.py1
-rw-r--r--Lib/test/test_fork1.py2
-rw-r--r--Lib/test/test_format.py36
-rw-r--r--Lib/test/test_fractions.py32
-rw-r--r--Lib/test/test_frame.py116
-rw-r--r--Lib/test/test_frozen.py79
-rw-r--r--Lib/test/test_ftplib.py91
-rw-r--r--Lib/test/test_funcattrs.py7
-rw-r--r--Lib/test/test_functools.py945
-rw-r--r--Lib/test/test_future.py8
-rw-r--r--Lib/test/test_gc.py94
-rw-r--r--Lib/test/test_gdb.py31
-rw-r--r--Lib/test/test_generators.py78
-rw-r--r--Lib/test/test_genericpath.py82
-rw-r--r--Lib/test/test_genexps.py4
-rw-r--r--Lib/test/test_getargs2.py112
-rw-r--r--Lib/test/test_glob.py22
-rw-r--r--Lib/test/test_grammar.py7
-rw-r--r--Lib/test/test_gzip.py51
-rw-r--r--Lib/test/test_hash.py146
-rw-r--r--Lib/test/test_hashlib.py154
-rw-r--r--Lib/test/test_hmac.py99
-rw-r--r--Lib/test/test_html.py86
-rw-r--r--Lib/test/test_htmlparser.py98
-rw-r--r--Lib/test/test_httplib.py30
-rw-r--r--Lib/test/test_httpservers.py21
-rw-r--r--Lib/test/test_imaplib.py34
-rw-r--r--Lib/test/test_imp.py89
-rw-r--r--Lib/test/test_import.py102
-rw-r--r--Lib/test/test_importhooks.py250
-rw-r--r--Lib/test/test_importlib/__main__.py13
-rw-r--r--Lib/test/test_importlib/abc.py9
-rw-r--r--Lib/test/test_importlib/builtin/test_finder.py60
-rw-r--r--Lib/test/test_importlib/builtin/test_loader.py47
-rw-r--r--Lib/test/test_importlib/extension/test_case_sensitivity.py29
-rw-r--r--Lib/test/test_importlib/extension/test_finder.py25
-rw-r--r--Lib/test/test_importlib/extension/test_loader.py30
-rw-r--r--Lib/test/test_importlib/extension/test_path_hook.py20
-rw-r--r--Lib/test/test_importlib/extension/util.py1
-rw-r--r--Lib/test/test_importlib/frozen/test_finder.py49
-rw-r--r--Lib/test/test_importlib/frozen/test_loader.py156
-rw-r--r--Lib/test/test_importlib/import_/test___loader__.py70
-rw-r--r--Lib/test/test_importlib/import_/test___package__.py56
-rw-r--r--Lib/test/test_importlib/import_/test_api.py73
-rw-r--r--Lib/test/test_importlib/import_/test_caching.py32
-rw-r--r--Lib/test/test_importlib/import_/test_fromlist.py36
-rw-r--r--Lib/test/test_importlib/import_/test_meta_path.py66
-rw-r--r--Lib/test/test_importlib/import_/test_packages.py44
-rw-r--r--Lib/test/test_importlib/import_/test_path.py76
-rw-r--r--Lib/test/test_importlib/import_/test_relative_imports.py53
-rw-r--r--Lib/test/test_importlib/import_/util.py20
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py (renamed from Lib/test/namespace_pkgs/both_portions/foo/one.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py (renamed from Lib/test/namespace_pkgs/both_portions/foo/two.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/missing_directory.zip (renamed from Lib/test/namespace_pkgs/missing_directory.zip)bin515 -> 515 bytes-rw-r--r--Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py (renamed from Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty (renamed from Lib/test/namespace_pkgs/module_and_namespace_package/a_test/empty)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip (renamed from Lib/test/namespace_pkgs/nested_portion1.zip)bin556 -> 556 bytes-rw-r--r--Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py (renamed from Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py (renamed from Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py (renamed from Lib/test/namespace_pkgs/portion1/foo/one.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py (renamed from Lib/test/namespace_pkgs/portion2/foo/two.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py (renamed from Lib/test/namespace_pkgs/project1/parent/child/one.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py (renamed from Lib/test/namespace_pkgs/project2/parent/child/two.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py (renamed from Lib/test/namespace_pkgs/project3/parent/child/three.py)0
-rw-r--r--Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip (renamed from Lib/test/namespace_pkgs/top_level_portion1.zip)bin332 -> 332 bytes-rw-r--r--Lib/test/test_importlib/source/test_abc_loader.py906
-rw-r--r--Lib/test/test_importlib/source/test_case_sensitivity.py53
-rw-r--r--Lib/test/test_importlib/source/test_file_loader.py230
-rw-r--r--Lib/test/test_importlib/source/test_finder.py83
-rw-r--r--Lib/test/test_importlib/source/test_path_hook.py18
-rw-r--r--Lib/test/test_importlib/source/test_source_encoding.py66
-rw-r--r--Lib/test/test_importlib/source/util.py1
-rw-r--r--Lib/test/test_importlib/test_abc.py930
-rw-r--r--Lib/test/test_importlib/test_api.py330
-rw-r--r--Lib/test/test_importlib/test_locks.py71
-rw-r--r--Lib/test/test_importlib/test_namespace_pkgs.py (renamed from Lib/test/test_namespace_pkgs.py)13
-rw-r--r--Lib/test/test_importlib/test_spec.py957
-rw-r--r--Lib/test/test_importlib/test_util.py450
-rw-r--r--Lib/test/test_importlib/test_windows.py29
-rw-r--r--Lib/test/test_importlib/util.py105
-rw-r--r--Lib/test/test_index.py3
-rw-r--r--Lib/test/test_inspect.py842
-rw-r--r--Lib/test/test_int.py50
-rw-r--r--Lib/test/test_io.py170
-rw-r--r--Lib/test/test_ioctl.py6
-rw-r--r--Lib/test/test_ipaddress.py12
-rw-r--r--Lib/test/test_iterlen.py62
-rw-r--r--Lib/test/test_itertools.py48
-rw-r--r--Lib/test/test_json/test_decode.py23
-rw-r--r--Lib/test/test_json/test_enum.py120
-rw-r--r--Lib/test/test_json/test_fail.py89
-rw-r--r--Lib/test/test_json/test_indent.py4
-rw-r--r--Lib/test/test_keyword.py138
-rw-r--r--Lib/test/test_keywordonlyarg.py12
-rw-r--r--Lib/test/test_kqueue.py29
-rw-r--r--Lib/test/test_largefile.py2
-rw-r--r--Lib/test/test_locale.py42
-rw-r--r--Lib/test/test_logging.py472
-rw-r--r--Lib/test/test_long.py2
-rw-r--r--Lib/test/test_lzma.py46
-rw-r--r--Lib/test/test_mailbox.py2
-rw-r--r--Lib/test/test_marshal.py163
-rw-r--r--Lib/test/test_memoryio.py12
-rw-r--r--Lib/test/test_memoryview.py9
-rw-r--r--Lib/test/test_minidom.py65
-rw-r--r--Lib/test/test_mmap.py26
-rw-r--r--Lib/test/test_module.py64
-rw-r--r--Lib/test/test_multibytecodec.py73
-rw-r--r--Lib/test/test_multiprocessing_fork.py7
-rw-r--r--Lib/test/test_multiprocessing_forkserver.py7
-rw-r--r--Lib/test/test_multiprocessing_main_handling.py285
-rw-r--r--Lib/test/test_multiprocessing_spawn.py7
-rw-r--r--Lib/test/test_nntplib.py2
-rw-r--r--Lib/test/test_normalization.py2
-rw-r--r--Lib/test/test_ntpath.py34
-rw-r--r--Lib/test/test_openpty.py2
-rw-r--r--Lib/test/test_operator.py122
-rw-r--r--Lib/test/test_optparse.py8
-rw-r--r--Lib/test/test_os.py329
-rw-r--r--Lib/test/test_ossaudiodev.py4
-rw-r--r--Lib/test/test_pathlib.py1840
-rw-r--r--Lib/test/test_pdb.py348
-rw-r--r--Lib/test/test_peepholer.py295
-rw-r--r--Lib/test/test_pep247.py12
-rw-r--r--Lib/test/test_pep277.py4
-rw-r--r--Lib/test/test_pep3151.py2
-rw-r--r--Lib/test/test_pep352.py5
-rw-r--r--Lib/test/test_pickle.py4
-rw-r--r--Lib/test/test_pkg.py20
-rw-r--r--Lib/test/test_pkgimport.py2
-rw-r--r--Lib/test/test_pkgutil.py62
-rw-r--r--Lib/test/test_plistlib.py533
-rw-r--r--Lib/test/test_poll.py20
-rw-r--r--Lib/test/test_poplib.py190
-rw-r--r--Lib/test/test_posix.py85
-rw-r--r--Lib/test/test_posixpath.py62
-rw-r--r--Lib/test/test_pprint.py50
-rw-r--r--Lib/test/test_print.py74
-rw-r--r--Lib/test/test_profile.py29
-rw-r--r--Lib/test/test_pty.py2
-rw-r--r--Lib/test/test_py_compile.py46
-rw-r--r--Lib/test/test_pyclbr.py4
-rw-r--r--Lib/test/test_pydoc.py327
-rw-r--r--Lib/test/test_pyexpat.py12
-rw-r--r--Lib/test/test_random.py207
-rw-r--r--Lib/test/test_range.py9
-rw-r--r--Lib/test/test_re.py240
-rw-r--r--Lib/test/test_regrtest.py275
-rw-r--r--Lib/test/test_reprlib.py6
-rw-r--r--Lib/test/test_resource.py32
-rw-r--r--Lib/test/test_runpy.py183
-rw-r--r--Lib/test/test_sched.py5
-rw-r--r--Lib/test/test_scope.py17
-rw-r--r--Lib/test/test_select.py4
-rw-r--r--Lib/test/test_selectors.py453
-rw-r--r--Lib/test/test_set.py2
-rw-r--r--Lib/test/test_shelve.py13
-rw-r--r--Lib/test/test_shutil.py43
-rw-r--r--Lib/test/test_signal.py60
-rw-r--r--Lib/test/test_site.py41
-rw-r--r--Lib/test/test_slice.py114
-rw-r--r--Lib/test/test_smtplib.py23
-rw-r--r--Lib/test/test_smtpnet.py30
-rw-r--r--Lib/test/test_sndhdr.py2
-rw-r--r--Lib/test/test_socket.py392
-rw-r--r--Lib/test/test_socketserver.py20
-rw-r--r--Lib/test/test_source_encoding.py (renamed from Lib/test/test_pep263.py)77
-rw-r--r--Lib/test/test_ssl.py835
-rw-r--r--Lib/test/test_stat.py61
-rw-r--r--Lib/test/test_statistics.py1574
-rw-r--r--Lib/test/test_strftime.py23
-rw-r--r--Lib/test/test_strtod.py2
-rw-r--r--Lib/test/test_struct.py89
-rw-r--r--Lib/test/test_structseq.py2
-rw-r--r--Lib/test/test_subprocess.py206
-rw-r--r--Lib/test/test_sunau.py28
-rw-r--r--Lib/test/test_sundry.py37
-rw-r--r--Lib/test/test_super.py33
-rw-r--r--Lib/test/test_support.py1
-rw-r--r--Lib/test/test_syntax.py4
-rw-r--r--Lib/test/test_sys.py106
-rw-r--r--Lib/test/test_sysconfig.py66
-rw-r--r--Lib/test/test_syslog.py4
-rw-r--r--Lib/test/test_tarfile.py191
-rw-r--r--Lib/test/test_tcl.py34
-rw-r--r--Lib/test/test_telnetlib.py106
-rw-r--r--Lib/test/test_tempfile.py57
-rw-r--r--Lib/test/test_textwrap.py159
-rw-r--r--Lib/test/test_thread.py2
-rw-r--r--Lib/test/test_threaded_import.py14
-rw-r--r--Lib/test/test_threading.py454
-rw-r--r--Lib/test/test_threadsignals.py8
-rw-r--r--Lib/test/test_time.py148
-rw-r--r--Lib/test/test_timeout.py2
-rw-r--r--Lib/test/test_tools.py9
-rw-r--r--Lib/test/test_traceback.py82
-rw-r--r--Lib/test/test_tracemalloc.py820
-rw-r--r--Lib/test/test_types.py37
-rw-r--r--Lib/test/test_ucn.py2
-rw-r--r--Lib/test/test_unicode.py403
-rw-r--r--Lib/test/test_unicodedata.py4
-rw-r--r--Lib/test/test_urllib.py103
-rw-r--r--Lib/test/test_urllib2.py136
-rw-r--r--Lib/test/test_urllib2_localnet.py27
-rw-r--r--Lib/test/test_urllib2net.py54
-rw-r--r--Lib/test/test_urllibnet.py45
-rw-r--r--Lib/test/test_urlparse.py8
-rw-r--r--Lib/test/test_userdict.py9
-rw-r--r--Lib/test/test_venv.py209
-rw-r--r--Lib/test/test_wait3.py12
-rw-r--r--Lib/test/test_warnings.py32
-rw-r--r--Lib/test/test_wave.py21
-rw-r--r--Lib/test/test_weakref.py316
-rw-r--r--Lib/test/test_winreg.py28
-rw-r--r--Lib/test/test_winsound.py2
-rw-r--r--Lib/test/test_xml_etree.py200
-rw-r--r--Lib/test/test_xml_etree_c.py9
-rw-r--r--Lib/test/test_xmlrpc.py74
-rw-r--r--Lib/test/test_xmlrpc_net.py30
-rw-r--r--Lib/test/test_zipfile.py254
-rw-r--r--Lib/test/test_zipfile64.py4
-rw-r--r--Lib/test/test_zipimport.py37
-rw-r--r--Lib/test/test_zipimport_support.py6
-rw-r--r--Lib/test/tf_inherit_check.py2
-rw-r--r--Lib/textwrap.py73
-rw-r--r--Lib/threading.py191
-rwxr-xr-xLib/timeit.py46
-rw-r--r--Lib/tkinter/__init__.py47
-rw-r--r--Lib/tkinter/filedialog.py4
-rw-r--r--Lib/tkinter/test/test_tkinter/test_widgets.py4
-rw-r--r--Lib/tkinter/tix.py2
-rw-r--r--Lib/token.py6
-rw-r--r--Lib/tokenize.py2
-rwxr-xr-xLib/trace.py10
-rw-r--r--Lib/traceback.py259
-rw-r--r--Lib/tracemalloc.py487
-rw-r--r--Lib/turtle.py22
-rwxr-xr-xLib/turtledemo/__main__.py2
-rw-r--r--Lib/types.py57
-rw-r--r--Lib/unittest/__main__.py3
-rw-r--r--Lib/unittest/case.py393
-rw-r--r--Lib/unittest/loader.py76
-rw-r--r--Lib/unittest/main.py247
-rw-r--r--Lib/unittest/mock.py248
-rw-r--r--Lib/unittest/result.py27
-rw-r--r--Lib/unittest/suite.py31
-rw-r--r--Lib/unittest/test/__main__.py18
-rw-r--r--Lib/unittest/test/support.py38
-rw-r--r--Lib/unittest/test/test_assertions.py4
-rw-r--r--Lib/unittest/test/test_break.py4
-rw-r--r--Lib/unittest/test/test_case.py267
-rw-r--r--Lib/unittest/test/test_discovery.py169
-rw-r--r--Lib/unittest/test/test_functiontestcase.py4
-rw-r--r--Lib/unittest/test/test_loader.py4
-rw-r--r--Lib/unittest/test/test_program.py72
-rw-r--r--Lib/unittest/test/test_result.py92
-rw-r--r--Lib/unittest/test/test_runner.py16
-rw-r--r--Lib/unittest/test/test_setups.py7
-rw-r--r--Lib/unittest/test/test_skipping.py80
-rw-r--r--Lib/unittest/test/test_suite.py81
-rw-r--r--Lib/unittest/test/testmock/__main__.py18
-rw-r--r--Lib/unittest/test/testmock/testcallable.py4
-rw-r--r--Lib/unittest/test/testmock/testhelpers.py59
-rw-r--r--Lib/unittest/test/testmock/testmock.py143
-rw-r--r--Lib/unittest/test/testmock/testpatch.py1
-rw-r--r--Lib/unittest/test/testmock/testwith.py83
-rw-r--r--Lib/unittest/util.py37
-rw-r--r--Lib/urllib/error.py14
-rw-r--r--Lib/urllib/parse.py18
-rw-r--r--Lib/urllib/request.py195
-rw-r--r--Lib/uuid.py11
-rw-r--r--Lib/venv/__init__.py80
-rw-r--r--Lib/venv/scripts/nt/Activate.ps145
-rw-r--r--Lib/venv/scripts/nt/Deactivate.ps119
-rw-r--r--Lib/venv/scripts/nt/activate.bat2
-rw-r--r--Lib/venv/scripts/posix/activate4
-rw-r--r--Lib/venv/scripts/posix/activate.csh37
-rw-r--r--Lib/venv/scripts/posix/activate.fish74
-rw-r--r--Lib/warnings.py8
-rw-r--r--Lib/wave.py88
-rw-r--r--Lib/weakref.py198
-rwxr-xr-xLib/webbrowser.py95
-rw-r--r--Lib/wsgiref/__init__.py2
-rw-r--r--Lib/xml/dom/expatbuilder.py10
-rw-r--r--Lib/xml/etree/ElementInclude.py6
-rw-r--r--Lib/xml/etree/ElementPath.py15
-rw-r--r--Lib/xml/etree/ElementTree.py1431
-rw-r--r--Lib/xmlrpc/client.py10
-rw-r--r--Lib/xmlrpc/server.py7
-rw-r--r--Lib/zipfile.py202
-rw-r--r--Mac/BuildScript/README.txt12
-rwxr-xr-xMac/BuildScript/build-installer.py101
-rw-r--r--Mac/BuildScript/resources/ReadMe.txt30
-rw-r--r--Mac/BuildScript/resources/Welcome.rtf18
-rwxr-xr-xMac/BuildScript/scripts/postflight.ensurepip65
-rw-r--r--Mac/Makefile.in129
-rw-r--r--Mac/README10
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/Help_Indexing_Tool_Suite.py110
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/Miscellaneous_Standards.py49
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/Required_Suite.py32
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/Standard_Suite.py343
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/__init__.py78
-rw-r--r--Mac/Tools/Doc/HelpIndexingTool/odds_and_ends.py49
-rw-r--r--Mac/Tools/Doc/README35
-rw-r--r--Mac/Tools/Doc/setup.py213
-rwxr-xr-xMac/Tools/bundlebuilder.py2
-rwxr-xr-xMac/Tools/plistlib_generate_testdata.py103
-rw-r--r--Makefile.pre.in171
-rw-r--r--Misc/ACKS68
-rw-r--r--Misc/HISTORY4101
-rw-r--r--Misc/NEWS3559
-rw-r--r--Misc/README4
-rw-r--r--Misc/RPM/python-3.4.spec (renamed from Misc/RPM/python-3.3.spec)7
-rw-r--r--Misc/SpecialBuilds.txt12
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/2 to 3 - Module Deletion.tmCommand64
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/Build Docs.tmCommand23
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/Build.tmCommand25
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/Go to Issue.tmCommand20
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/Open Docs.tmCommand32
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Commands/Open PEP.tmCommand23
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/Snippets/2 to 3 - Module Deletion (docs).tmSnippet16
-rw-r--r--Misc/TextMate/Python-Dev.tmbundle/info.plist37
-rw-r--r--Misc/Vim/python.vim148
-rw-r--r--Misc/Vim/syntax_test.py62
-rw-r--r--Misc/Vim/vim_syntax.py229
-rw-r--r--Misc/Vim/vimrc87
-rw-r--r--Misc/coverity_model.c132
-rw-r--r--Misc/python-config.in2
-rw-r--r--Misc/python-config.sh.in111
-rw-r--r--Misc/python-wing5.wpr18
-rw-r--r--Misc/python.man14
-rw-r--r--Modules/Setup.dist22
-rw-r--r--Modules/_bisectmodule.c5
-rw-r--r--Modules/_bz2module.c196
-rw-r--r--Modules/_codecsmodule.c6
-rw-r--r--Modules/_collectionsmodule.c412
-rw-r--r--Modules/_cryptmodule.c82
-rw-r--r--Modules/_csv.c18
-rw-r--r--Modules/_ctypes/_ctypes.c80
-rw-r--r--Modules/_ctypes/callbacks.c23
-rw-r--r--Modules/_ctypes/callproc.c105
-rw-r--r--Modules/_ctypes/libffi.diff36
-rw-r--r--Modules/_ctypes/libffi/src/dlmalloc.c8
-rw-r--r--Modules/_ctypes/libffi/testsuite/libffi.call/a.out0
-rw-r--r--Modules/_ctypes/libffi_osx/ffi.c11
-rw-r--r--Modules/_ctypes/libffi_osx/x86/x86-ffi64.c11
-rw-r--r--Modules/_ctypes/stgdict.c4
-rw-r--r--Modules/_curses_panel.c8
-rw-r--r--Modules/_cursesmodule.c258
-rw-r--r--Modules/_datetimemodule.c191
-rw-r--r--Modules/_dbmmodule.c196
-rw-r--r--Modules/_elementtree.c426
-rw-r--r--Modules/_freeze_importlib.c6
-rw-r--r--Modules/_gdbmmodule.c38
-rw-r--r--Modules/_gestalt.c84
-rw-r--r--Modules/_hashopenssl.c254
-rw-r--r--Modules/_heapqmodule.c5
-rw-r--r--Modules/_io/_iomodule.c49
-rw-r--r--Modules/_io/_iomodule.h7
-rw-r--r--Modules/_io/bufferedio.c89
-rw-r--r--Modules/_io/fileio.c209
-rw-r--r--Modules/_io/iobase.c129
-rw-r--r--Modules/_io/textio.c114
-rw-r--r--Modules/_json.c169
-rw-r--r--Modules/_localemodule.c39
-rw-r--r--Modules/_lsprof.c32
-rw-r--r--Modules/_lzmamodule.c377
-rw-r--r--Modules/_multiprocessing/multiprocessing.c32
-rw-r--r--Modules/_multiprocessing/multiprocessing.h16
-rw-r--r--Modules/_multiprocessing/semaphore.c98
-rw-r--r--Modules/_opcode.c109
-rw-r--r--Modules/_operator.c (renamed from Modules/operator.c)125
-rw-r--r--Modules/_pickle.c3194
-rw-r--r--Modules/_posixsubprocess.c178
-rw-r--r--Modules/_randommodule.c109
-rw-r--r--Modules/_sqlite/cache.c1
-rw-r--r--Modules/_sqlite/connection.c109
-rw-r--r--Modules/_sqlite/cursor.c17
-rw-r--r--Modules/_sqlite/module.c16
-rw-r--r--Modules/_sqlite/prepare_protocol.c1
-rw-r--r--Modules/_sqlite/row.c14
-rw-r--r--Modules/_sqlite/sqlitecompat.h63
-rw-r--r--Modules/_sqlite/statement.c33
-rw-r--r--Modules/_sqlite/util.c2
-rw-r--r--Modules/_sre.c1911
-rw-r--r--Modules/_ssl.c1458
-rw-r--r--Modules/_stat.c563
-rw-r--r--Modules/_struct.c359
-rw-r--r--Modules/_testbuffer.c62
-rw-r--r--Modules/_testcapimodule.c486
-rw-r--r--Modules/_testembed.c94
-rw-r--r--Modules/_testimportmultiple.c57
-rw-r--r--Modules/_threadmodule.c98
-rw-r--r--Modules/_tkinter.c666
-rw-r--r--Modules/_tracemalloc.c1429
-rw-r--r--Modules/_weakref.c63
-rw-r--r--Modules/_winapi.c44
-rw-r--r--Modules/arraymodule.c122
-rw-r--r--Modules/atexitmodule.c121
-rw-r--r--Modules/audioop.c1299
-rw-r--r--Modules/binascii.c658
-rw-r--r--Modules/cjkcodecs/_codecs_cn.c293
-rw-r--r--Modules/cjkcodecs/_codecs_hk.c93
-rw-r--r--Modules/cjkcodecs/_codecs_iso2022.c475
-rw-r--r--Modules/cjkcodecs/_codecs_jp.c459
-rw-r--r--Modules/cjkcodecs/_codecs_kr.c204
-rw-r--r--Modules/cjkcodecs/_codecs_tw.c93
-rw-r--r--Modules/cjkcodecs/alg_jisx0201.h75
-rw-r--r--Modules/cjkcodecs/cjkcodecs.h203
-rw-r--r--Modules/cjkcodecs/emu_jisx0213_2000.h33
-rw-r--r--Modules/cjkcodecs/mappings_cn.h2
-rw-r--r--Modules/cjkcodecs/mappings_jisx0213_pair.h2
-rw-r--r--Modules/cjkcodecs/multibytecodec.c344
-rw-r--r--Modules/cjkcodecs/multibytecodec.h27
-rw-r--r--Modules/clinic/_bz2module.c.h162
-rw-r--r--Modules/clinic/_lzmamodule.c.h245
-rw-r--r--Modules/clinic/_pickle.c.h491
-rw-r--r--Modules/clinic/audioop.c.h889
-rw-r--r--Modules/clinic/binascii.c.h478
-rw-r--r--Modules/clinic/zlibmodule.c.h453
-rw-r--r--Modules/cmathmodule.c2
-rw-r--r--Modules/expat/xmlparse.c12
-rw-r--r--Modules/expat/xmlrole.c4
-rw-r--r--Modules/expat/xmltok.c4
-rw-r--r--Modules/faulthandler.c35
-rw-r--r--Modules/fcntlmodule.c178
-rw-r--r--Modules/fpectlmodule.c29
-rw-r--r--Modules/gcmodule.c289
-rw-r--r--Modules/getpath.c136
-rw-r--r--Modules/grpmodule.c16
-rw-r--r--Modules/hashlib.h33
-rw-r--r--Modules/hashtable.c518
-rw-r--r--Modules/hashtable.h128
-rw-r--r--Modules/itertoolsmodule.c55
-rw-r--r--Modules/main.c102
-rw-r--r--Modules/md5module.c21
-rw-r--r--Modules/mmapmodule.c22
-rw-r--r--Modules/ossaudiodev.c13
-rw-r--r--Modules/overlapped.c1380
-rw-r--r--Modules/parsermodule.c154
-rw-r--r--Modules/posixmodule.c2708
-rw-r--r--Modules/pwdmodule.c17
-rw-r--r--Modules/pyexpat.c69
-rw-r--r--Modules/python.c27
-rw-r--r--Modules/readline.c170
-rw-r--r--Modules/resource.c191
-rw-r--r--Modules/selectmodule.c271
-rw-r--r--Modules/sha1module.c21
-rw-r--r--Modules/sha256module.c44
-rw-r--r--Modules/sha512module.c43
-rw-r--r--Modules/signalmodule.c47
-rw-r--r--Modules/socketmodule.c1351
-rw-r--r--Modules/socketmodule.h6
-rw-r--r--Modules/spwdmodule.c22
-rw-r--r--Modules/sre.h10
-rw-r--r--Modules/sre_constants.h2
-rw-r--r--Modules/sre_lib.h1343
-rw-r--r--Modules/symtablemodule.c42
-rw-r--r--Modules/syslogmodule.c66
-rw-r--r--Modules/timemodule.c103
-rw-r--r--Modules/unicodedata.c77
-rw-r--r--Modules/unicodedata_db.h1988
-rw-r--r--Modules/unicodename_db.h33019
-rw-r--r--Modules/xxlimited.c2
-rw-r--r--Modules/zipimport.c68
-rw-r--r--Modules/zlib/ChangeLog266
-rw-r--r--Modules/zlib/FAQ12
-rw-r--r--Modules/zlib/INDEX13
-rw-r--r--Modules/zlib/Makefile.in111
-rw-r--r--Modules/zlib/README24
-rw-r--r--Modules/zlib/adler32.c68
-rw-r--r--Modules/zlib/algorithm.txt4
-rw-r--r--Modules/zlib/compress.c2
-rwxr-xr-xModules/zlib/configure521
-rw-r--r--Modules/zlib/crc32.c83
-rw-r--r--Modules/zlib/crc32.h2
-rw-r--r--Modules/zlib/deflate.c267
-rw-r--r--Modules/zlib/deflate.h12
-rw-r--r--Modules/zlib/example.c90
-rw-r--r--Modules/zlib/gzguts.h103
-rw-r--r--Modules/zlib/gzio.c1026
-rw-r--r--Modules/zlib/gzlib.c197
-rw-r--r--Modules/zlib/gzread.c431
-rw-r--r--Modules/zlib/gzwrite.c196
-rw-r--r--Modules/zlib/infback.c16
-rw-r--r--Modules/zlib/inffast.c6
-rw-r--r--Modules/zlib/inffixed.h6
-rw-r--r--Modules/zlib/inflate.c136
-rw-r--r--Modules/zlib/inftrees.c54
-rw-r--r--Modules/zlib/inftrees.h2
-rw-r--r--Modules/zlib/make_vms.com403
-rw-r--r--Modules/zlib/minigzip.c213
-rw-r--r--Modules/zlib/trees.c54
-rw-r--r--Modules/zlib/uncompr.c2
-rw-r--r--Modules/zlib/zconf.h201
-rw-r--r--Modules/zlib/zconf.h.cmakein201
-rw-r--r--Modules/zlib/zconf.h.in201
-rw-r--r--Modules/zlib/zconf.in.h332
-rw-r--r--Modules/zlib/zlib.318
-rw-r--r--Modules/zlib/zlib.h345
-rw-r--r--Modules/zlib/zlib.map15
-rw-r--r--Modules/zlib/zutil.c26
-rw-r--r--Modules/zlib/zutil.h103
-rw-r--r--Modules/zlibmodule.c710
-rw-r--r--Objects/abstract.c144
-rw-r--r--Objects/bytearrayobject.c440
-rw-r--r--Objects/bytes_methods.c30
-rw-r--r--Objects/bytesobject.c269
-rw-r--r--Objects/classobject.c34
-rw-r--r--Objects/codeobject.c5
-rw-r--r--Objects/complexobject.c28
-rw-r--r--Objects/descrobject.c98
-rw-r--r--Objects/dictobject.c418
-rw-r--r--Objects/exceptions.c274
-rw-r--r--Objects/fileobject.c6
-rw-r--r--Objects/floatobject.c37
-rw-r--r--Objects/frameobject.c136
-rw-r--r--Objects/funcobject.c101
-rw-r--r--Objects/genobject.c108
-rw-r--r--Objects/iterobject.c13
-rw-r--r--Objects/listobject.c106
-rw-r--r--Objects/longobject.c198
-rw-r--r--Objects/memoryobject.c4
-rw-r--r--Objects/methodobject.c91
-rw-r--r--Objects/moduleobject.c122
-rw-r--r--Objects/namespaceobject.c62
-rw-r--r--Objects/object.c348
-rw-r--r--Objects/obmalloc.c584
-rw-r--r--Objects/rangeobject.c205
-rw-r--r--Objects/setobject.c531
-rw-r--r--Objects/sliceobject.c204
-rw-r--r--Objects/stringlib/asciilib.h1
-rw-r--r--Objects/stringlib/codecs.h116
-rw-r--r--Objects/stringlib/eq.h4
-rw-r--r--Objects/stringlib/fastsearch.h8
-rw-r--r--Objects/stringlib/find_max_char.h4
-rw-r--r--Objects/stringlib/join.h133
-rw-r--r--Objects/stringlib/partition.h10
-rw-r--r--Objects/stringlib/replace.h53
-rw-r--r--Objects/stringlib/split.h4
-rw-r--r--Objects/stringlib/stringdefs.h1
-rw-r--r--Objects/stringlib/transmogrify.h8
-rw-r--r--Objects/stringlib/ucs1lib.h1
-rw-r--r--Objects/stringlib/ucs2lib.h1
-rw-r--r--Objects/stringlib/ucs4lib.h1
-rw-r--r--Objects/stringlib/undef.h1
-rw-r--r--Objects/stringlib/unicode_format.h141
-rw-r--r--Objects/stringlib/unicodedefs.h6
-rw-r--r--Objects/structseq.c86
-rw-r--r--Objects/tupleobject.c121
-rw-r--r--Objects/typeobject.c1125
-rw-r--r--Objects/unicodectype.c2
-rw-r--r--Objects/unicodeobject.c4824
-rw-r--r--Objects/unicodetype_db.h25
-rw-r--r--Objects/weakrefobject.c7
-rw-r--r--PC/VC6/_ctypes.dsp131
-rw-r--r--PC/VC6/_ctypes_test.dsp99
-rw-r--r--PC/VC6/_elementtree.dsp111
-rw-r--r--PC/VC6/_msi.dsp99
-rw-r--r--PC/VC6/_multiprocessing.dsp107
-rw-r--r--PC/VC6/_socket.dsp99
-rw-r--r--PC/VC6/_sqlite3.dsp131
-rw-r--r--PC/VC6/_ssl.dsp89
-rw-r--r--PC/VC6/_ssl.mak22
-rw-r--r--PC/VC6/_testcapi.dsp99
-rw-r--r--PC/VC6/_tkinter.dsp103
-rw-r--r--PC/VC6/build_ssl.py228
-rw-r--r--PC/VC6/build_tkinter.py81
-rw-r--r--PC/VC6/bz2.dsp99
-rw-r--r--PC/VC6/make_versioninfo.dsp108
-rw-r--r--PC/VC6/pcbuild.dsw306
-rw-r--r--PC/VC6/pyexpat.dsp111
-rw-r--r--PC/VC6/python.dsp100
-rw-r--r--PC/VC6/pythoncore.dsp780
-rw-r--r--PC/VC6/pythonw.dsp101
-rw-r--r--PC/VC6/readme.txt192
-rw-r--r--PC/VC6/rmpyc.py25
-rwxr-xr-xPC/VC6/rt.bat41
-rw-r--r--PC/VC6/select.dsp99
-rw-r--r--PC/VC6/tcl852.patch11
-rw-r--r--PC/VC6/unicodedata.dsp99
-rw-r--r--PC/VC6/w9xpopen.dsp97
-rw-r--r--PC/VC6/winsound.dsp99
-rw-r--r--PC/VS7.1/Uninstal.wse514
-rw-r--r--PC/VS7.1/_ctypes.vcproj311
-rw-r--r--PC/VS7.1/_ctypes_test.vcproj242
-rw-r--r--PC/VS7.1/_elementtree.vcproj264
-rw-r--r--PC/VS7.1/_msi.vcproj252
-rw-r--r--PC/VS7.1/_socket.vcproj254
-rw-r--r--PC/VS7.1/_sqlite3.vcproj283
-rw-r--r--PC/VS7.1/_ssl.mak38
-rw-r--r--PC/VS7.1/_ssl.vcproj84
-rw-r--r--PC/VS7.1/_testcapi.vcproj247
-rw-r--r--PC/VS7.1/_tkinter.vcproj261
-rw-r--r--PC/VS7.1/amd64_ml64.bat17
-rw-r--r--PC/VS7.1/build_ssl.bat12
-rw-r--r--PC/VS7.1/build_ssl.py181
-rw-r--r--PC/VS7.1/bz2.vcproj271
-rw-r--r--PC/VS7.1/db.build10
-rw-r--r--PC/VS7.1/field3.py35
-rw-r--r--PC/VS7.1/installer.bmpbin58806 -> 0 bytes-rw-r--r--PC/VS7.1/make_buildinfo.c92
-rw-r--r--PC/VS7.1/make_buildinfo.vcproj122
-rw-r--r--PC/VS7.1/make_versioninfo.vcproj142
-rw-r--r--PC/VS7.1/pcbuild.sln275
-rw-r--r--PC/VS7.1/pyexpat.vcproj263
-rw-r--r--PC/VS7.1/python.build20
-rw-r--r--PC/VS7.1/python.iss341
-rw-r--r--PC/VS7.1/python.vcproj274
-rw-r--r--PC/VS7.1/python20.wse3117
-rw-r--r--PC/VS7.1/pythoncore.vcproj826
-rw-r--r--PC/VS7.1/pythonw.vcproj261
-rw-r--r--PC/VS7.1/readme.txt337
-rw-r--r--PC/VS7.1/rmpyc.py25
-rwxr-xr-xPC/VS7.1/rt.bat52
-rw-r--r--PC/VS7.1/select.vcproj258
-rw-r--r--PC/VS7.1/unicodedata.vcproj247
-rw-r--r--PC/VS7.1/w9xpopen.vcproj121
-rw-r--r--PC/VS7.1/winsound.vcproj251
-rw-r--r--PC/VS8.0/_ctypes.vcproj705
-rw-r--r--PC/VS8.0/_ctypes_test.vcproj521
-rw-r--r--PC/VS8.0/_elementtree.vcproj613
-rw-r--r--PC/VS8.0/_hashlib.vcproj537
-rw-r--r--PC/VS8.0/_msi.vcproj529
-rw-r--r--PC/VS8.0/_multiprocessing.vcproj545
-rw-r--r--PC/VS8.0/_socket.vcproj537
-rw-r--r--PC/VS8.0/_sqlite3.vcproj613
-rw-r--r--PC/VS8.0/_ssl.vcproj537
-rw-r--r--PC/VS8.0/_tkinter.vcproj541
-rw-r--r--PC/VS8.0/bdist_wininst.vcproj270
-rw-r--r--PC/VS8.0/build.bat17
-rw-r--r--PC/VS8.0/build_env.bat1
-rw-r--r--PC/VS8.0/build_pgo.bat41
-rw-r--r--PC/VS8.0/build_ssl.bat12
-rw-r--r--PC/VS8.0/build_ssl.py277
-rw-r--r--PC/VS8.0/build_tkinter.py85
-rw-r--r--PC/VS8.0/bz2.vcproj581
-rw-r--r--PC/VS8.0/debug.vsprops15
-rw-r--r--PC/VS8.0/env.bat5
-rw-r--r--PC/VS8.0/field3.py35
-rw-r--r--PC/VS8.0/idle.bat15
-rw-r--r--PC/VS8.0/kill_python.c178
-rw-r--r--PC/VS8.0/kill_python.vcproj279
-rw-r--r--PC/VS8.0/make_buildinfo.c116
-rw-r--r--PC/VS8.0/make_buildinfo.vcproj101
-rw-r--r--PC/VS8.0/make_versioninfo.vcproj324
-rw-r--r--PC/VS8.0/pcbuild.sln560
-rw-r--r--PC/VS8.0/pginstrument.vsprops34
-rw-r--r--PC/VS8.0/pgupdate.vsprops14
-rw-r--r--PC/VS8.0/pyd.vsprops28
-rw-r--r--PC/VS8.0/pyd_d.vsprops36
-rw-r--r--PC/VS8.0/pyexpat.vcproj553
-rw-r--r--PC/VS8.0/pyproject.vsprops87
-rw-r--r--PC/VS8.0/python.vcproj637
-rw-r--r--PC/VS8.0/pythoncore.vcproj1921
-rw-r--r--PC/VS8.0/pythonw.vcproj618
-rw-r--r--PC/VS8.0/release.vsprops15
-rw-r--r--PC/VS8.0/rmpyc.py25
-rw-r--r--PC/VS8.0/rt.bat52
-rw-r--r--PC/VS8.0/select.vcproj537
-rw-r--r--PC/VS8.0/sqlite3.vcproj537
-rw-r--r--PC/VS8.0/sqlite3.vsprops14
-rw-r--r--PC/VS8.0/ssl.vcproj189
-rw-r--r--PC/VS8.0/unicodedata.vcproj533
-rw-r--r--PC/VS8.0/w9xpopen.vcproj576
-rw-r--r--PC/VS8.0/winsound.vcproj523
-rw-r--r--PC/VS8.0/x64.vsprops22
-rw-r--r--PC/VS9.0/_sqlite3.vcproj4
-rw-r--r--PC/VS9.0/_ssl.vcproj18
-rw-r--r--PC/VS9.0/_testimportmultiple.vcproj (renamed from PC/VS8.0/_testcapi.vcproj)10
-rw-r--r--PC/VS9.0/kill_python.c2
-rw-r--r--PC/VS9.0/pcbuild.sln42
-rw-r--r--PC/VS9.0/pyproject.vsprops6
-rw-r--r--PC/VS9.0/pythoncore.vcproj90
-rw-r--r--PC/VS9.0/w9xpopen.vcproj576
-rw-r--r--PC/_msi.c80
-rw-r--r--PC/bdist_wininst/install.c2
-rw-r--r--PC/config.c10
-rw-r--r--PC/example_nt/example.vcproj4
-rw-r--r--PC/getpathp.c43
-rw-r--r--PC/launcher.c254
-rw-r--r--PC/msvcrtmodule.c5
-rw-r--r--PC/os2emx/Makefile672
-rw-r--r--PC/os2emx/README.os2emx663
-rw-r--r--PC/os2emx/config.c164
-rw-r--r--PC/os2emx/dlfcn.c223
-rw-r--r--PC/os2emx/dlfcn.h51
-rw-r--r--PC/os2emx/dllentry.c42
-rw-r--r--PC/os2emx/getpathp.c418
-rw-r--r--PC/os2emx/pyconfig.h332
-rw-r--r--PC/os2emx/python33.def1314
-rw-r--r--PC/os2emx/pythonpm.c124
-rw-r--r--PC/os2vacpp/_tkinter.def8
-rw-r--r--PC/os2vacpp/config.c99
-rw-r--r--PC/os2vacpp/getpathp.c482
-rw-r--r--PC/os2vacpp/makefile1549
-rw-r--r--PC/os2vacpp/makefile.omk1047
-rw-r--r--PC/os2vacpp/pyconfig.h212
-rw-r--r--PC/os2vacpp/python.def479
-rw-r--r--PC/os2vacpp/readme.txt119
-rw-r--r--PC/pyconfig.h21
-rw-r--r--PC/python.mk5
-rw-r--r--PC/python3.def1396
-rw-r--r--PC/python3.mak12
-rw-r--r--PC/python34gen.py (renamed from PC/python33gen.py)8
-rw-r--r--PC/python34stub.def (renamed from PC/python33stub.def)4
-rw-r--r--PC/readme.txt13
-rw-r--r--PC/w9xpopen.c112
-rw-r--r--PC/winreg.c12
-rw-r--r--PCbuild/_overlapped.vcxproj234
-rw-r--r--PCbuild/_overlapped.vcxproj.filters (renamed from PCbuild/w9xpopen.vcxproj.filters)4
-rw-r--r--PCbuild/_sqlite3.vcxproj1
-rw-r--r--PCbuild/_sqlite3.vcxproj.filters3
-rw-r--r--PCbuild/_ssl.vcxproj16
-rw-r--r--PCbuild/_testembed.vcxproj161
-rw-r--r--PCbuild/_testembed.vcxproj.filters22
-rw-r--r--PCbuild/_testimportmultiple.vcxproj (renamed from PCbuild/w9xpopen.vcxproj)167
-rw-r--r--PCbuild/_testimportmultiple.vcxproj.filters13
-rw-r--r--PCbuild/build_ssl.bat4
-rw-r--r--PCbuild/build_tkinter.py12
-rw-r--r--PCbuild/kill_python.c2
-rw-r--r--PCbuild/pcbuild.sln70
-rw-r--r--PCbuild/pyproject.props19
-rw-r--r--PCbuild/python.vcxproj8
-rw-r--r--PCbuild/pythoncore.vcxproj15
-rw-r--r--PCbuild/pythoncore.vcxproj.filters35
-rw-r--r--PCbuild/readme.txt487
-rw-r--r--PCbuild/xxlimited.vcxproj11
-rw-r--r--Parser/Python.asdl12
-rw-r--r--Parser/asdl.py33
-rwxr-xr-xParser/asdl_c.py69
-rw-r--r--Parser/grammar.c10
-rw-r--r--Parser/grammar1.c8
-rw-r--r--Parser/myreadline.c48
-rw-r--r--Parser/node.c4
-rw-r--r--Parser/parser.c36
-rw-r--r--Parser/parsetok.c93
-rw-r--r--Parser/pgenmain.c3
-rw-r--r--Parser/tokenizer.c29
-rw-r--r--Parser/tokenizer.h6
-rw-r--r--Python/Python-ast.c370
-rw-r--r--Python/_warnings.c245
-rw-r--r--Python/asdl.c4
-rw-r--r--Python/ast.c239
-rw-r--r--Python/bltinmodule.c181
-rw-r--r--Python/ceval.c2352
-rw-r--r--Python/codecs.c201
-rw-r--r--Python/compile.c490
-rw-r--r--Python/condvar.h2
-rw-r--r--Python/dynload_aix.c12
-rw-r--r--Python/dynload_os2.c42
-rw-r--r--Python/dynload_shlib.c62
-rw-r--r--Python/errors.c285
-rw-r--r--Python/fileutils.c380
-rw-r--r--Python/formatter_unicode.c37
-rw-r--r--Python/frozen.c4
-rw-r--r--Python/frozenmain.c58
-rw-r--r--Python/future.c53
-rw-r--r--Python/getargs.c121
-rw-r--r--Python/getcwd.c83
-rw-r--r--Python/import.c905
-rw-r--r--Python/importdl.c2
-rw-r--r--Python/importdl.h5
-rw-r--r--Python/importlib.h8642
-rw-r--r--Python/marshal.c799
-rw-r--r--Python/modsupport.c12
-rw-r--r--Python/mystrtoul.c32
-rw-r--r--Python/opcode_targets.h4
-rw-r--r--Python/peephole.c55
-rw-r--r--Python/pyarena.c12
-rw-r--r--Python/pyhash.c427
-rw-r--r--Python/pystate.c152
-rw-r--r--Python/pystrtod.c6
-rw-r--r--Python/pythonrun.c542
-rw-r--r--Python/pytime.c35
-rw-r--r--Python/random.c136
-rw-r--r--Python/strdup.c2
-rw-r--r--Python/symtable.c187
-rw-r--r--Python/sysmodule.c259
-rw-r--r--Python/thread.c38
-rw-r--r--Python/thread_nt.h19
-rw-r--r--Python/thread_os2.h267
-rw-r--r--Python/thread_pth.h177
-rw-r--r--Python/thread_pthread.h15
-rw-r--r--Python/traceback.c35
-rw-r--r--README16
-rw-r--r--Tools/buildbot/external-amd64.bat18
-rw-r--r--Tools/buildbot/external-common.bat19
-rw-r--r--Tools/buildbot/external.bat16
-rw-r--r--Tools/buildbot/test-amd64.bat2
-rw-r--r--Tools/buildbot/test.bat2
-rwxr-xr-xTools/clinic/clinic.py4199
-rw-r--r--Tools/clinic/clinic_test.py801
-rw-r--r--Tools/clinic/cpp.py191
-rw-r--r--Tools/freeze/bkfile.py4
-rwxr-xr-xTools/freeze/freeze.py18
-rw-r--r--Tools/freeze/makeconfig.py2
-rw-r--r--Tools/freeze/test/Makefile11
-rw-r--r--Tools/freeze/test/ok.py2
-rwxr-xr-xTools/gdb/libpython.py44
-rwxr-xr-xTools/i18n/makelocalealias.py27
-rwxr-xr-xTools/i18n/msgfmt.py2
-rw-r--r--Tools/importbench/importbench.py31
-rw-r--r--Tools/iobench/iobench.py4
-rw-r--r--Tools/msi/msi.py67
-rw-r--r--Tools/parser/unparse.py15
-rw-r--r--Tools/scripts/README126
-rwxr-xr-xTools/scripts/byext.py2
-rwxr-xr-xTools/scripts/byteyears.py2
-rwxr-xr-xTools/scripts/checkpip.py32
-rwxr-xr-xTools/scripts/checkpyc.py12
-rwxr-xr-xTools/scripts/copytime.py4
-rwxr-xr-xTools/scripts/diff.py4
-rwxr-xr-xTools/scripts/finddiv.py2
-rwxr-xr-xTools/scripts/findlinksto.py2
-rwxr-xr-xTools/scripts/fixcid.py8
-rwxr-xr-xTools/scripts/ftpmirror.py20
-rwxr-xr-xTools/scripts/linktree.py6
-rwxr-xr-xTools/scripts/ndiff.py2
-rwxr-xr-xTools/scripts/parse_html5_entities.py105
-rwxr-xr-xTools/scripts/pathfix.py10
-rwxr-xr-xTools/scripts/pindent.py4
-rwxr-xr-x[-rw-r--r--]Tools/scripts/pydocgui.pyw0
-rwxr-xr-xTools/scripts/reindent.py2
-rwxr-xr-xTools/scripts/treesync.py2
-rwxr-xr-xTools/scripts/untabify.py4
-rwxr-xr-xTools/scripts/which.py2
-rw-r--r--Tools/scripts/win_add2path.py2
-rw-r--r--Tools/ssl/test_multiple_versions.py241
-rw-r--r--Tools/stringbench/stringbench.py2
-rw-r--r--Tools/unicode/gencodec.py18
-rw-r--r--Tools/unicode/makeunicodedata.py8
-rw-r--r--Tools/unicode/python-mappings/CP273.TXT258
-rwxr-xr-xconfigure582
-rw-r--r--configure.ac420
-rw-r--r--pyconfig.h.in37
-rw-r--r--setup.py30
1519 files changed, 168390 insertions, 108264 deletions
diff --git a/.gitignore b/.gitignore
index 49299ee..59d3832 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,11 +15,13 @@ Doc/tools/jinja2/
Doc/tools/pygments/
Doc/tools/sphinx/
Lib/lib2to3/*.pickle
+Lib/test/data/*
Lib/_sysconfigdata.py
Lib/plat-mac/errors.rsrc.df.rsrc
Makefile
Makefile.pre
Misc/python.pc
+Misc/python-config.sh
Modules/Setup
Modules/Setup.config
Modules/Setup.local
@@ -57,6 +59,8 @@ platform
pybuilddir.txt
pyconfig.h
python
+python-config
+python-config.py
python.exe
python-gdb.py
python.exe-gdb.py
diff --git a/.hgeol b/.hgeol
index 73a4770..2919f76 100644
--- a/.hgeol
+++ b/.hgeol
@@ -26,6 +26,7 @@
**.psd = BIN
**.tar = BIN
**.wav = BIN
+**.whl = BIN
**.xar = BIN
**.zip = BIN
diff --git a/.hgignore b/.hgignore
index c42dd03..c67ffb8 100644
--- a/.hgignore
+++ b/.hgignore
@@ -19,21 +19,20 @@ platform$
pyconfig.h$
python$
python.exe$
+python-config$
+python-config.py$
reflog.txt$
tags$
Lib/plat-mac/errors.rsrc.df.rsrc
-Doc/tools/sphinx/
-Doc/tools/docutils/
-Doc/tools/jinja/
-Doc/tools/jinja2/
-Doc/tools/pygments/
Misc/python.pc
+Misc/python-config.sh$
Modules/Setup$
Modules/Setup.config
Modules/Setup.local
Modules/config.c
Modules/ld_so_aix$
Parser/pgen$
+^lcov-report/
^core
^python-gdb.py
^python.exe-gdb.py
@@ -81,7 +80,12 @@ PCbuild/*.suo
PCbuild/*.*sdf
PCbuild/Win32-temp-*
PCbuild/x64-temp-*
+PCbuild/*-pgi
+PCbuild/*-pgo
PCbuild/amd64
+PCbuild/ipch
+Tools/unicode/build/
+Tools/unicode/MAPPINGS/
BuildLog.htm
__pycache__
Modules/_freeze_importlib
@@ -89,3 +93,7 @@ Modules/_testembed
.coverage
coverage/
htmlcov/
+*.gcda
+*.gcno
+*.gcov
+coverage.info
diff --git a/.hgtags b/.hgtags
index 3885d2c..9c3af09 100644
--- a/.hgtags
+++ b/.hgtags
@@ -123,3 +123,14 @@ fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1
9ec811df548ed154a9bf9815383a916d6df31b98 v3.3.5rc1
ca5635efe090f78806188ac2758f9948596aa8b2 v3.3.5rc2
62cf4e77f78564714e7ea3d4bf1479ca1fbd0758 v3.3.5
+46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1
+9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2
+dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3
+e245b0d7209bb6d0e19316e1e2af1aa9c2139104 v3.4.0a4
+3405dc9a6afaa0a06dd1f6f182ec5c998dce6f5f v3.4.0b1
+ba32913eb13ec545a46dd0ce18035b6c416f0d78 v3.4.0b2
+a97ce3ecc96af79bd2e1ac66ce48d9138e0ca749 v3.4.0b3
+5e088cea8660677969113741c1313d570d977e02 v3.4.0rc1
+a300712ed38c9a242b736c44e806caea25a6dc05 v3.4.0rc2
+8a81cdab3e9d521daaef989fade94b16455fc3b8 v3.4.0rc3
+04f714765c13824c3bc2835d7b008908862e083a v3.4.0
diff --git a/.hgtouch b/.hgtouch
index a629de9..7e3a5e7 100644
--- a/.hgtouch
+++ b/.hgtouch
@@ -4,8 +4,7 @@
Python/importlib.h: Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
-Include/ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py
-Include/Python-ast.h: Include/ast.h
+Include/Python-ast.h: Parser/Python.asdl Parser/asdl.py Parser/asdl_c.py
Python/Python-ast.c: Include/Python-ast.h
Python/opcode_targets.h: Python/makeopcodetargets.py Lib/opcode.py
diff --git a/Doc/Makefile b/Doc/Makefile
index 82f5bef..12fc698 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -5,7 +5,7 @@
# You can set these variables from the command line.
PYTHON = python
-SVNROOT = http://svn.python.org/projects
+SPHINXBUILD = sphinx-build
SPHINXOPTS =
PAPER =
SOURCES =
@@ -14,14 +14,13 @@ DISTVERSION = $(shell $(PYTHON) tools/sphinxext/patchlevel.py)
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
$(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
-.PHONY: help checkout update build html htmlhelp latex text changes linkcheck \
+.PHONY: help build html htmlhelp latex text changes linkcheck \
suspicious coverage doctest pydoc-topics htmlview clean dist check serve \
autobuild-dev autobuild-stable
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " clean to remove build files"
- @echo " update to update build tools"
@echo " html to make standalone HTML files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@@ -37,30 +36,8 @@ help:
@echo " check to run a check for frequent markup errors"
@echo " serve to serve the documentation on the localhost (8000)"
-# Note: if you update versions here, do the same in make.bat and README.txt
-checkout:
- @if [ ! -d tools/sphinx ]; then \
- echo "Checking out Sphinx..."; \
- svn checkout $(SVNROOT)/external/Sphinx-1.2/sphinx tools/sphinx; \
- fi
- @if [ ! -d tools/docutils ]; then \
- echo "Checking out Docutils..."; \
- svn checkout $(SVNROOT)/external/docutils-0.11/docutils tools/docutils; \
- fi
- @if [ ! -d tools/jinja2 ]; then \
- echo "Checking out Jinja..."; \
- svn checkout $(SVNROOT)/external/Jinja-2.3.1/jinja2 tools/jinja2; \
- fi
- @if [ ! -d tools/pygments ]; then \
- echo "Checking out Pygments..."; \
- svn checkout $(SVNROOT)/external/Pygments-1.6/pygments tools/pygments; \
- fi
-
-update: clean checkout
-
-build: checkout
- mkdir -p build/$(BUILDER) build/doctrees
- $(PYTHON) tools/sphinx-build.py $(ALLSPHINXOPTS)
+build:
+ $(SPHINXBUILD) $(ALLSPHINXOPTS)
@echo
html: BUILDER = html
@@ -120,10 +97,6 @@ htmlview: html
clean:
-rm -rf build/*
- -rm -rf tools/sphinx
- -rm -rf tools/pygments
- -rm -rf tools/jinja2
- -rm -rf tools/docutils
dist:
rm -rf dist
@@ -163,16 +136,10 @@ dist:
cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip
cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2
- # archive the epub build
+ # copy the epub build
rm -rf build/epub
make epub
- mkdir -p dist/python-$(DISTVERSION)-docs-epub
- cp -pPR build/epub/*.epub dist/python-$(DISTVERSION)-docs-epub/
- tar -C dist -cf dist/python-$(DISTVERSION)-docs-epub.tar python-$(DISTVERSION)-docs-epub
- bzip2 -9 -k dist/python-$(DISTVERSION)-docs-epub.tar
- (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-epub.zip python-$(DISTVERSION)-docs-epub)
- rm -r dist/python-$(DISTVERSION)-docs-epub
- rm dist/python-$(DISTVERSION)-docs-epub.tar
+ cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
check:
$(PYTHON) tools/rstlint.py -i tools
@@ -184,7 +151,6 @@ serve:
# for development releases: always build
autobuild-dev:
- make update
make dist SPHINXOPTS='-A daily=1 -A versionswitcher=1'
-make suspicious
diff --git a/Doc/README.txt b/Doc/README.txt
index a494c89..f4f6d81 100644
--- a/Doc/README.txt
+++ b/Doc/README.txt
@@ -3,34 +3,31 @@ Python Documentation README
This directory contains the reStructuredText (reST) sources to the Python
documentation. You don't need to build them yourself, prebuilt versions are
-available at http://docs.python.org/download/.
+available at <http://docs.python.org/download/>.
Documentation on the authoring Python documentation, including information about
both style and markup, is available in the "Documenting Python" chapter of the
-documentation.
+developers guide <http://docs.python.org/devguide/documenting.html>.
Building the docs
=================
-You need to have Python 2 installed; the toolset used to build the
-docs is written in Python. It is called *Sphinx*, it is not included in this
-tree, but maintained separately. Also needed are the docutils, supplying the
-base markup that Sphinx uses, Jinja, a templating engine, and optionally
-Pygments, a code highlighter.
+You need to have Sphinx <http://sphinx-doc.org/> installed; it is the toolset
+used to build the docs. It is not included in this tree, but maintained
+separately and available from PyPI <http://pypi.python.org/pypi/Sphinx>.
Using make
----------
-Luckily, a Makefile has been prepared so that on Unix, provided you have
-installed Python and Subversion, you can just run ::
+A Makefile has been prepared so that on Unix, provided you have installed
+Sphinx, you can just run ::
make html
-to check out the necessary toolset in the `tools/` subdirectory and build the
-HTML output files. To view the generated HTML, point your favorite browser at
-the top-level index `build/html/index.html` after running "make".
+to build the HTML output files. To view the generated HTML, point your favorite
+browser at the top-level index `build/html/index.html` after running "make".
On Windows, we try to emulate the Makefile as closely as possible with a
``make.bat`` file.
@@ -78,8 +75,6 @@ Available make targets are:
* "suspicious", which checks the parsed markup for text that looks like
malformed and thus unconverted reST.
-A "make update" updates the Subversion checkouts in `tools/`.
-
Without make
------------
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index 28a434e..2f02241 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -295,6 +295,8 @@ Other objects
the object pointer is stored. If the Python object does not have the required
type, :exc:`TypeError` is raised.
+.. _o_ampersand:
+
``O&`` (object) [*converter*, *anything*]
Convert a Python object to a C variable through a *converter* function. This
takes two arguments: the first is a function, the second is the address of a C
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 65904ee..2d56386 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -74,26 +74,35 @@ intrinsic to the Python language.
.. _mapobjects:
-Mapping Objects
-===============
+Container Objects
+=================
.. index:: object: mapping
.. toctree::
dict.rst
+ set.rst
.. _otherobjects:
-Other Objects
-=============
+Function Objects
+================
.. toctree::
- set.rst
function.rst
method.rst
+ cell.rst
+ code.rst
+
+
+Other Objects
+=============
+
+.. toctree::
+
file.rst
module.rst
iterator.rst
@@ -102,7 +111,6 @@ Other Objects
memoryview.rst
weakref.rst
capsule.rst
- cell.rst
gen.rst
datetime.rst
- code.rst
+
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index 6bacc32..aeff640 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -84,7 +84,7 @@ Dictionary Objects
on failure.
-.. c:function:: int PyDict_DelItemString(PyObject *p, char *key)
+.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key)
Remove the entry in dictionary *p* which has a key specified by the string
*key*. Return ``0`` on success or ``-1`` on failure.
@@ -110,6 +110,15 @@ Dictionary Objects
:c:type:`char\*`, rather than a :c:type:`PyObject\*`.
+.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default)
+
+ This is the same as the Python-level :meth:`dict.setdefault`. If present, it
+ returns the value corresponding to *key* from the dictionary *p*. If the key
+ is not in the dict, it is inserted with value *defaultobj* and *defaultobj*
+ is returned. This function evaluates the hash function of *key* only once,
+ instead of evaluating it independently for the lookup and the insertion.
+
+
.. c:function:: PyObject* PyDict_Items(PyObject *p)
Return a :c:type:`PyListObject` containing all the items from the dictionary.
@@ -192,8 +201,11 @@ Dictionary Objects
.. c:function:: int PyDict_Update(PyObject *a, PyObject *b)
- This is the same as ``PyDict_Merge(a, b, 1)`` in C, or ``a.update(b)`` in
- Python. Return ``0`` on success or ``-1`` if an exception was raised.
+ This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to
+ ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall
+ back to the iterating over a sequence of key value pairs if the second
+ argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an
+ exception was raised.
.. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override)
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 0aa892d..6fda191 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -90,6 +90,16 @@ in various ways. There is a separate error indicator for each thread.
the class in that case. If the values are already normalized, nothing happens.
The delayed normalization is implemented to improve performance.
+ .. note::
+
+ This function *does not* implicitly set the ``__traceback__``
+ attribute on the exception value. If setting the traceback
+ appropriately is desired, the following additional snippet is needed::
+
+ if (tb != NULL) {
+ PyException_SetTraceback(val, tb);
+ }
+
.. c:function:: void PyErr_Clear()
@@ -231,11 +241,20 @@ in various ways. There is a separate error indicator for each thread.
exception instance.
+.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2)
+
+ Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second
+ filename object, for raising errors when a function that takes two filenames
+ fails.
+
+ .. versionadded:: 3.4
+
+
.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename)
Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename
is given as a C string. *filename* is decoded from the filesystem encoding
- (:func:`sys.getfilesystemencoding`).
+ (:func:`os.fsdecode`).
.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr)
@@ -256,18 +275,11 @@ in various ways. There is a separate error indicator for each thread.
specifying the exception type to be raised. Availability: Windows.
-.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilenameObject(int ierr, PyObject *filenameObject)
-
- Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior
- that if *filenameObject* is not *NULL*, it is passed to the constructor of
- :exc:`WindowsError` as a third parameter. Availability: Windows.
-
-
.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename)
Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the
filename is given as a C string. *filename* is decoded from the filesystem
- encoding (:func:`sys.getfilesystemencoding`). Availability: Windows.
+ encoding (:func:`os.fsdecode`). Availability: Windows.
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename)
@@ -277,6 +289,15 @@ in various ways. There is a separate error indicator for each thread.
Availability: Windows.
+.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2)
+
+ Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`,
+ but accepts a second filename object.
+ Availability: Windows.
+
+ .. versionadded:: 3.4
+
+
.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename)
Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional
@@ -293,13 +314,20 @@ in various ways. There is a separate error indicator for each thread.
.. versionadded:: 3.3
-.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset)
+.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset)
Set file, line, and offset information for the current exception. If the
current exception is not a :exc:`SyntaxError`, then it sets additional
attributes, which make the exception printing subsystem think the exception
- is a :exc:`SyntaxError`. *filename* is decoded from the filesystem encoding
- (:func:`sys.getfilesystemencoding`).
+ is a :exc:`SyntaxError`.
+
+ .. versionadded:: 3.4
+
+
+.. c:function:: void PyErr_SyntaxLocationEx(char *filename, int lineno, int col_offset)
+
+ Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string
+ decoded from the filesystem encoding (:func:`os.fsdecode`).
.. versionadded:: 3.2
@@ -355,15 +383,22 @@ in various ways. There is a separate error indicator for each thread.
documentation. There is no C API for warning control.
-.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry)
+.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry)
Issue a warning message with explicit control over all warning attributes. This
is a straightforward wrapper around the Python function
:func:`warnings.warn_explicit`, see there for more information. The *module*
and *registry* arguments may be set to *NULL* to get the default effect
- described there. *message* and *module* are UTF-8 encoded strings,
- *filename* is decoded from the filesystem encoding
- (:func:`sys.getfilesystemencoding`).
+ described there.
+
+ .. versionadded:: 3.4
+
+
+.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry)
+
+ Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and
+ *module* are UTF-8 encoded strings, and *filename* is decoded from the
+ filesystem encoding (:func:`os.fsdecode`).
.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...)
diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst
index cc190c9..6f2ecee 100644
--- a/Doc/c-api/file.rst
+++ b/Doc/c-api/file.rst
@@ -17,7 +17,7 @@ error reporting in the interpreter; third-party code is advised to access
the :mod:`io` APIs instead.
-.. c:function:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *errors, char *newline, int closefd)
+.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd)
Create a Python file object from the file descriptor of an already
opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline*
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index 270152e..6cd2b8b 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -118,7 +118,7 @@ Importing Modules
encoded string instead of a Unicode object.
-.. c:function:: PyObject* PyImport_ExecCodeModule(char *name, PyObject *co)
+.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co)
.. index:: builtin: compile
@@ -145,7 +145,7 @@ Importing Modules
:c:func:`PyImport_ExecCodeModuleWithPathnames`.
-.. c:function:: PyObject* PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname)
+.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname)
Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of
the module object is set to *pathname* if it is non-``NULL``.
@@ -162,7 +162,7 @@ Importing Modules
.. versionadded:: 3.3
-.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname)
+.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname)
Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and
*cpathname* are UTF-8 encoded strings. Attempts are also made to figure out
@@ -246,7 +246,7 @@ Importing Modules
.. versionadded:: 3.3
-.. c:function:: int PyImport_ImportFrozenModule(char *name)
+.. c:function:: int PyImport_ImportFrozenModule(const char *name)
Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a
UTF-8 encoded string instead of a Unicode object.
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 4dad2c8..0587e15 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -86,6 +86,36 @@ Process-wide parameters
=======================
+.. c:function:: int Py_SetStandardStreamEncoding(char *encoding, char *errors)
+
+ .. index::
+ single: Py_Initialize()
+ single: main()
+ triple: stdin; stdout; sdterr
+
+ This function should be called before :c:func:`Py_Initialize`, if it is
+ called at all. It specifies which encoding and error handling to use
+ with standard IO, with the same meanings as in :func:`str.encode`.
+
+ It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code
+ to control IO encoding when the environment variable does not work.
+
+ ``encoding`` and/or ``errors`` may be NULL to use
+ :envvar:`PYTHONIOENCODING` and/or default values (depending on other
+ settings).
+
+ Note that :data:`sys.stderr` always uses the "backslashreplace" error
+ handler, regardless of this (or any other) setting.
+
+ If :c:func:`Py_Finalize` is called, this function will need to be called
+ again in order to affect subsequent calls to :c:func:`Py_Initialize`.
+
+ Returns 0 if successful, a nonzero value on error (e.g. calling after the
+ interpreter has already been initialized).
+
+ .. versionadded:: 3.4
+
+
.. c:function:: void Py_SetProgramName(wchar_t *name)
.. index::
@@ -329,7 +359,11 @@ Process-wide parameters
.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv)
- This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set to 1.
+ This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set
+ to 1 unless the :program:`python` interpreter was started with the
+ :option:`-I`.
+
+ .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`.
.. c:function:: void Py_SetPythonHome(wchar_t *home)
@@ -658,6 +692,20 @@ with sub-interpreters:
made on the main thread. This is mainly a helper/diagnostic function.
+.. c:function:: int PyGILState_Check()
+
+ Return 1 if the current thread is holding the GIL and 0 otherwise.
+ This function can be called from any thread at any time.
+ Only if it has had its Python thread state initialized and currently is
+ holding the GIL will it return 1.
+ This is mainly a helper/diagnostic function. It can be useful
+ for example in callback contexts or memory allocation functions when
+ knowing that the GIL is locked can allow the caller to perform sensitive
+ actions or otherwise behave differently.
+
+ .. versionadded:: 3.4
+
+
The following macros are normally used without a trailing semicolon; look for
example usage in the Python source distribution.
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index d5430fd..b348015 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -80,7 +80,7 @@ All integers are implemented as "long" integer objects of arbitrary size.
*NULL* on failure.
-.. c:function:: PyObject* PyLong_FromString(char *str, char **pend, int base)
+.. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base)
Return a new :c:type:`PyLongObject` based on the string value in *str*, which
is interpreted according to the radix in *base*. If *pend* is non-*NULL*,
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index 0ef2961..2803fd0 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -22,7 +22,7 @@ Mapping Protocol
expression ``len(o)``.
-.. c:function:: int PyMapping_DelItemString(PyObject *o, char *key)
+.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key)
Remove the mapping for object *key* from the object *o*. Return ``-1`` on
failure. This is equivalent to the Python statement ``del o[key]``.
@@ -67,13 +67,13 @@ Mapping Protocol
the Python expression ``list(o.items())``.
-.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, char *key)
+.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key)
Return element of *o* corresponding to the object *key* or *NULL* on failure.
This is the equivalent of the Python expression ``o[key]``.
-.. c:function:: int PyMapping_SetItemString(PyObject *o, char *key, PyObject *v)
+.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v)
Map the object *key* to the value *v* in object *o*. Returns ``-1`` on failure.
This is the equivalent of the Python statement ``o[key] = v``.
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 8afa56a..a82e1c2 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -84,6 +84,48 @@ the C library allocator as shown in the previous example, the allocated memory
for the I/O buffer escapes completely the Python memory manager.
+Raw Memory Interface
+====================
+
+The following function sets are wrappers to the system allocator. These
+functions are thread-safe, the :term:`GIL <global interpreter lock>` does not
+need to be held.
+
+The default raw memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. versionadded:: 3.4
+
+.. c:function:: void* PyMem_RawMalloc(size_t n)
+
+ Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the
+ allocated memory, or *NULL* if the request fails. Requesting zero bytes
+ returns a distinct non-*NULL* pointer if possible, as if
+ ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have
+ been initialized in any way.
+
+
+.. c:function:: void* PyMem_RawRealloc(void *p, size_t n)
+
+ Resizes the memory block pointed to by *p* to *n* bytes. The contents will
+ be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*,
+ the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to
+ zero, the memory block is resized but is not freed, and the returned pointer
+ is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a
+ previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If
+ the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains
+ a valid pointer to the previous memory area.
+
+
+.. c:function:: void PyMem_RawFree(void *p)
+
+ Frees the memory block pointed to by *p*, which must have been returned by a
+ previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`.
+ Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined
+ behavior occurs. If *p* is *NULL*, no operation is performed.
+
+
.. _memoryinterface:
Memory Interface
@@ -91,8 +133,16 @@ Memory Interface
The following function sets, modeled after the ANSI C standard, but specifying
behavior when requesting zero bytes, are available for allocating and releasing
-memory from the Python heap:
+memory from the Python heap.
+
+The default memory block allocator uses the following functions:
+:c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when
+requesting zero bytes.
+
+.. warning::
+ The :term:`GIL <global interpreter lock>` must be held when using these
+ functions.
.. c:function:: void* PyMem_Malloc(size_t n)
@@ -155,6 +205,125 @@ versions and is therefore deprecated in extension modules.
:c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`.
+Customize Memory Allocators
+===========================
+
+.. versionadded:: 3.4
+
+.. c:type:: PyMemAllocator
+
+ Structure used to describe a memory block allocator. The structure has
+ four fields:
+
+ +----------------------------------------------------------+---------------------------------------+
+ | Field | Meaning |
+ +==========================================================+=======================================+
+ | ``void *ctx`` | user context passed as first argument |
+ +----------------------------------------------------------+---------------------------------------+
+ | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block |
+ +----------------------------------------------------------+---------------------------------------+
+ | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block |
+ +----------------------------------------------------------+---------------------------------------+
+ | ``void free(void *ctx, void *ptr)`` | free a memory block |
+ +----------------------------------------------------------+---------------------------------------+
+
+.. c:type:: PyMemAllocatorDomain
+
+ Enum used to identify an allocator domain. Domains:
+
+ * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`,
+ :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree`
+ * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`,
+ :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free`
+ * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`,
+ :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free`
+
+
+.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+ Get the memory block allocator of the specified domain.
+
+
+.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator)
+
+ Set the memory block allocator of the specified domain.
+
+ The new allocator must return a distinct non-NULL pointer when requesting
+ zero bytes.
+
+ For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be
+ thread-safe: the :term:`GIL <global interpreter lock>` is not held when the
+ allocator is called.
+
+ If the new allocator is not a hook (does not call the previous allocator),
+ the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the
+ debug hooks on top on the new allocator.
+
+
+.. c:function:: void PyMem_SetupDebugHooks(void)
+
+ Setup hooks to detect bugs in the following Python memory allocator
+ functions:
+
+ - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`,
+ :c:func:`PyMem_RawFree`
+ - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free`
+ - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`,
+ :c:func:`PyObject_Free`
+
+ Newly allocated memory is filled with the byte ``0xCB``, freed memory is
+ filled with the byte ``0xDB``. Additionnal checks:
+
+ - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer
+ allocated by :c:func:`PyMem_Malloc`
+ - detect write before the start of the buffer (buffer underflow)
+ - detect write after the end of the buffer (buffer overflow)
+
+ The function does nothing if Python is not compiled is debug mode.
+
+
+Customize PyObject Arena Allocator
+==================================
+
+Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This
+allocator is optimized for small objects with a short lifetime. It uses memory
+mappings called "arenas" with a fixed size of 256 KB. It falls back to
+:c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger
+than 512 bytes. *pymalloc* is the default allocator used by
+:c:func:`PyObject_Malloc`.
+
+The default arena allocator uses the following functions:
+
+* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows,
+* :c:func:`mmap` and :c:func:`munmap` if available,
+* :c:func:`malloc` and :c:func:`free` otherwise.
+
+.. versionadded:: 3.4
+
+.. c:type:: PyObjectArenaAllocator
+
+ Structure used to describe an arena allocator. The structure has
+ three fields:
+
+ +--------------------------------------------------+---------------------------------------+
+ | Field | Meaning |
+ +==================================================+=======================================+
+ | ``void *ctx`` | user context passed as first argument |
+ +--------------------------------------------------+---------------------------------------+
+ | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes |
+ +--------------------------------------------------+---------------------------------------+
+ | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena |
+ +--------------------------------------------------+---------------------------------------+
+
+.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+ Get the arena allocator.
+
+.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
+
+ Set the arena allocator.
+
+
.. _memoryexamples:
Examples
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index bd46170..985a347 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -35,13 +35,20 @@ There are only a few functions special to module objects.
single: __name__ (module attribute)
single: __doc__ (module attribute)
single: __file__ (module attribute)
+ single: __package__ (module attribute)
+ single: __loader__ (module attribute)
Return a new module object with the :attr:`__name__` attribute set to *name*.
- Only the module's :attr:`__doc__` and :attr:`__name__` attributes are filled in;
- the caller is responsible for providing a :attr:`__file__` attribute.
+ The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and
+ :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set
+ to ``None``); the caller is responsible for providing a :attr:`__file__`
+ attribute.
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ :attr:`__package__` and :attr:`__loader__` are set to ``None``.
+
.. c:function:: PyObject* PyModule_New(const char *name)
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index 0aba360..3e7c3b6 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -149,6 +149,9 @@ Object Protocol
representation on success, *NULL* on failure. This is the equivalent of the
Python expression ``repr(o)``. Called by the :func:`repr` built-in function.
+ .. versionchanged:: 3.4
+ This function now includes a debug assertion to help ensure that it
+ does not silently discard an active exception.
.. c:function:: PyObject* PyObject_ASCII(PyObject *o)
@@ -170,6 +173,10 @@ Object Protocol
Python expression ``str(o)``. Called by the :func:`str` built-in function
and, therefore, by the :func:`print` function.
+ .. versionchanged:: 3.4
+ This function now includes a debug assertion to help ensure that it
+ does not silently discard an active exception.
+
.. c:function:: PyObject* PyObject_Bytes(PyObject *o)
.. index:: builtin: bytes
@@ -240,7 +247,7 @@ attribute is considered sufficient for this determination.
of the Python expression ``callable_object(*args)``.
-.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, char *format, ...)
+.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...)
Call a callable Python object *callable*, with a variable number of C arguments.
The C arguments are described using a :c:func:`Py_BuildValue` style format
@@ -250,8 +257,11 @@ attribute is considered sufficient for this determination.
pass :c:type:`PyObject \*` args, :c:func:`PyObject_CallFunctionObjArgs` is a
faster alternative.
+ .. versionchanged:: 3.4
+ The type of *format* was changed from ``char *``.
+
-.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, char *method, char *format, ...)
+.. c:function:: PyObject* PyObject_CallMethod(PyObject *o, const char *method, const char *format, ...)
Call the method named *method* of object *o* with a variable number of C
arguments. The C arguments are described by a :c:func:`Py_BuildValue` format
@@ -261,6 +271,9 @@ attribute is considered sufficient for this determination.
Note that if you only pass :c:type:`PyObject \*` args,
:c:func:`PyObject_CallMethodObjArgs` is a faster alternative.
+ .. versionchanged:: 3.4
+ The types of *method* and *format* were changed from ``char *``.
+
.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL)
@@ -342,6 +355,16 @@ attribute is considered sufficient for this determination.
returned. This is the equivalent to the Python expression ``len(o)``.
+.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t default)
+
+ Return an estimated length for the object *o*. First try to return its
+ actual length, then an estimate using :meth:`~object.__length_hint__`, and
+ finally return the default value. On error return ``-1``. This is the
+ equivalent to the Python expression ``operator.length_hint(o, default)``.
+
+ .. versionadded:: 3.4
+
+
.. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key)
Return element of *o* corresponding to the object *key* or *NULL* on failure.
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 184affb..3922d50 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -129,6 +129,14 @@ type.
Initializes a struct sequence type *type* from *desc* in place.
+.. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc)
+
+ The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on
+ failure.
+
+ .. versionadded:: 3.4
+
+
.. c:type:: PyStructSequence_Desc
Contains the meta information of a struct sequence type to create.
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 5d83254..8dc040b 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -97,3 +97,13 @@ Type Objects
types. This allows the caller to reference other heap types as base types.
.. versionadded:: 3.3
+
+.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot)
+
+ Return the function pointer stored in the given slot. If the
+ result is *NULL*, this indicates that either the slot is *NULL*,
+ or that the function was called with invalid parameters.
+ Callers will typically cast the result pointer into the appropriate
+ function type.
+
+ .. versionadded:: 3.4
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index f3089d0..efd8574 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -464,6 +464,14 @@ type objects) *must* have the :attr:`ob_size` field.
:const:`Py_TPFLAGS_HAVE_VERSION_TAG`.
+ .. data:: Py_TPFLAGS_HAVE_FINALIZE
+
+ This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the
+ type structure.
+
+ .. versionadded:: 3.4
+
+
.. c:member:: char* PyTypeObject.tp_doc
An optional pointer to a NUL-terminated C string giving the docstring for this
@@ -967,6 +975,47 @@ type objects) *must* have the :attr:`ob_size` field.
This field is not inherited; it is calculated fresh by :c:func:`PyType_Ready`.
+.. c:member:: destructor PyTypeObject.tp_finalize
+
+ An optional pointer to an instance finalization function. Its signature is
+ :c:type:`destructor`::
+
+ void tp_finalize(PyObject *)
+
+ If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when
+ finalizing an instance. It is called either from the garbage
+ collector (if the instance is part of an isolated reference cycle) or
+ just before the object is deallocated. Either way, it is guaranteed
+ to be called before attempting to break reference cycles, ensuring
+ that it finds the object in a sane state.
+
+ :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status;
+ therefore, a recommended way to write a non-trivial finalizer is::
+
+ static void
+ local_finalize(PyObject *self)
+ {
+ PyObject *error_type, *error_value, *error_traceback;
+
+ /* Save the current exception, if any. */
+ PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+ /* ... */
+
+ /* Restore the saved exception. */
+ PyErr_Restore(error_type, error_value, error_traceback);
+ }
+
+ For this field to be taken into account (even through inheritance),
+ you must also set the :const:`Py_TPFLAGS_HAVE_FINALIZE` flags bit.
+
+ This field is inherited by subtypes.
+
+ .. versionadded:: 3.4
+
+ .. seealso:: "Safe object finalization" (:pep:`442`)
+
+
.. c:member:: PyObject* PyTypeObject.tp_cache
Unused. Not inherited. Internal use only.
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 3649cfb..c7ed5e5 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -526,12 +526,23 @@ APIs:
The `"%lld"` and `"%llu"` format specifiers are only available
when :const:`HAVE_LONG_LONG` is defined.
+ .. note::
+ The width formatter unit is number of characters rather than bytes.
+ The precision formatter unit is number of bytes for ``"%s"`` and
+ ``"%V"`` (if the ``PyObject*`` argument is NULL), and a number of
+ characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"``
+ (if the ``PyObject*`` argument is not NULL).
+
.. versionchanged:: 3.2
Support for ``"%lld"`` and ``"%llu"`` added.
.. versionchanged:: 3.3
Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added.
+ .. versionchanged:: 3.4
+ Support width and precision formatter for ``"%s"``, ``"%A"``, ``"%U"``,
+ ``"%V"``, ``"%S"``, ``"%R"`` added.
+
.. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs)
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 14ef8df..f7ed4c7 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -144,6 +144,37 @@ the same library that the Python runtime is using.
(:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF.
+.. c:var:: int (*PyOS_InputHook)(void)
+
+ Can be set to point to a function with the prototype
+ ``int func(void)``. The function will be called when Python's
+ interpreter prompt is about to become idle and wait for user input
+ from the terminal. The return value is ignored. Overriding this
+ hook can be used to integrate the interpreter's prompt with other
+ event loops, as done in the :file:`Modules/_tkinter.c` in the
+ Python source code.
+
+
+.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *)
+
+ Can be set to point to a function with the prototype
+ ``char *func(FILE *stdin, FILE *stdout, char *prompt)``,
+ overriding the default function used to read a single line of input
+ at the interpreter's prompt. The function is expected to output
+ the string *prompt* if it's not *NULL*, and then read a line of
+ input from the provided standard input file, returning the
+ resulting string. For example, The :mod:`readline` module sets
+ this hook to provide line-editing and tab-completion features.
+
+ The result must be a string allocated by :c:func:`PyMem_RawMalloc` or
+ :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred.
+
+ .. versionchanged:: 3.4
+ The result must be allocated by :c:func:`PyMem_RawMalloc` or
+ :c:func:`PyMem_RawRealloc`, instead of being allocated by
+ :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`.
+
+
.. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start)
This is a simplified interface to
@@ -235,16 +266,15 @@ the same library that the Python runtime is using.
*optimize* set to ``-1``.
-.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize)
+.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize)
Parse and compile the Python source code in *str*, returning the resulting code
object. The start token is given by *start*; this can be used to constrain the
code which can be compiled and should be :const:`Py_eval_input`,
:const:`Py_file_input`, or :const:`Py_single_input`. The filename specified by
*filename* is used to construct the code object and may appear in tracebacks or
- :exc:`SyntaxError` exception messages, it is decoded from the filesystem
- encoding (:func:`sys.getfilesystemencoding`). This returns *NULL* if the
- code cannot be parsed or compiled.
+ :exc:`SyntaxError` exception messages. This returns *NULL* if the code
+ cannot be parsed or compiled.
The integer *optimize* specifies the optimization level of the compiler; a
value of ``-1`` selects the optimization level of the interpreter as given by
@@ -252,8 +282,15 @@ the same library that the Python runtime is using.
``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false)
or ``2`` (docstrings are removed too).
- .. versionadded:: 3.2
+ .. versionadded:: 3.4
+
+
+.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize)
+ Like :c:func:`Py_CompileStringExFlags`, but *filename* is a byte string
+ decoded from the filesystem encoding (:func:`os.fsdecode`).
+
+ .. versionadded:: 3.2
.. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
@@ -285,6 +322,10 @@ the same library that the Python runtime is using.
it causes an exception to immediately be thrown; this is used for the
:meth:`~generator.throw` methods of generator objects.
+ .. versionchanged:: 3.4
+ This function now includes a debug assertion to help ensure that it
+ does not silently discard an active exception.
+
.. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
@@ -338,4 +379,3 @@ the same library that the Python runtime is using.
This bit can be set in *flags* to cause division operator ``/`` to be
interpreted as "true division" according to :pep:`238`.
-
diff --git a/Doc/conf.py b/Doc/conf.py
index 5b63cad..c0ed8c6 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -61,6 +61,8 @@ add_module_names = True
# By default, highlight as Python 3.
highlight_language = 'python3'
+needs_sphinx = '1.1'
+
# Options for HTML output
# -----------------------
@@ -118,11 +120,11 @@ _stdauthor = r'Guido van Rossum\\Fred L. Drake, Jr., editor'
latex_documents = [
('c-api/index', 'c-api.tex',
'The Python/C API', _stdauthor, 'manual'),
- ('distutils/index', 'distutils.tex',
+ ('distributing/index', 'distributing.tex',
'Distributing Python Modules', _stdauthor, 'manual'),
('extending/index', 'extending.tex',
'Extending and Embedding Python', _stdauthor, 'manual'),
- ('install/index', 'install.tex',
+ ('installing/index', 'installing.tex',
'Installing Python Modules', _stdauthor, 'manual'),
('library/index', 'library.tex',
'The Python Library Reference', _stdauthor, 'manual'),
diff --git a/Doc/contents.rst b/Doc/contents.rst
index c0c6af3..29b07db 100644
--- a/Doc/contents.rst
+++ b/Doc/contents.rst
@@ -11,8 +11,8 @@
library/index.rst
extending/index.rst
c-api/index.rst
- distutils/index.rst
- install/index.rst
+ distributing/index.rst
+ installing/index.rst
howto/index.rst
faq/index.rst
glossary.rst
@@ -21,3 +21,11 @@
bugs.rst
copyright.rst
license.rst
+
+.. include legacy packaging docs in build
+
+.. toctree::
+ :hidden:
+
+ distutils/index.rst
+ install/index.rst
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index a42584c..6025617 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -29,7 +29,7 @@
# reference to the item argument!
# The parameter names are as they appear in the API manual, not the source
-# code.
+# code.
PyBool_FromLong:PyObject*::+1:
PyBool_FromLong:long:v:0:
@@ -220,6 +220,11 @@ PyDict_GetItemString:PyObject*::0:
PyDict_GetItemString:PyObject*:p:0:
PyDict_GetItemString:const char*:key::
+PyDict_SetDefault:PyObject*::0:
+PyDict_SetDefault:PyObject*:p:0:
+PyDict_SetDefault:PyObject*:key:0:conditionally +1 if inserted into the dict
+PyDict_SetDefault:PyObject*:default:0:conditionally +1 if inserted into the dict
+
PyDict_Items:PyObject*::+1:
PyDict_Items:PyObject*:p:0:
@@ -503,13 +508,13 @@ PyImport_AddModule:const char*:name::
PyImport_Cleanup:void:::
PyImport_ExecCodeModule:PyObject*::+1:
-PyImport_ExecCodeModule:char*:name::
+PyImport_ExecCodeModule:const char*:name::
PyImport_ExecCodeModule:PyObject*:co:0:
PyImport_ExecCodeModuleEx:PyObject*::+1:
-PyImport_ExecCodeModuleEx:char*:name::
+PyImport_ExecCodeModuleEx:const char*:name::
PyImport_ExecCodeModuleEx:PyObject*:co:0:
-PyImport_ExecCodeModuleEx:char*:pathname::
+PyImport_ExecCodeModuleEx:const char*:pathname::
PyImport_GetMagicNumber:long:::
@@ -519,7 +524,7 @@ PyImport_Import:PyObject*::+1:
PyImport_Import:PyObject*:name:0:
PyImport_ImportFrozenModule:int:::
-PyImport_ImportFrozenModule:char*:::
+PyImport_ImportFrozenModule:const char*:::
PyImport_ImportModule:PyObject*::+1:
PyImport_ImportModule:const char*:name::
@@ -668,7 +673,7 @@ PyLong_FromUnsignedLongLong:PyObject*::+1:
PyLong_FromUnsignedLongLong:unsigned long long:v::
PyLong_FromString:PyObject*::+1:
-PyLong_FromString:char*:str::
+PyLong_FromString:const char*:str::
PyLong_FromString:char**:pend::
PyLong_FromString:int:base::
@@ -696,7 +701,7 @@ PyMapping_DelItemString:const char*:key::
PyMapping_GetItemString:PyObject*::+1:
PyMapping_GetItemString:PyObject*:o:0:
-PyMapping_GetItemString:char*:key::
+PyMapping_GetItemString:const char*:key::
PyMapping_HasKey:int:::
PyMapping_HasKey:PyObject*:o:0:
@@ -704,7 +709,7 @@ PyMapping_HasKey:PyObject*:key::
PyMapping_HasKeyString:int:::
PyMapping_HasKeyString:PyObject*:o:0:
-PyMapping_HasKeyString:char*:key::
+PyMapping_HasKeyString:const char*:key::
PyMapping_Items:PyObject*::+1:
PyMapping_Items:PyObject*:o:0:
@@ -717,7 +722,7 @@ PyMapping_Length:PyObject*:o:0:
PyMapping_SetItemString:int:::
PyMapping_SetItemString:PyObject*:o:0:
-PyMapping_SetItemString:char*:key::
+PyMapping_SetItemString:const char*:key::
PyMapping_SetItemString:PyObject*:v:+1:
PyMapping_Values:PyObject*::+1:
@@ -730,7 +735,7 @@ PyMarshal_ReadObjectFromFile:PyObject*::+1:
PyMarshal_ReadObjectFromFile:FILE*:file::
PyMarshal_ReadObjectFromString:PyObject*::+1:
-PyMarshal_ReadObjectFromString:char*:string::
+PyMarshal_ReadObjectFromString:const char*:string::
PyMarshal_ReadObjectFromString:int:len::
PyMarshal_WriteObjectToString:PyObject*::+1:
@@ -902,7 +907,7 @@ PyNumber_Xor:PyObject*::+1:
PyNumber_Xor:PyObject*:o1:0:
PyNumber_Xor:PyObject*:o2:0:
-PyObject_AsFileDescriptor:int:::
+PyObject_AsFileDescriptor:int:::
PyObject_AsFileDescriptor:PyObject*:o:0:
PyObject_Call:PyObject*::+1:
@@ -912,7 +917,7 @@ PyObject_Call:PyObject*:kw:0:
PyObject_CallFunction:PyObject*::+1:
PyObject_CallFunction:PyObject*:callable_object:0:
-PyObject_CallFunction:char*:format::
+PyObject_CallFunction:const char*:format::
PyObject_CallFunction::...::
PyObject_CallFunctionObjArgs:PyObject*::+1:
@@ -921,8 +926,8 @@ PyObject_CallFunctionObjArgs::...::
PyObject_CallMethod:PyObject*::+1:
PyObject_CallMethod:PyObject*:o:0:
-PyObject_CallMethod:char*:m::
-PyObject_CallMethod:char*:format::
+PyObject_CallMethod:const char*:m::
+PyObject_CallMethod:const char*:format::
PyObject_CallMethod::...::
PyObject_CallMethodObjArgs:PyObject*::+1:
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
new file mode 100644
index 0000000..d671041
--- /dev/null
+++ b/Doc/distributing/index.rst
@@ -0,0 +1,143 @@
+.. _distributing-index:
+
+###############################
+ Distributing Python Modules
+###############################
+
+:Email: distutils-sig@python.org
+
+
+As a popular open source development project, Python has an active
+supporting community of contributors and users that also make their software
+available for other Python developers to use under open source license terms.
+
+This allows Python users to share and collaborate effectively, benefiting
+from the solutions others have already created to common (and sometimes
+even rare!) problems, as well as potentially contributing their own
+solutions to the common pool.
+
+This guide covers the distribution part of the process. For a guide to
+installing other Python projects, refer to the
+:ref:`installation guide <installing-index>`.
+
+.. note::
+
+ For corporate and other institutional users, be aware that many
+ organisations have their own policies around using and contributing to
+ open source software. Please take such policies into account when making
+ use of the distribution and installation tools provided with Python.
+
+
+Key terms
+=========
+
+* the `Python Package Index <https://pypi.python.org/pypi>`__ is a public
+ repository of open source licensed packages made available for use by
+ other Python users
+* the `Python Packaging Authority
+ <http://packaging.python.org/en/latest/future.html>`__ are the group of
+ developers and documentation authors responsible for the maintenance and
+ evolution of the standard packaging tools and the associated metadata and
+ file format standards. They maintain a variety of tools, documentation
+ and issue trackers on both `GitHub <https://github.com/pypa>`__ and
+ `BitBucket <https://bitbucket.org/pypa/>`__.
+* ``distutils`` is the original build and distribution system first added to
+ the Python standard library in 1998. While direct use of ``distutils`` is
+ being phased out, it still laid the foundation for the current packaging
+ and distribution infrastructure, and it not only remains part of the
+ standard library, but its name lives on in other ways (such as the name
+ of the mailing list used to coordinate Python packaging standards
+ development).
+
+
+Open source licensing and collaboration
+=======================================
+
+In most parts of the world, software is automatically covered by copyright.
+This means that other developers require explicit permission to copy, use,
+modify and redistribute the software.
+
+Open source licensing is a way of explicitly granting such permission in a
+relatively consistent way, allowing developers to share and collaborate
+efficiently by making common solutions to various problems freely available.
+This leaves many developers free to spend more time focusing on the problems
+that are relatively unique to their specific situation.
+
+The distribution tools provided with Python are designed to make it
+reasonably straightforward for developers to make their own contributions
+back to that common pool of software if they choose to do so.
+
+The same distribution tools can also be used to distribute software within
+an organisation, regardless of whether that software is published as open
+source software or not.
+
+
+Installing the tools
+====================
+
+The standard library does not include build tools that support modern
+Python packaging standards, as the core development team has found that it
+is important to have standard tools that work consistently, even on older
+versions of Python.
+
+The currently recommended build and distribution tools can be installed
+using ``pip``::
+
+ pip install setuptools wheel twine
+
+
+Reading the guide
+=================
+
+The Python Packaging User Guide covers the various key steps and elements
+involved in creating a project
+
+* `Project structure`_
+* `Building and packaging the project`_
+* `Uploading the project to the Python Package Index`_
+
+.. _Project structure: \
+ http://packaging.python.org/en/latest/tutorial.html#creating-your-own-project
+.. _Building and packaging the project: \
+ http://packaging.python.org/en/latest/tutorial.html#building-packaging-your-project
+.. _Uploading the project to the Python Package Index: \
+ http://packaging.python.org/en/latest/tutorial.html#uploading-your-project-to-pypi
+
+
+How do I...?
+============
+
+These are quick answers or links for some common tasks.
+
+... choose a name for my project?
+---------------------------------
+
+This isn't an easy topic, but here are a few tips:
+
+* check the Python Package Index to see if the name is already in use
+* check popular hosting sites like GitHub, BitBucket, etc to see if there
+ is already a project with that name
+* check what comes up in a web search for the name you're considering
+* avoid particularly common words, especially ones with multiple meanings,
+ as they can make it difficult for users to find your software when
+ searching for it
+
+
+... create and distribute binary extensions?
+--------------------------------------------
+
+This is actually quite a complex topic, with a variety of alternatives
+available depending on exactly what you're aiming to achieve. See the
+Python Packaging User Guide for more information and recommendations.
+
+.. seealso::
+
+ `Python Packaging User Guide: Binary Extensions
+ <http://packaging.python.org/en/latest/extensions.html>`__
+
+.. other topics:
+
+ Once the Development & Deployment part of PPUG is fleshed out, some of
+ those sections should be linked from new questions here (most notably,
+ we should have a question about avoiding depending on PyPI that links to
+ http://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches)
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 9853f02..e1357fa 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -853,17 +853,6 @@ Windows. It also contains the Mingw32CCompiler class which handles the mingw32
port of GCC (same as cygwin in no-cygwin mode).
-:mod:`distutils.emxccompiler` --- OS/2 EMX Compiler
-===================================================
-
-.. module:: distutils.emxccompiler
- :synopsis: OS/2 EMX Compiler support
-
-
-This module provides the EMXCCompiler class, a subclass of
-:class:`UnixCCompiler` that handles the EMX port of the GNU C compiler to OS/2.
-
-
:mod:`distutils.archive_util` --- Archiving utilities
======================================================
@@ -1004,7 +993,7 @@ directories.
Files in *src* that begin with :file:`.nfs` are skipped (more information on
these files is available in answer D2 of the `NFS FAQ page
- <http://nfs.sourceforge.net/#section_d>`_.
+ <http://nfs.sourceforge.net/#section_d>`_).
.. versionchanged:: 3.3.1
NFS files are ignored.
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index d1ab7db..83c68ae 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -239,7 +239,8 @@ tedious and error-prone, so it's usually best to put them in the setup
configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If
you distribute or package many Python module distributions, you might want to
put options that apply to all of them in your personal Distutils configuration
-file (:file:`~/.pydistutils.cfg`).
+file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable
+this file, you can pass the :option:`--no-user-cfg` option to :file:`setup.py`.
There are three steps to building a binary RPM package, all of which are
handled automatically by the Distutils:
diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst
index 1a6f04c..90d1c1a 100644
--- a/Doc/distutils/index.rst
+++ b/Doc/distutils/index.rst
@@ -1,8 +1,8 @@
.. _distutils-index:
-###############################
- Distributing Python Modules
-###############################
+##############################################
+ Distributing Python Modules (Legacy version)
+##############################################
:Authors: Greg Ward, Anthony Baxter
:Email: distutils-sig@python.org
diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst
index 1666436..427b7b1 100644
--- a/Doc/distutils/sourcedist.rst
+++ b/Doc/distutils/sourcedist.rst
@@ -26,16 +26,16 @@ to create a gzipped tarball and a zip file. The available formats are:
+===========+=========================+=========+
| ``zip`` | zip file (:file:`.zip`) | (1),(3) |
+-----------+-------------------------+---------+
-| ``gztar`` | gzip'ed tar file | (2),(4) |
+| ``gztar`` | gzip'ed tar file | \(2) |
| | (:file:`.tar.gz`) | |
+-----------+-------------------------+---------+
-| ``bztar`` | bzip2'ed tar file | \(4) |
+| ``bztar`` | bzip2'ed tar file | |
| | (:file:`.tar.bz2`) | |
+-----------+-------------------------+---------+
| ``ztar`` | compressed tar file | \(4) |
| | (:file:`.tar.Z`) | |
+-----------+-------------------------+---------+
-| ``tar`` | tar file (:file:`.tar`) | \(4) |
+| ``tar`` | tar file (:file:`.tar`) | |
+-----------+-------------------------+---------+
Notes:
@@ -51,8 +51,16 @@ Notes:
of the standard Python library since Python 1.6)
(4)
- requires external utilities: :program:`tar` and possibly one of :program:`gzip`,
- :program:`bzip2`, or :program:`compress`
+ requires the :program:`compress` program. Notice that this format is now
+ pending for deprecation and will be removed in the future versions of Python.
+
+When using any ``tar`` format (``gztar``, ``bztar``, ``ztar`` or
+``tar``), under Unix you can specify the ``owner`` and ``group`` names
+that will be set for each member of the archive.
+
+For example, if you want all files of the archive to be owned by root::
+
+ python setup.py sdist --owner=root --group=root
.. _manifest:
@@ -68,7 +76,7 @@ source distribution:
:option:`packages` options
* all C source files mentioned in the :option:`ext_modules` or
- :option:`libraries` options (
+ :option:`libraries` options
.. XXX getting C library sources currently broken---no
:meth:`get_source_files` method in :file:`build_clib.py`!
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index bd66086..6cb686a 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -285,14 +285,14 @@ be directly useful to you:
* ``pythonX.Y-config --cflags`` will give you the recommended flags when
compiling::
- $ /opt/bin/python3.3-config --cflags
- -I/opt/include/python3.3m -I/opt/include/python3.3m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes
+ $ /opt/bin/python3.4-config --cflags
+ -I/opt/include/python3.4m -I/opt/include/python3.4m -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes
* ``pythonX.Y-config --ldflags`` will give you the recommended flags when
linking::
- $ /opt/bin/python3.3-config --ldflags
- -L/opt/lib/python3.3/config-3.3m -lpthread -ldl -lutil -lm -lpython3.3m -Xlinker -export-dynamic
+ $ /opt/bin/python3.4-config --ldflags
+ -L/opt/lib/python3.4/config-3.4m -lpthread -ldl -lutil -lm -lpython3.4m -Xlinker -export-dynamic
.. note::
To avoid confusion between several Python installations (and especially
diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst
index 44a7f92..dd43926 100644
--- a/Doc/extending/index.rst
+++ b/Doc/extending/index.rst
@@ -21,14 +21,31 @@ Python) that give the language its wide application range.
For a detailed description of the whole Python/C API, see the separate
:ref:`c-api-index`.
-.. note::
- This guide only covers the basic tools for creating extensions provided
- as part of this version of CPython. Third party tools may offer simpler
- alternatives. Refer to the `binary extensions section
- <https://python-packaging-user-guide.readthedocs.org/en/latest/extensions.html>`__
- in the Python Packaging User Guide for more information.
+Recommended third party tools
+=============================
+This guide only covers the basic tools for creating extensions provided
+as part of this version of CPython. Third party tools like Cython,
+``cffi``, SWIG and Numba offer both simpler and more sophisticated
+approaches to creating C and C++ extensions for Python.
+
+.. seealso::
+
+ `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/en/latest/extensions.html>`_
+ The Python Packaging User Guide not only covers several available
+ tools that simplify the creation of binary extensions, but also
+ discusses the various reasons why creating an extension module may be
+ desirable in the first place.
+
+
+Creating extensions without third party tools
+=============================================
+
+This section of the guide covers creating C and C++ extensions without
+assistance from third party tools. It is intended primarily for creators
+of those tools, rather than being a recommended way to create your own
+C extensions.
.. toctree::
:maxdepth: 2
@@ -38,4 +55,17 @@ For a detailed description of the whole Python/C API, see the separate
newtypes.rst
building.rst
windows.rst
+
+Embedding the CPython runtime in a larger application
+=====================================================
+
+Sometimes, rather than creating an extension that runs inside the Python
+interpreter as the main application, it is desirable to instead embed
+the CPython runtime inside a larger application. This section covers
+some of the details involved in doing that successfully.
+
+.. toctree::
+ :maxdepth: 2
+ :numbered:
+
embedding.rst
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index f484ba4..45b5721 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -157,7 +157,8 @@ to :const:`Py_TPFLAGS_DEFAULT`. ::
Py_TPFLAGS_DEFAULT, /* tp_flags */
All types should include this constant in their flags. It enables all of the
-members defined by the current version of Python.
+members defined until at least Python 3.3. If you need further members,
+you will need to OR the corresponding flags.
We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. ::
@@ -928,8 +929,9 @@ Finalization and De-allocation
This function is called when the reference count of the instance of your type is
reduced to zero and the Python interpreter wants to reclaim it. If your type
-has memory to free or other clean-up to perform, put it here. The object itself
-needs to be freed here as well. Here is an example of this function::
+has memory to free or other clean-up to perform, you can put it here. The
+object itself needs to be freed here as well. Here is an example of this
+function::
static void
newdatatype_dealloc(newdatatypeobject * obj)
@@ -980,6 +982,22 @@ done. This can be done using the :c:func:`PyErr_Fetch` and
Py_TYPE(obj)->tp_free((PyObject*)self);
}
+.. note::
+ There are limitations to what you can safely do in a deallocator function.
+ First, if your type supports garbage collection (using :c:member:`~PyTypeObject.tp_traverse`
+ and/or :c:member:`~PyTypeObject.tp_clear`), some of the object's members can have been
+ cleared or finalized by the time :c:member:`~PyTypeObject.tp_dealloc` is called. Second, in
+ :c:member:`~PyTypeObject.tp_dealloc`, your object is in an unstable state: its reference
+ count is equal to zero. Any call to a non-trivial object or API (as in the
+ example above) might end up calling :c:member:`~PyTypeObject.tp_dealloc` again, causing a
+ double free and a crash.
+
+ Starting with Python 3.4, it is recommended not to put any complex
+ finalization code in :c:member:`~PyTypeObject.tp_dealloc`, and instead use the new
+ :c:member:`~PyTypeObject.tp_finalize` type method.
+
+ .. seealso::
+ :pep:`442` explains the new finalization scheme.
.. index::
single: string; object representation
diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst
index a9a234b..d196e86 100644
--- a/Doc/faq/extending.rst
+++ b/Doc/faq/extending.rst
@@ -95,8 +95,8 @@ To test the type of an object, first make sure it isn't *NULL*, and then use
There is also a high-level API to Python objects which is provided by the
so-called 'abstract' interface -- read ``Include/abstract.h`` for further
details. It allows interfacing with any kind of Python sequence using calls
-like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc.) as well
-as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et.
+like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc. as well
+as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et
al.) and mappings in the PyMapping APIs.
diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst
index 2566932..5f4ff17 100644
--- a/Doc/faq/library.rst
+++ b/Doc/faq/library.rst
@@ -211,7 +211,7 @@ using curses, but curses is a fairly large module to learn.
try:
c = sys.stdin.read(1)
print("Got character", repr(c))
- except IOError:
+ except OSError:
pass
finally:
termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
@@ -224,7 +224,11 @@ using curses, but curses is a fairly large module to learn.
:func:`termios.tcsetattr` turns off stdin's echoing and disables canonical
mode. :func:`fcntl.fnctl` is used to obtain stdin's file descriptor flags
and modify them for non-blocking mode. Since reading stdin when it is empty
- results in an :exc:`IOError`, this error is caught and ignored.
+ results in an :exc:`OSError`, this error is caught and ignored.
+
+ .. versionchanged:: 3.3
+ *sys.stdin.read* used to raise :exc:`IOError`. Starting from Python 3.3
+ :exc:`IOError` is alias for :exc:`OSError`.
Threads
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index 280d5e1..d514a80 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -1751,12 +1751,12 @@ When I edit an imported module and reimport it, the changes don't show up. Why
For reasons of efficiency as well as consistency, Python only reads the module
file on the first time a module is imported. If it didn't, in a program
consisting of many modules where each one imports the same basic module, the
-basic module would be parsed and re-parsed many times. To force rereading of a
+basic module would be parsed and re-parsed many times. To force re-reading of a
changed module, do this::
- import imp
+ import importlib
import modname
- imp.reload(modname)
+ importlib.reload(modname)
Warning: this technique is not 100% fool-proof. In particular, modules
containing statements like ::
@@ -1768,10 +1768,10 @@ module contains class definitions, existing class instances will *not* be
updated to use the new class definition. This can result in the following
paradoxical behaviour:
- >>> import imp
+ >>> import importlib
>>> import cls
>>> c = cls.C() # Create an instance of C
- >>> imp.reload(cls)
+ >>> importlib.reload(cls)
<module 'cls' from 'cls.py'>
>>> isinstance(c, cls.C) # isinstance is false?!?
False
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index b48eb1e..f71a1f7 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -310,6 +310,15 @@ Glossary
>>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81
285
+ generic function
+ A function composed of multiple functions implementing the same operation
+ for different types. Which implementation should be used during a call is
+ determined by the dispatch algorithm.
+
+ See also the :term:`single dispatch` glossary entry, the
+ :func:`functools.singledispatch` decorator, and :pep:`443`.
+
+
GIL
See :term:`global interpreter lock`.
@@ -530,6 +539,10 @@ Glossary
See also :term:`package`.
+ module spec
+ A namespace containing the import-related information used to load a
+ module.
+
MRO
See :term:`method resolution order`.
@@ -671,20 +684,27 @@ Glossary
positional argument
See :term:`argument`.
- provisional package
- A provisional package is one which has been deliberately excluded from
+ provisional API
+ A provisional API is one which has been deliberately excluded from
the standard library's backwards compatibility guarantees. While major
- changes to such packages are not expected, as long as they are marked
+ changes to such interfaces are not expected, as long as they are marked
provisional, backwards incompatible changes (up to and including removal
- of the package) may occur if deemed necessary by core developers. Such
+ of the interface) may occur if deemed necessary by core developers. Such
changes will not be made gratuitously -- they will occur only if serious
- flaws are uncovered that were missed prior to the inclusion of the
- package.
+ fundamental flaws are uncovered that were missed prior to the inclusion
+ of the API.
+
+ Even for provisional APIs, backwards incompatible changes are seen as
+ a "solution of last resort" - every attempt will still be made to find
+ a backwards compatible resolution to any identified problems.
This process allows the standard library to continue to evolve over
time, without locking in problematic design errors for extended periods
of time. See :pep:`411` for more details.
+ provisional package
+ See :term:`provisional API`.
+
Python 3000
Nickname for the Python 3.x release line (coined long ago when the
release of version 3 was something in the distant future.) This is also
@@ -771,6 +791,10 @@ Glossary
interface can be registered explicitly using
:func:`~abc.register`.
+ single dispatch
+ A form of :term:`generic function` dispatch where the implementation is
+ chosen based on the type of a single argument.
+
slice
An object usually containing a portion of a :term:`sequence`. A slice is
created using the subscript notation, ``[]`` with colons between numbers
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
new file mode 100644
index 0000000..750ddbe
--- /dev/null
+++ b/Doc/howto/clinic.rst
@@ -0,0 +1,1686 @@
+**********************
+Argument Clinic How-To
+**********************
+
+:author: Larry Hastings
+
+
+.. topic:: Abstract
+
+ Argument Clinic is a preprocessor for CPython C files.
+ Its purpose is to automate all the boilerplate involved
+ with writing argument parsing code for "builtins".
+ This document shows you how to convert your first C
+ function to work with Argument Clinic, and then introduces
+ some advanced topics on Argument Clinic usage.
+
+ Currently Argument Clinic is considered internal-only
+ for CPython. Its use is not supported for files outside
+ CPython, and no guarantees are made regarding backwards
+ compatibility for future versions. In other words: if you
+ maintain an external C extension for CPython, you're welcome
+ to experiment with Argument Clinic in your own code. But the
+ version of Argument Clinic that ships with CPython 3.5 *could*
+ be totally incompatible and break all your code.
+
+The Goals Of Argument Clinic
+============================
+
+Argument Clinic's primary goal
+is to take over responsibility for all argument parsing code
+inside CPython. This means that, when you convert a function
+to work with Argument Clinic, that function should no longer
+do any of its own argument parsing--the code generated by
+Argument Clinic should be a "black box" to you, where CPython
+calls in at the top, and your code gets called at the bottom,
+with ``PyObject *args`` (and maybe ``PyObject *kwargs``)
+magically converted into the C variables and types you need.
+
+In order for Argument Clinic to accomplish its primary goal,
+it must be easy to use. Currently, working with CPython's
+argument parsing library is a chore, requiring maintaining
+redundant information in a surprising number of places.
+When you use Argument Clinic, you don't have to repeat yourself.
+
+Obviously, no one would want to use Argument Clinic unless
+it's solving their problem--and without creating new problems of
+its own.
+So it's paramount that Argument Clinic generate correct code.
+It'd be nice if the code was faster, too, but at the very least
+it should not introduce a major speed regression. (Eventually Argument
+Clinic *should* make a major speedup possible--we could
+rewrite its code generator to produce tailor-made argument
+parsing code, rather than calling the general-purpose CPython
+argument parsing library. That would make for the fastest
+argument parsing possible!)
+
+Additionally, Argument Clinic must be flexible enough to
+work with any approach to argument parsing. Python has
+some functions with some very strange parsing behaviors;
+Argument Clinic's goal is to support all of them.
+
+Finally, the original motivation for Argument Clinic was
+to provide introspection "signatures" for CPython builtins.
+It used to be, the introspection query functions would throw
+an exception if you passed in a builtin. With Argument
+Clinic, that's a thing of the past!
+
+One idea you should keep in mind, as you work with
+Argument Clinic: the more information you give it, the
+better job it'll be able to do.
+Argument Clinic is admittedly relatively simple right
+now. But as it evolves it will get more sophisticated,
+and it should be able to do many interesting and smart
+things with all the information you give it.
+
+
+Basic Concepts And Usage
+========================
+
+Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``.
+If you run that script, specifying a C file as an argument::
+
+ % python3 Tools/clinic/clinic.py foo.c
+
+Argument Clinic will scan over the file looking for lines that
+look exactly like this::
+
+ /*[clinic input]
+
+When it finds one, it reads everything up to a line that looks
+exactly like this::
+
+ [clinic start generated code]*/
+
+Everything in between these two lines is input for Argument Clinic.
+All of these lines, including the beginning and ending comment
+lines, are collectively called an Argument Clinic "block".
+
+When Argument Clinic parses one of these blocks, it
+generates output. This output is rewritten into the C file
+immediately after the block, followed by a comment containing a checksum.
+The Argument Clinic block now looks like this::
+
+ /*[clinic input]
+ ... clinic input goes here ...
+ [clinic start generated code]*/
+ ... clinic output goes here ...
+ /*[clinic end generated code: checksum=...]*/
+
+If you run Argument Clinic on the same file a second time, Argument Clinic
+will discard the old output and write out the new output with a fresh checksum
+line. However, if the input hasn't changed, the output won't change either.
+
+You should never modify the output portion of an Argument Clinic block. Instead,
+change the input until it produces the output you want. (That's the purpose of the
+checksum--to detect if someone changed the output, as these edits would be lost
+the next time Argument Clinic writes out fresh output.)
+
+For the sake of clarity, here's the terminology we'll use with Argument Clinic:
+
+* The first line of the comment (``/*[clinic input]``) is the *start line*.
+* The last line of the initial comment (``[clinic start generated code]*/``) is the *end line*.
+* The last line (``/*[clinic end generated code: checksum=...]*/``) is the *checksum line*.
+* In between the start line and the end line is the *input*.
+* In between the end line and the checksum line is the *output*.
+* All the text collectively, from the start line to the checksum line inclusively,
+ is the *block*. (A block that hasn't been successfully processed by Argument
+ Clinic yet doesn't have output or a checksum line, but it's still considered
+ a block.)
+
+
+Converting Your First Function
+==============================
+
+The best way to get a sense of how Argument Clinic works is to
+convert a function to work with it. Here, then, are the bare
+minimum steps you'd need to follow to convert a function to
+work with Argument Clinic. Note that for code you plan to
+check in to CPython, you really should take the conversion farther,
+using some of the advanced concepts you'll see later on in
+the document (like "return converters" and "self converters").
+But we'll keep it simple for this walkthrough so you can learn.
+
+Let's dive in!
+
+0. Make sure you're working with a freshly updated checkout
+ of the CPython trunk.
+
+1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple`
+ or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted
+ to work with Argument Clinic yet.
+ For my example I'm using ``_pickle.Pickler.dump()``.
+
+2. If the call to the ``PyArg_Parse`` function uses any of the
+ following format units::
+
+ O&
+ O!
+ es
+ es#
+ et
+ et#
+
+ or if it has multiple calls to :c:func:`PyArg_ParseTuple`,
+ you should choose a different function. Argument Clinic *does*
+ support all of these scenarios. But these are advanced
+ topics--let's do something simpler for your first function.
+
+ Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple`
+ or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different
+ types for the same argument, or if the function uses something besides
+ PyArg_Parse functions to parse its arguments, it probably
+ isn't suitable for conversion to Argument Clinic. Argument Clinic
+ doesn't support generic functions or polymorphic parameters.
+
+3. Add the following boilerplate above the function, creating our block::
+
+ /*[clinic input]
+ [clinic start generated code]*/
+
+4. Cut the docstring and paste it in between the ``[clinic]`` lines,
+ removing all the junk that makes it a properly quoted C string.
+ When you're done you should have just the text, based at the left
+ margin, with no line wider than 80 characters.
+ (Argument Clinic will preserve indents inside the docstring.)
+
+ If the old docstring had a first line that looked like a function
+ signature, throw that line away. (The docstring doesn't need it
+ anymore--when you use ``help()`` on your builtin in the future,
+ the first line will be built automatically based on the function's
+ signature.)
+
+ Sample::
+
+ /*[clinic input]
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+5. If your docstring doesn't have a "summary" line, Argument Clinic will
+ complain. So let's make sure it has one. The "summary" line should
+ be a paragraph consisting of a single 80-column line
+ at the beginning of the docstring.
+
+ (Our example docstring consists solely of a summary line, so the sample
+ code doesn't have to change for this step.)
+
+6. Above the docstring, enter the name of the function, followed
+ by a blank line. This should be the Python name of the function,
+ and should be the full dotted path
+ to the function--it should start with the name of the module,
+ include any sub-modules, and if the function is a method on
+ a class it should include the class name too.
+
+ Sample::
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+7. If this is the first time that module or class has been used with Argument
+ Clinic in this C file,
+ you must declare the module and/or class. Proper Argument Clinic hygiene
+ prefers declaring these in a separate block somewhere near the
+ top of the C file, in the same way that include files and statics go at
+ the top. (In our sample code we'll just show the two blocks next to
+ each other.)
+
+ The name of the class and module should be the same as the one
+ seen by Python. Check the name defined in the :c:type:`PyModuleDef`
+ or :c:type:`PyTypeObject` as appropriate.
+
+ When you declare a class, you must also specify two aspects of its type
+ in C: the type declaration you'd use for a pointer to an instance of
+ this class, and a pointer to the :c:type:`PyTypeObject` for this class.
+
+ Sample::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+
+
+
+8. Declare each of the parameters to the function. Each parameter
+ should get its own line. All the parameter lines should be
+ indented from the function name and the docstring.
+
+ The general form of these parameter lines is as follows::
+
+ name_of_parameter: converter
+
+ If the parameter has a default value, add that after the
+ converter::
+
+ name_of_parameter: converter = default_value
+
+ Argument Clinic's support for "default values" is quite sophisticated;
+ please see :ref:`the section below on default values <default_values>`
+ for more information.
+
+ Add a blank line below the parameters.
+
+ What's a "converter"? It establishes both the type
+ of the variable used in C, and the method to convert the Python
+ value into a C value at runtime.
+ For now you're going to use what's called a "legacy converter"--a
+ convenience syntax intended to make porting old code into Argument
+ Clinic easier.
+
+ For each parameter, copy the "format unit" for that
+ parameter from the ``PyArg_Parse()`` format argument and
+ specify *that* as its converter, as a quoted
+ string. ("format unit" is the formal name for the one-to-three
+ character substring of the ``format`` parameter that tells
+ the argument parsing function what the type of the variable
+ is and how to convert it. For more on format units please
+ see :ref:`arg-parsing`.)
+
+ For multicharacter format units like ``z#``, use the
+ entire two-or-three character string.
+
+ Sample::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ obj: 'O'
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+9. If your function has ``|`` in the format string, meaning some
+ parameters have default values, you can ignore it. Argument
+ Clinic infers which parameters are optional based on whether
+ or not they have default values.
+
+ If your function has ``$`` in the format string, meaning it
+ takes keyword-only arguments, specify ``*`` on a line by
+ itself before the first keyword-only argument, indented the
+ same as the parameter lines.
+
+ (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.)
+
+
+10. If the existing C function calls :c:func:`PyArg_ParseTuple`
+ (as opposed to :c:func:`PyArg_ParseTupleAndKeywords`), then all its
+ arguments are positional-only.
+
+ To mark all parameters as positional-only in Argument Clinic,
+ add a ``/`` on a line by itself after the last parameter,
+ indented the same as the parameter lines.
+
+ Currently this is all-or-nothing; either all parameters are
+ positional-only, or none of them are. (In the future Argument
+ Clinic may relax this restriction.)
+
+ Sample::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ obj: 'O'
+ /
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+11. It's helpful to write a per-parameter docstring for each parameter.
+ But per-parameter docstrings are optional; you can skip this step
+ if you prefer.
+
+ Here's how to add a per-parameter docstring. The first line
+ of the per-parameter docstring must be indented further than the
+ parameter definition. The left margin of this first line establishes
+ the left margin for the whole per-parameter docstring; all the text
+ you write will be outdented by this amount. You can write as much
+ text as you like, across multiple lines if you wish.
+
+ Sample::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ obj: 'O'
+ The object to be pickled.
+ /
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+12. Save and close the file, then run ``Tools/clinic/clinic.py`` on it.
+ With luck everything worked and your block now has output! Reopen
+ the file in your text editor to see::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+ /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ obj: 'O'
+ The object to be pickled.
+ /
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+ PyDoc_STRVAR(_pickle_Pickler_dump__doc__,
+ "Write a pickled representation of obj to the open file.\n"
+ "\n"
+ ...
+ static PyObject *
+ _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj)
+ /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/
+
+ Obviously, if Argument Clinic didn't produce any output, it's because
+ it found an error in your input. Keep fixing your errors and retrying
+ until Argument Clinic processes your file without complaint.
+
+13. Double-check that the argument-parsing code Argument Clinic generated
+ looks basically the same as the existing code.
+
+ First, ensure both places use the same argument-parsing function.
+ The existing code must call either
+ :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`;
+ ensure that the code generated by Argument Clinic calls the
+ *exact* same function.
+
+ Second, the format string passed in to :c:func:`PyArg_ParseTuple` or
+ :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same
+ as the hand-written one in the existing function, up to the colon
+ or semi-colon.
+
+ (Argument Clinic always generates its format strings
+ with a ``:`` followed by the name of the function. If the
+ existing code's format string ends with ``;``, to provide
+ usage help, this change is harmless--don't worry about it.)
+
+ Third, for parameters whose format units require two arguments
+ (like a length variable, or an encoding string, or a pointer
+ to a conversion function), ensure that the second argument is
+ *exactly* the same between the two invocations.
+
+ Fourth, inside the output portion of the block you'll find a preprocessor
+ macro defining the appropriate static :c:type:`PyMethodDef` structure for
+ this builtin::
+
+ #define __PICKLE_PICKLER_DUMP_METHODDEF \
+ {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__},
+
+ This static structure should be *exactly* the same as the existing static
+ :c:type:`PyMethodDef` structure for this builtin.
+
+ If any of these items differ in *any way*,
+ adjust your Argument Clinic function specification and rerun
+ ``Tools/clinic/clinic.py`` until they *are* the same.
+
+
+14. Notice that the last line of its output is the declaration
+ of your "impl" function. This is where the builtin's implementation goes.
+ Delete the existing prototype of the function you're modifying, but leave
+ the opening curly brace. Now delete its argument parsing code and the
+ declarations of all the variables it dumps the arguments into.
+ Notice how the Python arguments are now arguments to this impl function;
+ if the implementation used different names for these variables, fix it.
+
+ Let's reiterate, just because it's kind of weird. Your code should now
+ look like this::
+
+ static return_type
+ your_function_impl(...)
+ /*[clinic end generated code: checksum=...]*/
+ {
+ ...
+
+ Argument Clinic generated the checksum line and the function prototype just
+ above it. You should write the opening (and closing) curly braces for the
+ function, and the implementation inside.
+
+ Sample::
+
+ /*[clinic input]
+ module _pickle
+ class _pickle.Pickler "PicklerObject *" "&Pickler_Type"
+ [clinic start generated code]*/
+ /*[clinic end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
+ /*[clinic input]
+ _pickle.Pickler.dump
+
+ obj: 'O'
+ The object to be pickled.
+ /
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+ PyDoc_STRVAR(__pickle_Pickler_dump__doc__,
+ "Write a pickled representation of obj to the open file.\n"
+ "\n"
+ ...
+ static PyObject *
+ _pickle_Pickler_dump_impl(PicklerObject *self, PyObject *obj)
+ /*[clinic end generated code: checksum=3bd30745bf206a48f8b576a1da3d90f55a0a4187]*/
+ {
+ /* Check whether the Pickler was initialized correctly (issue3664).
+ Developers often forget to call __init__() in their subclasses, which
+ would trigger a segfault without this check. */
+ if (self->write == NULL) {
+ PyErr_Format(PicklingError,
+ "Pickler.__init__() was not called by %s.__init__()",
+ Py_TYPE(self)->tp_name);
+ return NULL;
+ }
+
+ if (_Pickler_ClearBuffer(self) < 0)
+ return NULL;
+
+ ...
+
+15. Remember the macro with the :c:type:`PyMethodDef` structure for this
+ function? Find the existing :c:type:`PyMethodDef` structure for this
+ function and replace it with a reference to the macro. (If the builtin
+ is at module scope, this will probably be very near the end of the file;
+ if the builtin is a class method, this will probably be below but relatively
+ near to the implementation.)
+
+ Note that the body of the macro contains a trailing comma. So when you
+ replace the existing static :c:type:`PyMethodDef` structure with the macro,
+ *don't* add a comma to the end.
+
+ Sample::
+
+ static struct PyMethodDef Pickler_methods[] = {
+ __PICKLE_PICKLER_DUMP_METHODDEF
+ __PICKLE_PICKLER_CLEAR_MEMO_METHODDEF
+ {NULL, NULL} /* sentinel */
+ };
+
+
+16. Compile, then run the relevant portions of the regression-test suite.
+ This change should not introduce any new compile-time warnings or errors,
+ and there should be no externally-visible change to Python's behavior.
+
+ Well, except for one difference: ``inspect.signature()`` run on your function
+ should now provide a valid signature!
+
+ Congratulations, you've ported your first function to work with Argument Clinic!
+
+Advanced Topics
+===============
+
+Now that you've had some experience working with Argument Clinic, it's time
+for some advanced topics.
+
+
+Symbolic default values
+-----------------------
+
+The default value you provide for a parameter can't be any arbitrary
+expression. Currently the following are explicitly supported:
+
+* Numeric constants (integer and float)
+* String constants
+* ``True``, ``False``, and ``None``
+* Simple symbolic constants like ``sys.maxsize``, which must
+ start with the name of the module
+
+In case you're curious, this is implemented in ``from_builtin()``
+in ``Lib/inspect.py``.
+
+(In the future, this may need to get even more elaborate,
+to allow full expressions like ``CONSTANT - 1``.)
+
+
+Renaming the C functions and variables generated by Argument Clinic
+-------------------------------------------------------------------
+
+Argument Clinic automatically names the functions it generates for you.
+Occasionally this may cause a problem, if the generated name collides with
+the name of an existing C function. There's an easy solution: override the names
+used for the C functions. Just add the keyword ``"as"``
+to your function declaration line, followed by the function name you wish to use.
+Argument Clinic will use that function name for the base (generated) function,
+then add ``"_impl"`` to the end and use that for the name of the impl function.
+
+For example, if we wanted to rename the C function names generated for
+``pickle.Pickler.dump``, it'd look like this::
+
+ /*[clinic input]
+ pickle.Pickler.dump as pickler_dumper
+
+ ...
+
+The base function would now be named ``pickler_dumper()``,
+and the impl function would now be named ``pickler_dumper_impl()``.
+
+
+Similarly, you may have a problem where you want to give a parameter
+a specific Python name, but that name may be inconvenient in C. Argument
+Clinic allows you to give a parameter different names in Python and in C,
+using the same ``"as"`` syntax::
+
+ /*[clinic input]
+ pickle.Pickler.dump
+
+ obj: object
+ file as file_obj: object
+ protocol: object = NULL
+ *
+ fix_imports: bool = True
+
+Here, the name used in Python (in the signature and the ``keywords``
+array) would be ``file``, but the C variable would be named ``file_obj``.
+
+You can use this to rename the ``self`` parameter too!
+
+
+Converting functions using PyArg_UnpackTuple
+--------------------------------------------
+
+To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`,
+simply write out all the arguments, specifying each as an ``object``. You
+may specify the ``type`` argument to cast the type as appropriate. All
+arguments should be marked positional-only (add a ``/`` on a line by itself
+after the last argument).
+
+Currently the generated code will use :c:func:`PyArg_ParseTuple`, but this
+will change soon.
+
+Optional Groups
+---------------
+
+Some legacy functions have a tricky approach to parsing their arguments:
+they count the number of positional arguments, then use a ``switch`` statement
+to call one of several different :c:func:`PyArg_ParseTuple` calls depending on
+how many positional arguments there are. (These functions cannot accept
+keyword-only arguments.) This approach was used to simulate optional
+arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created.
+
+While functions using this approach can often be converted to
+use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values,
+it's not always possible. Some of these legacy functions have
+behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support.
+The most obvious example is the builtin function ``range()``, which has
+an optional argument on the *left* side of its required argument!
+Another example is ``curses.window.addch()``, which has a group of two
+arguments that must always be specified together. (The arguments are
+called ``x`` and ``y``; if you call the function passing in ``x``,
+you must also pass in ``y``--and if you don't pass in ``x`` you may not
+pass in ``y`` either.)
+
+In any case, the goal of Argument Clinic is to support argument parsing
+for all existing CPython builtins without changing their semantics.
+Therefore Argument Clinic supports
+this alternate approach to parsing, using what are called *optional groups*.
+Optional groups are groups of arguments that must all be passed in together.
+They can be to the left or the right of the required arguments. They
+can *only* be used with positional-only parameters.
+
+.. note:: Optional groups are *only* intended for use when converting
+ functions that make multiple calls to :c:func:`PyArg_ParseTuple`!
+ Functions that use *any* other approach for parsing arguments
+ should *almost never* be converted to Argument Clinic using
+ optional groups. Functions using optional groups currently
+ cannot have accurate sigantures in Python, because Python just
+ doesn't understand the concept. Please avoid using optional
+ groups wherever possible.
+
+To specify an optional group, add a ``[`` on a line by itself before
+the parameters you wish to group together, and a ``]`` on a line by itself
+after these parameters. As an example, here's how ``curses.window.addch``
+uses optional groups to make the first two parameters and the last
+parameter optional::
+
+ /*[clinic input]
+
+ curses.window.addch
+
+ [
+ x: int
+ X-coordinate.
+ y: int
+ Y-coordinate.
+ ]
+
+ ch: object
+ Character to add.
+
+ [
+ attr: long
+ Attributes for the character.
+ ]
+ /
+
+ ...
+
+
+Notes:
+
+* For every optional group, one additional parameter will be passed into the
+ impl function representing the group. The parameter will be an int named
+ ``group_{direction}_{number}``,
+ where ``{direction}`` is either ``right`` or ``left`` depending on whether the group
+ is before or after the required parameters, and ``{number}`` is a monotonically
+ increasing number (starting at 1) indicating how far away the group is from
+ the required parameters. When the impl is called, this parameter will be set
+ to zero if this group was unused, and set to non-zero if this group was used.
+ (By used or unused, I mean whether or not the parameters received arguments
+ in this invocation.)
+
+* If there are no required arguments, the optional groups will behave
+ as if they're to the right of the required arguments.
+
+* In the case of ambiguity, the argument parsing code
+ favors parameters on the left (before the required parameters).
+
+* Optional groups can only contain positional-only parameters.
+
+* Optional groups are *only* intended for legacy code. Please do not
+ use optional groups for new code.
+
+
+Using real Argument Clinic converters, instead of "legacy converters"
+---------------------------------------------------------------------
+
+To save time, and to minimize how much you need to learn
+to achieve your first port to Argument Clinic, the walkthrough above tells
+you to use "legacy converters". "Legacy converters" are a convenience,
+designed explicitly to make porting existing code to Argument Clinic
+easier. And to be clear, their use is acceptable when porting code for
+Python 3.4.
+
+However, in the long term we probably want all our blocks to
+use Argument Clinic's real syntax for converters. Why? A couple
+reasons:
+
+* The proper converters are far easier to read and clearer in their intent.
+* There are some format units that are unsupported as "legacy converters",
+ because they require arguments, and the legacy converter syntax doesn't
+ support specifying arguments.
+* In the future we may have a new argument parsing library that isn't
+ restricted to what :c:func:`PyArg_ParseTuple` supports; this flexibility
+ won't be available to parameters using legacy converters.
+
+Therefore, if you don't mind a little extra effort, please use the normal
+converters instead of legacy converters.
+
+In a nutshell, the syntax for Argument Clinic (non-legacy) converters
+looks like a Python function call. However, if there are no explicit
+arguments to the function (all functions take their default values),
+you may omit the parentheses. Thus ``bool`` and ``bool()`` are exactly
+the same converters.
+
+All arguments to Argument Clinic converters are keyword-only.
+All Argument Clinic converters accept the following arguments:
+
+ ``c_default``
+ The default value for this parameter when defined in C.
+ Specifically, this will be the initializer for the variable declared
+ in the "parse function". See :ref:`the section on default values <default_values>`
+ for how to use this.
+ Specified as a string.
+
+ ``annotation``
+ The annotation value for this parameter. Not currently supported,
+ because PEP 8 mandates that the Python library may not use
+ annotations.
+
+In addition, some converters accept additional arguments. Here is a list
+of these arguments, along with their meanings:
+
+ ``bitwise``
+ Only supported for unsigned integers. The native integer value of this
+ Python argument will be written to the parameter without any range checking,
+ even for negative values.
+
+ ``converter``
+ Only supported by the ``object`` converter. Specifies the name of a
+ :ref:`C "converter function" <o_ampersand>`
+ to use to convert this object to a native type.
+
+ ``encoding``
+ Only supported for strings. Specifies the encoding to use when converting
+ this string from a Python str (Unicode) value into a C ``char *`` value.
+
+ ``length``
+ Only supported for strings. If true, requests that the length of the
+ string be passed in to the impl function, just after the string parameter,
+ in a parameter named ``<parameter_name>_length``.
+
+ ``nullable``
+ Only supported for strings. If true, this parameter may also be set to
+ ``None``, in which case the C parameter will be set to ``NULL``.
+
+ ``subclass_of``
+ Only supported for the ``object`` converter. Requires that the Python
+ value be a subclass of a Python type, as expressed in C.
+
+ ``types``
+ Only supported for the ``object`` (and ``self``) converter. Specifies
+ the C type that will be used to declare the variable. Default value is
+ ``"PyObject *"``.
+
+ ``types``
+ A string containing a list of Python types (and possibly pseudo-types);
+ this restricts the allowable Python argument to values of these types.
+ (This is not a general-purpose facility; as a rule it only supports
+ specific lists of types as shown in the legacy converter table.)
+
+ ``zeroes``
+ Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are
+ permitted inside the value.
+
+Please note, not every possible combination of arguments will work.
+Often these arguments are implemented internally by specific ``PyArg_ParseTuple``
+*format units*, with specific behavior. For example, currently you cannot
+call ``str`` and pass in ``zeroes=True`` without also specifying an ``encoding``;
+although it's perfectly reasonable to think this would work, these semantics don't
+map to any existing format unit. So Argument Clinic doesn't support it. (Or, at
+least, not yet.)
+
+Below is a table showing the mapping of legacy converters into real
+Argument Clinic converters. On the left is the legacy converter,
+on the right is the text you'd replace it with.
+
+========= =================================================================================
+``'B'`` ``unsigned_char(bitwise=True)``
+``'b'`` ``unsigned_char``
+``'c'`` ``char``
+``'C'`` ``int(types='str')``
+``'d'`` ``double``
+``'D'`` ``Py_complex``
+``'es#'`` ``str(encoding='name_of_encoding', length=True, zeroes=True)``
+``'es'`` ``str(encoding='name_of_encoding')``
+``'et#'`` ``str(encoding='name_of_encoding', types='bytes bytearray str', length=True)``
+``'et'`` ``str(encoding='name_of_encoding', types='bytes bytearray str')``
+``'f'`` ``float``
+``'h'`` ``short``
+``'H'`` ``unsigned_short(bitwise=True)``
+``'i'`` ``int``
+``'I'`` ``unsigned_int(bitwise=True)``
+``'k'`` ``unsigned_long(bitwise=True)``
+``'K'`` ``unsigned_PY_LONG_LONG(bitwise=True)``
+``'L'`` ``PY_LONG_LONG``
+``'n'`` ``Py_ssize_t``
+``'O!'`` ``object(subclass_of='&PySomething_Type')``
+``'O&'`` ``object(converter='name_of_c_function')``
+``'O'`` ``object``
+``'p'`` ``bool``
+``'s#'`` ``str(length=True)``
+``'S'`` ``PyBytesObject``
+``'s'`` ``str``
+``'s*'`` ``Py_buffer(types='str bytes bytearray buffer')``
+``'u#'`` ``Py_UNICODE(length=True)``
+``'u'`` ``Py_UNICODE``
+``'U'`` ``unicode``
+``'w*'`` ``Py_buffer(types='bytearray rwbuffer')``
+``'y#'`` ``str(types='bytes', length=True)``
+``'Y'`` ``PyByteArrayObject``
+``'y'`` ``str(types='bytes')``
+``'y*'`` ``Py_buffer``
+``'Z#'`` ``Py_UNICODE(nullable=True, length=True)``
+``'z#'`` ``str(nullable=True, length=True)``
+``'Z'`` ``Py_UNICODE(nullable=True)``
+``'z'`` ``str(nullable=True)``
+``'z*'`` ``Py_buffer(types='str bytes bytearray buffer', nullable=True)``
+========= =================================================================================
+
+As an example, here's our sample ``pickle.Pickler.dump`` using the proper
+converter::
+
+ /*[clinic input]
+ pickle.Pickler.dump
+
+ obj: object
+ The object to be pickled.
+ /
+
+ Write a pickled representation of obj to the open file.
+ [clinic start generated code]*/
+
+Argument Clinic will show you all the converters it has
+available. For each converter it'll show you all the parameters
+it accepts, along with the default value for each parameter.
+Just run ``Tools/clinic/clinic.py --converters`` to see the full list.
+
+Py_buffer
+---------
+
+When using the ``Py_buffer`` converter
+(or the ``'s*'``, ``'w*'``, ``'*y'``, or ``'z*'`` legacy converters),
+you *must* not call :c:func:`PyBuffer_Release` on the provided buffer.
+Argument Clinic generates code that does it for you (in the parsing function).
+
+
+
+Advanced converters
+-------------------
+
+Remeber those format units you skipped for your first
+time because they were advanced? Here's how to handle those too.
+
+The trick is, all those format units take arguments--either
+conversion functions, or types, or strings specifying an encoding.
+(But "legacy converters" don't support arguments. That's why we
+skipped them for your first function.) The argument you specified
+to the format unit is now an argument to the converter; this
+argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``),
+or ``encoding`` (for all the format units that start with ``e``).
+
+When using ``subclass_of``, you may also want to use the other
+custom argument for ``object()``: ``type``, which lets you set the type
+actually used for the parameter. For example, if you want to ensure
+that the object is a subclass of ``PyUnicode_Type``, you probably want
+to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``.
+
+One possible problem with using Argument Clinic: it takes away some possible
+flexibility for the format units starting with ``e``. When writing a
+``PyArg_Parse`` call by hand, you could theoretically decide at runtime what
+encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must
+be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate;
+it made supporting this format unit much easier, and may allow for future optimizations.
+This restriction doesn't seem unreasonable; CPython itself always passes in static
+hard-coded encoding strings for parameters whose format units start with ``e``.
+
+
+.. _default_values:
+
+Parameter default values
+------------------------
+
+Default values for parameters can be any of a number of values.
+At their simplest, they can be string, int, or float literals::
+
+ foo: str = "abc"
+ bar: int = 123
+ bat: float = 45.6
+
+They can also use any of Python's built-in constants::
+
+ yep: bool = True
+ nope: bool = False
+ nada: object = None
+
+There's also special support for a default value of ``NULL``, and
+for simple expressions, documented in the following sections.
+
+
+The ``NULL`` default value
+--------------------------
+
+For string and object parameters, you can set them to ``None`` to indicate
+that there's no default. However, that means the C variable will be
+initialized to ``Py_None``. For convenience's sakes, there's a special
+value called ``NULL`` for just this reason: from Python's perspective it
+behaves like a default value of ``None``, but the C variable is initialized
+with ``NULL``.
+
+Expressions specified as default values
+---------------------------------------
+
+The default value for a parameter can be more than just a literal value.
+It can be an entire expression, using math operators and looking up attributes
+on objects. However, this support isn't exactly simple, because of some
+non-obvious semantics.
+
+Consider the following example::
+
+ foo: Py_ssize_t = sys.maxsize - 1
+
+``sys.maxsize`` can have different values on different platforms. Therefore
+Argument Clinic can't simply evaluate that expression locally and hard-code it
+in C. So it stores the default in such a way that it will get evaluated at
+runtime, when the user asks for the function's signature.
+
+What namespace is available when the expression is evaluated? It's evaluated
+in the context of the module the builtin came from. So, if your module has an
+attribute called "``max_widgets``", you may simply use it::
+
+ foo: Py_ssize_t = max_widgets
+
+If the symbol isn't found in the current module, it fails over to looking in
+``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you
+don't know in advance what modules the user will load into their interpreter,
+it's best to restrict yourself to modules that are preloaded by Python itself.)
+
+Evaluating default values only at runtime means Argument Clinic can't compute
+the correct equivalent C default value. So you need to tell it explicitly.
+When you use an expression, you must also specify the equivalent expression
+in C, using the ``c_default`` parameter to the converter::
+
+ foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1
+
+Another complication: Argument Clinic can't know in advance whether or not the
+expression you supply is valid. It parses it to make sure it looks legal, but
+it can't *actually* know. You must be very careful when using expressions to
+specify values that are guaranteed to be valid at runtime!
+
+Finally, because expressions must be representable as static C values, there
+are many restrictions on legal expressions. Here's a list of Python features
+you're not permitted to use:
+
+* Function calls.
+* Inline if statements (``3 if foo else 5``).
+* Automatic sequence unpacking (``*[1, 2, 3]``).
+* List/set/dict comprehensions and generator expressions.
+* Tuple/list/set/dict literals.
+
+
+
+Using a return converter
+------------------------
+
+By default the impl function Argument Clinic generates for you returns ``PyObject *``.
+But your C function often computes some C type, then converts it into the ``PyObject *``
+at the last moment. Argument Clinic handles converting your inputs from Python types
+into native C types--why not have it convert your return value from a native C type
+into a Python type too?
+
+That's what a "return converter" does. It changes your impl function to return
+some C type, then adds code to the generated (non-impl) function to handle converting
+that value into the appropriate ``PyObject *``.
+
+The syntax for return converters is similar to that of parameter converters.
+You specify the return converter like it was a return annotation on the
+function itself. Return converters behave much the same as parameter converters;
+they take arguments, the arguments are all keyword-only, and if you're not changing
+any of the default arguments you can omit the parentheses.
+
+(If you use both ``"as"`` *and* a return converter for your function,
+the ``"as"`` should come before the return converter.)
+
+There's one additional complication when using return converters: how do you
+indicate an error has occured? Normally, a function returns a valid (non-``NULL``)
+pointer for success, and ``NULL`` for failure. But if you use an integer return converter,
+all integers are valid. How can Argument Clinic detect an error? Its solution: each return
+converter implicitly looks for a special value that indicates an error. If you return
+that value, and an error has been set (``PyErr_Occurred()`` returns a true
+value), then the generated code will propogate the error. Otherwise it will
+encode the value you return like normal.
+
+Currently Argument Clinic supports only a few return converters::
+
+ bool
+ int
+ unsigned int
+ long
+ unsigned int
+ size_t
+ Py_ssize_t
+ float
+ double
+ DecodeFSDefault
+
+None of these take parameters. For the first three, return -1 to indicate
+error. For ``DecodeFSDefault``, the return type is ``char *``; return a NULL
+pointer to indicate an error.
+
+(There's also an experimental ``NoneType`` converter, which lets you
+return ``Py_None`` on success or ``NULL`` on failure, without having
+to increment the reference count on ``Py_None``. I'm not sure it adds
+enough clarity to be worth using.)
+
+To see all the return converters Argument Clinic supports, along with
+their parameters (if any),
+just run ``Tools/clinic/clinic.py --converters`` for the full list.
+
+
+Cloning existing functions
+--------------------------
+
+If you have a number of functions that look similar, you may be able to
+use Clinic's "clone" feature. When you clone an existing function,
+you reuse:
+
+* its parameters, including
+
+ * their names,
+
+ * their converters, with all parameters,
+
+ * their default values,
+
+ * their per-parameter docstrings,
+
+ * their *kind* (whether they're positional only,
+ positional or keyword, or keyword only), and
+
+* its return converter.
+
+The only thing not copied from the original function is its docstring;
+the syntax allows you to specify a new docstring.
+
+Here's the syntax for cloning a function::
+
+ /*[clinic input]
+ module.class.new_function [as c_basename] = module.class.existing_function
+
+ Docstring for new_function goes here.
+ [clinic start generated code]*/
+
+(The functions can be in different modules or classes. I wrote
+``module.class`` in the sample just to illustrate that you must
+use the full path to *both* functions.)
+
+Sorry, there's no syntax for partially-cloning a function, or cloning a function
+then modifying it. Cloning is an all-or nothing proposition.
+
+Also, the function you are cloning from must have been previously defined
+in the current file.
+
+Calling Python code
+-------------------
+
+The rest of the advanced topics require you to write Python code
+which lives inside your C file and modifies Argument Clinic's
+runtime state. This is simple: you simply define a Python block.
+
+A Python block uses different delimiter lines than an Argument
+Clinic function block. It looks like this::
+
+ /*[python input]
+ # python code goes here
+ [python start generated code]*/
+
+All the code inside the Python block is executed at the
+time it's parsed. All text written to stdout inside the block
+is redirected into the "output" after the block.
+
+As an example, here's a Python block that adds a static integer
+variable to the C code::
+
+ /*[python input]
+ print('static int __ignored_unused_variable__ = 0;')
+ [python start generated code]*/
+ static int __ignored_unused_variable__ = 0;
+ /*[python checksum:...]*/
+
+
+Using a "self converter"
+------------------------
+
+Argument Clinic automatically adds a "self" parameter for you
+using a default converter. It automatically sets the ``type``
+of this parameter to the "pointer to an instance" you specified
+when you declared the type. However, you can override
+Argument Clinic's converter and specify one yourself.
+Just add your own ``self`` parameter as the first parameter in a
+block, and ensure that its converter is an instance of
+``self_converter`` or a subclass thereof.
+
+What's the point? This lets you override the type of ``self``,
+or give it a different default name.
+
+How do you specify the custom type you want to cast ``self`` to?
+If you only have one or two functions with the same type for ``self``,
+you can directly use Argument Clinic's existing ``self`` converter,
+passing in the type you want to use as the ``type`` parameter::
+
+ /*[clinic input]
+
+ _pickle.Pickler.dump
+
+ self: self(type="PicklerObject *")
+ obj: object
+ /
+
+ Write a pickled representation of the given object to the open file.
+ [clinic start generated code]*/
+
+On the other hand, if you have a lot of functions that will use the same
+type for ``self``, it's best to create your own converter, subclassing
+``self_converter`` but overwriting the ``type`` member::
+
+ /*[python input]
+ class PicklerObject_converter(self_converter):
+ type = "PicklerObject *"
+ [python start generated code]*/
+
+ /*[clinic input]
+
+ _pickle.Pickler.dump
+
+ self: PicklerObject
+ obj: object
+ /
+
+ Write a pickled representation of the given object to the open file.
+ [clinic start generated code]*/
+
+
+
+Writing a custom converter
+--------------------------
+
+As we hinted at in the previous section... you can write your own converters!
+A converter is simply a Python class that inherits from ``CConverter``.
+The main purpose of a custom converter is if you have a parameter using
+the ``O&`` format unit--parsing this parameter means calling
+a :c:func:`PyArg_ParseTuple` "converter function".
+
+Your converter class should be named ``*something*_converter``.
+If the name follows this convention, then your converter class
+will be automatically registered with Argument Clinic; its name
+will be the name of your class with the ``_converter`` suffix
+stripped off. (This is accomplished with a metaclass.)
+
+You shouldn't subclass ``CConverter.__init__``. Instead, you should
+write a ``converter_init()`` function. ``converter_init()``
+always accepts a ``self`` parameter; after that, all additional
+parameters *must* be keyword-only. Any arguments passed in to
+the converter in Argument Clinic will be passed along to your
+``converter_init()``.
+
+There are some additional members of ``CConverter`` you may wish
+to specify in your subclass. Here's the current list:
+
+``type``
+ The C type to use for this variable.
+ ``type`` should be a Python string specifying the type, e.g. ``int``.
+ If this is a pointer type, the type string should end with ``' *'``.
+
+``default``
+ The Python default value for this parameter, as a Python value.
+ Or the magic value ``unspecified`` if there is no default.
+
+``py_default``
+ ``default`` as it should appear in Python code,
+ as a string.
+ Or ``None`` if there is no default.
+
+``c_default``
+ ``default`` as it should appear in C code,
+ as a string.
+ Or ``None`` if there is no default.
+
+``c_ignored_default``
+ The default value used to initialize the C variable when
+ there is no default, but not specifying a default may
+ result in an "uninitialized variable" warning. This can
+ easily happen when using option groups--although
+ properly-written code will never actually use this value,
+ the variable does get passed in to the impl, and the
+ C compiler will complain about the "use" of the
+ uninitialized value. This value should always be a
+ non-empty string.
+
+``converter``
+ The name of the C converter function, as a string.
+
+``impl_by_reference``
+ A boolean value. If true,
+ Argument Clinic will add a ``&`` in front of the name of
+ the variable when passing it into the impl function.
+
+``parse_by_reference``
+ A boolean value. If true,
+ Argument Clinic will add a ``&`` in front of the name of
+ the variable when passing it into :c:func:`PyArg_ParseTuple`.
+
+
+Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``::
+
+ /*[python input]
+
+ class uint_converter(CConverter):
+ type = 'unsigned int'
+ converter = 'uint_converter'
+
+ [python start generated code]*/
+ /*[python end generated code: checksum=da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
+This block adds a converter to Argument Clinic named ``uint``. Parameters
+declared as ``uint`` will be declared as type ``unsigned int``, and will
+be parsed by the ``'O&'`` format unit, which will call the ``uint_converter``
+converter function.
+``uint`` variables automatically support default values.
+
+More sophisticated custom converters can insert custom C code to
+handle initialization and cleanup.
+You can see more examples of custom converters in the CPython
+source tree; grep the C files for the string ``CConverter``.
+
+Writing a custom return converter
+---------------------------------
+
+Writing a custom return converter is much like writing
+a custom converter. Except it's somewhat simpler, because return
+converters are themselves much simpler.
+
+Return converters must subclass ``CReturnConverter``.
+There are no examples yet of custom return converters,
+because they are not widely used yet. If you wish to
+write your own return converter, please read ``Tools/clinic/clinic.py``,
+specifically the implementation of ``CReturnConverter`` and
+all its subclasses.
+
+METH_O and METH_NOARGS
+----------------------------------------------
+
+To convert a function using ``METH_O``, make sure the function's
+single argument is using the ``object`` converter, and mark the
+arguments as positional-only::
+
+ /*[clinic input]
+ meth_o_sample
+
+ argument: object
+ /
+ [clinic start generated code]*/
+
+
+To convert a function using ``METH_NOARGS``, just don't specify
+any arguments.
+
+You can still use a self converter, a return converter, and specify
+a ``type`` argument to the object converter for ``METH_O``.
+
+tp_new and tp_init functions
+----------------------------------------------
+
+You can convert ``tp_new`` and ``tp_init`` functions. Just name
+them ``__new__`` or ``__init__`` as appropriate. Notes:
+
+* The function name generated for ``__new__`` doesn't end in ``__new__``
+ like it would by default. It's just the name of the class, converted
+ into a valid C identifier.
+
+* No ``PyMethodDef`` ``#define`` is generated for these functions.
+
+* ``__init__`` functions return ``int``, not ``PyObject *``.
+
+* Use the docstring as the class docstring.
+
+* Although ``__new__`` and ``__init__`` functions must always
+ accept both the ``args`` and ``kwargs`` objects, when converting
+ you may specify any signature for these functions that you like.
+ (If your function doesn't support keywords, the parsing function
+ generated will throw an exception if it receives any.)
+
+Changing and redirecting Clinic's output
+----------------------------------------
+
+It can be inconvenient to have Clinic's output interspersed with
+your conventional hand-edited C code. Luckily, Clinic is configurable:
+you can buffer up its output for printing later (or earlier!), or write
+its output to a separate file. You can also add a prefix or suffix to
+every line of Clinic's generated output.
+
+While changing Clinic's output in this manner can be a boon to readability,
+it may result in Clinic code using types before they are defined, or
+your code attempting to use Clinic-generated code befire it is defined.
+These problems can be easily solved by rearranging the declarations in your file,
+or moving where Clinic's generated code goes. (This is why the default behavior
+of Clinic is to output everything into the current block; while many people
+consider this hampers readability, it will never require rearranging your
+code to fix definition-before-use problems.)
+
+Let's start with defining some terminology:
+
+*field*
+ A field, in this context, is a subsection of Clinic's output.
+ For example, the ``#define`` for the ``PyMethodDef`` structure
+ is a field, called ``methoddef_define``. Clinic has seven
+ different fields it can output per function definition::
+
+ docstring_prototype
+ docstring_definition
+ methoddef_define
+ impl_prototype
+ parser_prototype
+ parser_definition
+ impl_definition
+
+ All the names are of the form ``"<a>_<b>"``,
+ where ``"<a>"`` is the semantic object represented (the parsing function,
+ the impl function, the docstring, or the methoddef structure) and ``"<b>"``
+ represents what kind of statement the field is. Field names that end in
+ ``"_prototype"``
+ represent forward declarations of that thing, without the actual body/data
+ of the thing; field names that end in ``"_definition"`` represent the actual
+ definition of the thing, with the body/data of the thing. (``"methoddef"``
+ is special, it's the only one that ends with ``"_define"``, representing that
+ it's a preprocessor #define.)
+
+*destination*
+ A destination is a place Clinic can write output to. There are
+ five built-in destinations:
+
+ ``block``
+ The default destination: printed in the output section of
+ the current Clinic block.
+
+ ``buffer``
+ A text buffer where you can save text for later. Text sent
+ here is appended to the end of any exsiting text. It's an
+ error to have any text left in the buffer when Clinic finishes
+ processing a file.
+
+ ``file``
+ A separate "clinic file" that will be created automatically by Clinic.
+ The filename chosen for the file is ``{basename}.clinic{extension}``,
+ where ``basename`` and ``extension`` were assigned the output
+ from ``os.path.splitext()`` run on the current file. (Example:
+ the ``file`` destination for ``_pickle.c`` would be written to
+ ``_pickle.clinic.c``.)
+
+ **Important: When using a** ``file`` **destination, you**
+ *must check in* **the generated file!**
+
+ ``two-pass``
+ A buffer like ``buffer``. However, a two-pass buffer can only
+ be written once, and it prints out all text sent to it during
+ all of processing, even from Clinic blocks *after* the
+
+ ``suppress``
+ The text is suppressed--thrown away.
+
+
+Clinic defines five new directives that let you reconfigure its output.
+
+The first new directive is ``dump``::
+
+ dump <destination>
+
+This dumps the current contents of the named destination into the output of
+the current block, and empties it. This only works with ``buffer`` and
+``two-pass`` destinations.
+
+The second new directive is ``output``. The most basic form of ``output``
+is like this::
+
+ output <field> <destination>
+
+This tells Clinic to output *field* to *destination*. ``output`` also
+supports a special meta-destination, called ``everything``, which tells
+Clinic to output *all* fields to that *destination*.
+
+``output`` has a number of other functions::
+
+ output push
+ output pop
+ output preset <preset>
+
+
+``output push`` and ``output pop`` allow you to push and pop
+configurations on an internal configuration stack, so that you
+can temporarily modify the output configuration, then easily restore
+the previous configuration. Simply push before your change to save
+the current configuration, then pop when you wish to restore the
+previous configuration.
+
+``output preset`` sets Clinic's output to one of several built-in
+preset configurations, as follows:
+
+ ``block``
+ Clinic's original starting configuration. Writes everything
+ immediately after the input block.
+
+ Suppress the ``parser_prototype``
+ and ``docstring_prototype``, write everything else to ``block``.
+
+ ``file``
+ Designed to write everything to the "clinic file" that it can.
+ You then ``#include`` this file near the top of your file.
+ You may need to rearrange your file to make this work, though
+ usually this just means creating forward declarations for various
+ ``typedef`` and ``PyTypeObject`` definitions.
+
+ Suppress the ``parser_prototype``
+ and ``docstring_prototype``, write the ``impl_definition`` to
+ ``block``, and write everything else to ``file``.
+
+ The default filename is ``"{dirname}/clinic/{basename}.h"``.
+
+ ``buffer``
+ Save up all most of the output from Clinic, to be written into
+ your file near the end. For Python files implementing modules
+ or builtin types, it's recommended that you dump the buffer
+ just above the static structures for your module or
+ builtin type; these are normally very near the end. Using
+ ``buffer`` may require even more editing than ``file``, if
+ your file has static ``PyMethodDef`` arrays defined in the
+ middle of the file.
+
+ Suppress the ``parser_prototype``, ``impl_prototype``,
+ and ``docstring_prototype``, write the ``impl_definition`` to
+ ``block``, and write everything else to ``file``.
+
+ ``two-pass``
+ Similar to the ``buffer`` preset, but writes forward declarations to
+ the ``two-pass`` buffer, and definitions to the ``buffer``.
+ This is similar to the ``buffer`` preset, but may require
+ less editing than ``buffer``. Dump the ``two-pass`` buffer
+ near the top of your file, and dump the ``buffer`` near
+ the end just like you would when using the ``buffer`` preset.
+
+ Suppresses the ``impl_prototype``, write the ``impl_definition``
+ to ``block``, write ``docstring_prototype``, ``methoddef_define``,
+ and ``parser_prototype`` to ``two-pass``, write everything else
+ to ``buffer``.
+
+ ``partial-buffer``
+ Similar to the ``buffer`` preset, but writes more things to ``block``,
+ only writing the really big chunks of generated code to ``buffer``.
+ This avoids the definition-before-use problem of ``buffer`` completely,
+ at the small cost of having slightly more stuff in the block's output.
+ Dump the ``buffer`` near the end, just like you would when using
+ the ``buffer`` preset.
+
+ Suppresses the ``impl_prototype``, write the ``docstring_definition``
+ and ``parser_defintion`` to ``buffer``, write everything else to ``block``.
+
+The third new directive is ``destination``::
+
+ destination <name> <command> [...]
+
+This performs an operation on the destination named ``name``.
+
+There are two defined subcommands: ``new`` and ``clear``.
+
+The ``new`` subcommand works like this::
+
+ destination <name> new <type>
+
+This creates a new destination with name ``<name>`` and type ``<type>``.
+
+There are five destination types:
+
+ ``suppress``
+ Throws the text away.
+
+ ``block``
+ Writes the text to the current block. This is what Clinic
+ originally did.
+
+ ``buffer``
+ A simple text buffer, like the "buffer" builtin destination above.
+
+ ``file``
+ A text file. The file destination takes an extra argument,
+ a template to use for building the filename, like so:
+
+ destination <name> new <type> <file_template>
+
+ The template can use three strings internally that will be replaced
+ by bits of the filename:
+
+ {path}
+ The full path to the file, including directory and full filename.
+ {dirname}
+ The name of the directory the file is in.
+ {basename}
+ Just the name of the file, not including the directory.
+ {basename_root}
+ Basename with the extension clipped off
+ (everything up to but not including the last '.').
+ {basename_extension}
+ The last '.' and everything after it. If the basename
+ does not contain a period, this will be the empty string.
+
+ If there are no periods in the filename, {basename} and {filename}
+ are the same, and {extension} is empty. "{basename}{extension}"
+ is always exactly the same as "{filename}"."
+
+ ``two-pass``
+ A two-pass buffer, like the "two-pass" builtin destination above.
+
+
+The ``clear`` subcommand works like this::
+
+ destination <name> clear
+
+It removes all the accumulated text up to this point in the destination.
+(I don't know what you'd need this for, but I thought maybe it'd be
+useful while someone's experimenting.)
+
+The fourth new directive is ``set``::
+
+ set line_prefix "string"
+ set line_suffix "string"
+
+``set`` lets you set two internal variables in Clinic.
+``line_prefix`` is a string that will be prepended to every line of Clinic's output;
+``line_suffix`` is a string that will be appended to every line of Clinic's output.
+
+Both of these suport two format strings:
+
+ ``{block comment start}``
+ Turns into the string ``/*``, the start-comment text sequence for C files.
+
+ ``{block comment end}``
+ Turns into the string ``*/``, the end-comment text sequence for C files.
+
+The final new directive is one you shouldn't need to use directly,
+called ``preserve``::
+
+ preserve
+
+This tells Clinic that the current contents of the output should be kept, unmodifed.
+This is used internally by Clinic when dumping output into ``file`` files; wrapping
+it in a Clinic block lets Clinic use its existing checksum functionality to ensure
+the file was not modified by hand before it gets overwritten.
+
+
+The #ifdef trick
+----------------------------------------------
+
+If you're converting a function that isn't available on all platforms,
+there's a trick you can use to make life a little easier. The existing
+code probably looks like this::
+
+ #ifdef HAVE_FUNCTIONNAME
+ static module_functionname(...)
+ {
+ ...
+ }
+ #endif /* HAVE_FUNCTIONNAME */
+
+And then in the ``PyMethodDef`` structure at the bottom the existing code
+will have::
+
+ #ifdef HAVE_FUNCTIONNAME
+ {'functionname', ... },
+ #endif /* HAVE_FUNCTIONNAME */
+
+In this scenario, you should enclose the body of your impl function inside the ``#ifdef``,
+like so::
+
+ #ifdef HAVE_FUNCTIONNAME
+ /*[clinic input]
+ module.functionname
+ ...
+ [clinic start generated code]*/
+ static module_functionname(...)
+ {
+ ...
+ }
+ #endif /* HAVE_FUNCTIONNAME */
+
+Then, remove those three lines from the ``PyMethodDef`` structure,
+replacing them with the macro Argument Clinic generated::
+
+ MODULE_FUNCTIONNAME_METHODDEF
+
+(You can find the real name for this macro inside the generated code.
+Or you can calculate it yourself: it's the name of your function as defined
+on the first line of your block, but with periods changed to underscores,
+uppercased, and ``"_METHODDEF"`` added to the end.)
+
+Perhaps you're wondering: what if ``HAVE_FUNCTIONNAME`` isn't defined?
+The ``MODULE_FUNCTIONNAME_METHODDEF`` macro won't be defined either!
+
+Here's where Argument Clinic gets very clever. It actually detects that the
+Argument Clinic block might be deactivated by the ``#ifdef``. When that
+happens, it generates a little extra code that looks like this::
+
+ #ifndef MODULE_FUNCTIONNAME_METHODDEF
+ #define MODULE_FUNCTIONNAME_METHODDEF
+ #endif /* !defined(MODULE_FUNCTIONNAME_METHODDEF) */
+
+That means the macro always works. If the function is defined, this turns
+into the correct structure, including the trailing comma. If the function is
+undefined, this turns into nothing.
+
+However, this causes one ticklish problem: where should Argument Clinic put this
+extra code when using the "block" output preset? It can't go in the output block,
+because that could be decativated by the ``#ifdef``. (That's the whole point!)
+
+In this situation, Argument Clinic writes the extra code to the "buffer" destination.
+This may mean that you get a complaint from Argument Clinic::
+
+ Warning in file "Modules/posixmodule.c" on line 12357:
+ Destination buffer 'buffer' not empty at end of file, emptying.
+
+When this happens, just open your file, find the ``dump buffer`` block that
+Argument Clinic added to your file (it'll be at the very bottom), then
+move it above the ``PyMethodDef`` structure where that macro is used.
+
+
+
+Using Argument Clinic in Python files
+-------------------------------------
+
+It's actually possible to use Argument Clinic to preprocess Python files.
+There's no point to using Argument Clinic blocks, of course, as the output
+wouldn't make any sense to the Python interpreter. But using Argument Clinic
+to run Python blocks lets you use Python as a Python preprocessor!
+
+Since Python comments are different from C comments, Argument Clinic
+blocks embedded in Python files look slightly different. They look like this::
+
+ #/*[python input]
+ #print("def foo(): pass")
+ #[python start generated code]*/
+ def foo(): pass
+ #/*[python checksum:...]*/
diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst
index 7189c65..97c90c7 100644
--- a/Doc/howto/functional.rst
+++ b/Doc/howto/functional.rst
@@ -3,7 +3,7 @@
********************************
:Author: A. M. Kuchling
-:Release: 0.31
+:Release: 0.32
In this document, we'll take a tour of Python's features suitable for
implementing programs in a functional style. After an introduction to the
@@ -15,9 +15,9 @@ concepts of functional programming, we'll look at language features such as
Introduction
============
-This section explains the basic concept of functional programming; if you're
-just interested in learning about Python language features, skip to the next
-section.
+This section explains the basic concept of functional programming; if
+you're just interested in learning about Python language features,
+skip to the next section on :ref:`functional-howto-iterators`.
Programming languages support decomposing problems in several different ways:
@@ -173,6 +173,8 @@ new programs by arranging existing functions in a new configuration and writing
a few functions specialized for the current task.
+.. _functional-howto-iterators:
+
Iterators
=========
@@ -670,7 +672,7 @@ indexes at which certain conditions are met::
:func:`sorted(iterable, key=None, reverse=False) <sorted>` collects all the
elements of the iterable into a list, sorts the list, and returns the sorted
-result. The *key*, and *reverse* arguments are passed through to the
+result. The *key* and *reverse* arguments are passed through to the
constructed list's :meth:`~list.sort` method. ::
>>> import random
@@ -836,7 +838,8 @@ Another group of functions chooses a subset of an iterator's elements based on a
predicate.
:func:`itertools.filterfalse(predicate, iter) <itertools.filterfalse>` is the
-opposite, returning all elements for which the predicate returns false::
+opposite of :func:`filter`, returning all elements for which the predicate
+returns false::
itertools.filterfalse(is_even, itertools.count()) =>
1, 3, 5, 7, 9, 11, 13, 15, ...
@@ -864,6 +867,77 @@ iterable's results. ::
itertools.dropwhile(is_even, itertools.count()) =>
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ...
+:func:`itertools.compress(data, selectors) <itertools.compress>` takes two
+iterators and returns only those elements of *data* for which the corresponding
+element of *selectors* is true, stopping whenever either one is exhausted::
+
+ itertools.compress([1,2,3,4,5], [True, True, False, False, True]) =>
+ 1, 2, 5
+
+
+Combinatoric functions
+----------------------
+
+The :func:`itertools.combinations(iterable, r) <itertools.combinations>`
+returns an iterator giving all possible *r*-tuple combinations of the
+elements contained in *iterable*. ::
+
+ itertools.combinations([1, 2, 3, 4, 5], 2) =>
+ (1, 2), (1, 3), (1, 4), (1, 5),
+ (2, 3), (2, 4), (2, 5),
+ (3, 4), (3, 5),
+ (4, 5)
+
+ itertools.combinations([1, 2, 3, 4, 5], 3) =>
+ (1, 2, 3), (1, 2, 4), (1, 2, 5), (1, 3, 4), (1, 3, 5), (1, 4, 5),
+ (2, 3, 4), (2, 3, 5), (2, 4, 5),
+ (3, 4, 5)
+
+The elements within each tuple remain in the same order as
+*iterable* returned them. For example, the number 1 is always before
+2, 3, 4, or 5 in the examples above. A similar function,
+:func:`itertools.permutations(iterable, r=None) <itertools.permutations>`,
+removes this constraint on the order, returning all possible
+arrangements of length *r*::
+
+ itertools.permutations([1, 2, 3, 4, 5], 2) =>
+ (1, 2), (1, 3), (1, 4), (1, 5),
+ (2, 1), (2, 3), (2, 4), (2, 5),
+ (3, 1), (3, 2), (3, 4), (3, 5),
+ (4, 1), (4, 2), (4, 3), (4, 5),
+ (5, 1), (5, 2), (5, 3), (5, 4)
+
+ itertools.permutations([1, 2, 3, 4, 5]) =>
+ (1, 2, 3, 4, 5), (1, 2, 3, 5, 4), (1, 2, 4, 3, 5),
+ ...
+ (5, 4, 3, 2, 1)
+
+If you don't supply a value for *r* the length of the iterable is used,
+meaning that all the elements are permuted.
+
+Note that these functions produce all of the possible combinations by
+position and don't require that the contents of *iterable* are unique::
+
+ itertools.permutations('aba', 3) =>
+ ('a', 'b', 'a'), ('a', 'a', 'b'), ('b', 'a', 'a'),
+ ('b', 'a', 'a'), ('a', 'a', 'b'), ('a', 'b', 'a')
+
+The identical tuple ``('a', 'a', 'b')`` occurs twice, but the two 'a'
+strings came from different positions.
+
+The :func:`itertools.combinations_with_replacement(iterable, r) <itertools.combinations_with_replacement>`
+function relaxes a different constraint: elements can be repeated
+within a single tuple. Conceptually an element is selected for the
+first position of each tuple and then is replaced before the second
+element is selected. ::
+
+ itertools.combinations_with_replacement([1, 2, 3, 4, 5], 2) =>
+ (1, 1), (1, 2), (1, 3), (1, 4), (1, 5),
+ (2, 2), (2, 3), (2, 4), (2, 5),
+ (3, 3), (3, 4), (3, 5),
+ (4, 4), (4, 5),
+ (5, 5)
+
Grouping elements
-----------------
@@ -986,6 +1060,17 @@ write the obvious :keyword:`for` loop::
for i in [1,2,3]:
product *= i
+A related function is `itertools.accumulate(iterable, func=operator.add) <itertools.accumulate`.
+It performs the same calculation, but instead of returning only the
+final result, :func:`accumulate` returns an iterator that also yields
+each partial result::
+
+ itertools.accumulate([1,2,3,4,5]) =>
+ 1, 3, 6, 10, 15
+
+ itertools.accumulate([1,2,3,4,5], operator.mul) =>
+ 1, 2, 6, 24, 120
+
The operator module
-------------------
@@ -1159,51 +1244,6 @@ features in Python 2.5.
.. comment
- Topics to place
- -----------------------------
-
- XXX os.walk()
-
- XXX Need a large example.
-
- But will an example add much? I'll post a first draft and see
- what the comments say.
-
-.. comment
-
- Original outline:
- Introduction
- Idea of FP
- Programs built out of functions
- Functions are strictly input-output, no internal state
- Opposed to OO programming, where objects have state
-
- Why FP?
- Formal provability
- Assignment is difficult to reason about
- Not very relevant to Python
- Modularity
- Small functions that do one thing
- Debuggability:
- Easy to test due to lack of state
- Easy to verify output from intermediate steps
- Composability
- You assemble a toolbox of functions that can be mixed
-
- Tackling a problem
- Need a significant example
-
- Iterators
- Generators
- The itertools module
- List comprehensions
- Small functions and the lambda statement
- Built-in functions
- map
- filter
-
-.. comment
-
Handy little function for printing part of an iterator -- used
while writing this document.
@@ -1214,5 +1254,3 @@ features in Python 2.5.
sys.stdout.write(str(elem))
sys.stdout.write(', ')
print(elem[-1])
-
-
diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst
index 81a4f8b..2c9d699 100644
--- a/Doc/howto/index.rst
+++ b/Doc/howto/index.rst
@@ -28,4 +28,5 @@ Currently, the HOWTOs are:
webservers.rst
argparse.rst
ipaddress.rst
+ clinic.rst
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
index 563da9d..82c8f51 100644
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -416,7 +416,7 @@ module. Here is a basic working example::
Simple TCP socket-based logging receiver suitable for testing.
"""
- allow_reuse_address = 1
+ allow_reuse_address = True
def __init__(self, host='localhost',
port=logging.handlers.DEFAULT_TCP_LOGGING_PORT,
@@ -704,9 +704,7 @@ the basis for code meeting your own specific requirements::
break
logger = logging.getLogger(record.name)
logger.handle(record) # No level or filter logic applied - just do it!
- except (KeyboardInterrupt, SystemExit):
- raise
- except:
+ except Exception:
import sys, traceback
print('Whoops! Problem:', file=sys.stderr)
traceback.print_exc(file=sys.stderr)
@@ -1310,7 +1308,7 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration
}
For more information about this configuration, you can see the `relevant
-section <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_
+section <https://docs.djangoproject.com/en/1.6/topics/logging/#configuring-logging>`_
of the Django documentation.
.. _cookbook-rotator-namer:
diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst
index 55b1c86..dab58f4 100644
--- a/Doc/howto/logging.rst
+++ b/Doc/howto/logging.rst
@@ -901,10 +901,10 @@ provided:
disk files, rotating the log file at certain timed intervals.
#. :class:`~handlers.SocketHandler` instances send messages to TCP/IP
- sockets.
+ sockets. Since 3.4, Unix domain sockets are also supported.
#. :class:`~handlers.DatagramHandler` instances send messages to UDP
- sockets.
+ sockets. Since 3.4, Unix domain sockets are also supported.
#. :class:`~handlers.SMTPHandler` instances send messages to a designated
email address.
diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst
index 5203e53..fbe763b 100644
--- a/Doc/howto/regex.rst
+++ b/Doc/howto/regex.rst
@@ -271,8 +271,8 @@ performing string substitutions. ::
>>> import re
>>> p = re.compile('ab*')
- >>> p #doctest: +ELLIPSIS
- <_sre.SRE_Pattern object at 0x...>
+ >>> p
+ re.compile('ab*')
:func:`re.compile` also accepts an optional *flags* argument, used to enable
various special features and syntax variations. We'll go over the available
@@ -383,8 +383,8 @@ Python interpreter, import the :mod:`re` module, and compile a RE::
>>> import re
>>> p = re.compile('[a-z]+')
- >>> p #doctest: +ELLIPSIS
- <_sre.SRE_Pattern object at 0x...>
+ >>> p
+ re.compile('[a-z]+')
Now, you can try matching various strings against the RE ``[a-z]+``. An empty
string shouldn't match at all, since ``+`` means 'one or more repetitions'.
@@ -402,7 +402,7 @@ should store the result in a variable for later use. ::
>>> m = p.match('tempo')
>>> m #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(0, 5), match='tempo'>
Now you can query the :ref:`match object <match-objects>` for information
about the matching string. :ref:`match object <match-objects>` instances
@@ -441,7 +441,7 @@ case. ::
>>> print(p.match('::: message'))
None
>>> m = p.search('::: message'); print(m) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(4, 11), match='message'>
>>> m.group()
'message'
>>> m.span()
@@ -493,7 +493,7 @@ the RE string added as the first argument, and still return either ``None`` or a
>>> print(re.match(r'From\s+', 'Fromage amk'))
None
>>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(0, 5), match='From '>
Under the hood, these functions simply create a pattern object for you
and call the appropriate method on it. They also store the compiled
@@ -685,7 +685,7 @@ given location, they can obviously be matched an infinite number of times.
line, the RE to use is ``^From``. ::
>>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(0, 4), match='From'>
>>> print(re.search('^From', 'Reciting From Memory'))
None
@@ -697,11 +697,11 @@ given location, they can obviously be matched an infinite number of times.
or any location followed by a newline character. ::
>>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(6, 7), match='}'>
>>> print(re.search('}$', '{block} '))
None
>>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(6, 7), match='}'>
To match a literal ``'$'``, use ``\$`` or enclose it inside a character class,
as in ``[$]``.
@@ -726,7 +726,7 @@ given location, they can obviously be matched an infinite number of times.
>>> p = re.compile(r'\bclass\b')
>>> print(p.search('no class at all')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(3, 8), match='class'>
>>> print(p.search('the declassified algorithm'))
None
>>> print(p.search('one subclass is'))
@@ -744,7 +744,7 @@ given location, they can obviously be matched an infinite number of times.
>>> print(p.search('no class at all'))
None
>>> print(p.search('\b' + 'class' + '\b')) #doctest: +ELLIPSIS
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(0, 7), match='\x08class\x08'>
Second, inside a character class, where there's no use for this assertion,
``\b`` represents the backspace character, for compatibility with Python's
diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst
index 5a22660..3d5de32 100644
--- a/Doc/howto/urllib2.rst
+++ b/Doc/howto/urllib2.rst
@@ -507,7 +507,7 @@ than the URL you pass to .add_password() will also match. ::
-- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy`
environment variable is set), ``UnknownHandler``, ``HTTPHandler``,
``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``,
- ``FileHandler``, ``HTTPErrorProcessor``.
+ ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``.
``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme
component and the hostname and optionally the port number)
diff --git a/Doc/includes/email-alternative-new-api.py b/Doc/includes/email-alternative-new-api.py
new file mode 100644
index 0000000..c1255a6
--- /dev/null
+++ b/Doc/includes/email-alternative-new-api.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import smtplib
+
+from email.message import EmailMessage
+from email.headerregistry import Address
+from email.utils import make_msgid
+
+# Create the base text message.
+msg = EmailMessage()
+msg['Subject'] = "Ayons asperges pour le déjeuner"
+msg['From'] = Address("Pepé Le Pew", "pepe@example.com")
+msg['To'] = (Address("Penelope Pussycat", "penelope@example.com"),
+ Address("Fabrette Pussycat", "fabrette@example.com"))
+msg.set_content("""\
+Salut!
+
+Cela ressemble à un excellent recipie[1] déjeuner.
+
+[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718
+
+--Pepé
+""")
+
+# Add the html version. This converts the message into a multipart/alternative
+# container, with the original text message as the first part and the new html
+# message as the second part.
+asparagus_cid = make_msgid()
+msg.add_alternative("""\
+<html>
+ <head></head>
+ <body>
+ <p>Salut!<\p>
+ <p>Cela ressemble à un excellent
+ <a href="http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718>
+ recipie
+ </a> déjeuner.
+ </p>
+ <img src="cid:{asparagus_cid}" \>
+ </body>
+</html>
+""".format(asparagus_cid=asparagus_cid[1:-1]), subtype='html')
+# note that we needed to peel the <> off the msgid for use in the html.
+
+# Now add the related image to the html part.
+with open("roasted-asparagus.jpg", 'rb') as img:
+ msg.get_payload()[1].add_related(img.read(), 'image', 'jpeg',
+ cid=asparagus_cid)
+
+# Make a local copy of what we are going to send.
+with open('outgoing.msg', 'wb') as f:
+ f.write(bytes(msg))
+
+# Send the message via local SMTP server.
+with smtplib.SMTP('localhost') as s:
+ s.send_message(msg)
diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py
index cc5529e..3c7c770 100644
--- a/Doc/includes/email-dir.py
+++ b/Doc/includes/email-dir.py
@@ -8,7 +8,7 @@ import smtplib
# For guessing MIME type based on file name extension
import mimetypes
-from optparse import OptionParser
+from argparse import ArgumentParser
from email import encoders
from email.message import Message
@@ -22,44 +22,36 @@ COMMASPACE = ', '
def main():
- parser = OptionParser(usage="""\
+ parser = ArgumentParser(description="""\
Send the contents of a directory as a MIME message.
-
-Usage: %prog [options]
-
Unless the -o option is given, the email is sent by forwarding to your local
SMTP server, which then does the normal delivery process. Your local machine
must be running an SMTP server.
""")
- parser.add_option('-d', '--directory',
- type='string', action='store',
- help="""Mail the contents of the specified directory,
- otherwise use the current directory. Only the regular
- files in the directory are sent, and we don't recurse to
- subdirectories.""")
- parser.add_option('-o', '--output',
- type='string', action='store', metavar='FILE',
- help="""Print the composed message to FILE instead of
- sending the message to the SMTP server.""")
- parser.add_option('-s', '--sender',
- type='string', action='store', metavar='SENDER',
- help='The value of the From: header (required)')
- parser.add_option('-r', '--recipient',
- type='string', action='append', metavar='RECIPIENT',
- default=[], dest='recipients',
- help='A To: header value (at least one required)')
- opts, args = parser.parse_args()
- if not opts.sender or not opts.recipients:
- parser.print_help()
- sys.exit(1)
- directory = opts.directory
+ parser.add_argument('-d', '--directory',
+ help="""Mail the contents of the specified directory,
+ otherwise use the current directory. Only the regular
+ files in the directory are sent, and we don't recurse to
+ subdirectories.""")
+ parser.add_argument('-o', '--output',
+ metavar='FILE',
+ help="""Print the composed message to FILE instead of
+ sending the message to the SMTP server.""")
+ parser.add_argument('-s', '--sender', required=True,
+ help='The value of the From: header (required)')
+ parser.add_argument('-r', '--recipient', required=True,
+ action='append', metavar='RECIPIENT',
+ default=[], dest='recipients',
+ help='A To: header value (at least one required)')
+ args = parser.parse_args()
+ directory = args.directory
if not directory:
directory = '.'
# Create the enclosing (outer) message
outer = MIMEMultipart()
outer['Subject'] = 'Contents of directory %s' % os.path.abspath(directory)
- outer['To'] = COMMASPACE.join(opts.recipients)
- outer['From'] = opts.sender
+ outer['To'] = COMMASPACE.join(args.recipients)
+ outer['From'] = args.sender
outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
for filename in os.listdir(directory):
@@ -76,23 +68,19 @@ must be running an SMTP server.
ctype = 'application/octet-stream'
maintype, subtype = ctype.split('/', 1)
if maintype == 'text':
- fp = open(path)
- # Note: we should handle calculating the charset
- msg = MIMEText(fp.read(), _subtype=subtype)
- fp.close()
+ with open(path) as fp:
+ # Note: we should handle calculating the charset
+ msg = MIMEText(fp.read(), _subtype=subtype)
elif maintype == 'image':
- fp = open(path, 'rb')
- msg = MIMEImage(fp.read(), _subtype=subtype)
- fp.close()
+ with open(path, 'rb') as fp:
+ msg = MIMEImage(fp.read(), _subtype=subtype)
elif maintype == 'audio':
- fp = open(path, 'rb')
- msg = MIMEAudio(fp.read(), _subtype=subtype)
- fp.close()
+ with open(path, 'rb') as fp:
+ msg = MIMEAudio(fp.read(), _subtype=subtype)
else:
- fp = open(path, 'rb')
- msg = MIMEBase(maintype, subtype)
- msg.set_payload(fp.read())
- fp.close()
+ with open(path, 'rb') as fp:
+ msg = MIMEBase(maintype, subtype)
+ msg.set_payload(fp.read())
# Encode the payload using Base64
encoders.encode_base64(msg)
# Set the filename parameter
@@ -100,14 +88,12 @@ must be running an SMTP server.
outer.attach(msg)
# Now send or store the message
composed = outer.as_string()
- if opts.output:
- fp = open(opts.output, 'w')
- fp.write(composed)
- fp.close()
+ if args.output:
+ with open(args.output, 'w') as fp:
+ fp.write(composed)
else:
- s = smtplib.SMTP('localhost')
- s.sendmail(opts.sender, opts.recipients, composed)
- s.quit()
+ with smtplib.SMTP('localhost') as s:
+ s.sendmail(args.sender, args.recipients, composed)
if __name__ == '__main__':
diff --git a/Doc/includes/email-read-alternative-new-api.py b/Doc/includes/email-read-alternative-new-api.py
new file mode 100644
index 0000000..8ab4e9f
--- /dev/null
+++ b/Doc/includes/email-read-alternative-new-api.py
@@ -0,0 +1,74 @@
+import os
+import sys
+import tempfile
+import mimetypes
+import webbrowser
+
+# Import the email modules we'll need
+from email import policy
+from email.parser import BytesParser
+
+# An imaginary module that would make this work and be safe.
+from imaginary import magic_html_parser
+
+# In a real program you'd get the filename from the arguments.
+msg = BytesParser(policy=policy.default).parse(open('outgoing.msg', 'rb'))
+
+# Now the header items can be accessed as a dictionary, and any non-ASCII will
+# be converted to unicode:
+print('To:', msg['to'])
+print('From:', msg['from'])
+print('Subject:', msg['subject'])
+
+# If we want to print a priview of the message content, we can extract whatever
+# the least formatted payload is and print the first three lines. Of course,
+# if the message has no plain text part printing the first three lines of html
+# is probably useless, but this is just a conceptual example.
+simplest = msg.get_body(preferencelist=('plain', 'html'))
+print()
+print(''.join(simplest.get_content().splitlines(keepends=True)[:3]))
+
+ans = input("View full message?")
+if ans.lower()[0] == 'n':
+ sys.exit()
+
+# We can extract the richest alternative in order to display it:
+richest = msg.get_body()
+partfiles = {}
+if richest['content-type'].maintype == 'text':
+ if richest['content-type'].subtype == 'plain':
+ for line in richest.get_content().splitlines():
+ print(line)
+ sys.exit()
+ elif richest['content-type'].subtype == 'html':
+ body = richest
+ else:
+ print("Don't know how to display {}".format(richest.get_content_type()))
+ sys.exit()
+elif richest['content-type'].content_type == 'multipart/related':
+ body = richest.get_body(preferencelist=('html'))
+ for part in richest.iter_attachments():
+ fn = part.get_filename()
+ if fn:
+ extension = os.path.splitext(part.get_filename())[1]
+ else:
+ extension = mimetypes.guess_extension(part.get_content_type())
+ with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as f:
+ f.write(part.get_content())
+ # again strip the <> to go from email form of cid to html form.
+ partfiles[part['content-id'][1:-1]] = f.name
+else:
+ print("Don't know how to display {}".format(richest.get_content_type()))
+ sys.exit()
+with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
+ # The magic_html_parser has to rewrite the href="cid:...." attributes to
+ # point to the filenames in partfiles. It also has to do a safety-sanitize
+ # of the html. It could be written using html.parser.
+ f.write(magic_html_parser(body.get_content(), partfiles))
+webbrowser.open(f.name)
+os.remove(f.name)
+for fn in partfiles.values():
+ os.remove(fn)
+
+# Of course, there are lots of email messages that could break this simple
+# minded program, but it will handle the most common ones.
diff --git a/Doc/includes/email-unpack.py b/Doc/includes/email-unpack.py
index 3653543..574a0b6 100644
--- a/Doc/includes/email-unpack.py
+++ b/Doc/includes/email-unpack.py
@@ -8,41 +8,27 @@ import email
import errno
import mimetypes
-from optparse import OptionParser
+from argparse import ArgumentParser
def main():
- parser = OptionParser(usage="""\
+ parser = ArgumentParser(description="""\
Unpack a MIME message into a directory of files.
-
-Usage: %prog [options] msgfile
""")
- parser.add_option('-d', '--directory',
- type='string', action='store',
- help="""Unpack the MIME message into the named
- directory, which will be created if it doesn't already
- exist.""")
- opts, args = parser.parse_args()
- if not opts.directory:
- parser.print_help()
- sys.exit(1)
+ parser.add_argument('-d', '--directory', required=True,
+ help="""Unpack the MIME message into the named
+ directory, which will be created if it doesn't already
+ exist.""")
+ parser.add_argument('msgfile')
+ args = parser.parse_args()
- try:
- msgfile = args[0]
- except IndexError:
- parser.print_help()
- sys.exit(1)
+ with open(args.msgfile) as fp:
+ msg = email.message_from_file(fp)
try:
- os.mkdir(opts.directory)
- except OSError as e:
- # Ignore directory exists error
- if e.errno != errno.EEXIST:
- raise
-
- fp = open(msgfile)
- msg = email.message_from_file(fp)
- fp.close()
+ os.mkdir(args.directory)
+ except FileExistsError:
+ pass
counter = 1
for part in msg.walk():
@@ -59,9 +45,8 @@ Usage: %prog [options] msgfile
ext = '.bin'
filename = 'part-%03d%s' % (counter, ext)
counter += 1
- fp = open(os.path.join(opts.directory, filename), 'wb')
- fp.write(part.get_payload(decode=True))
- fp.close()
+ with open(os.path.join(args.directory, filename), 'wb') as fp:
+ fp.write(part.get_payload(decode=True))
if __name__ == '__main__':
diff --git a/Doc/includes/mp_benchmarks.py b/Doc/includes/mp_benchmarks.py
deleted file mode 100644
index 3763ea9..0000000
--- a/Doc/includes/mp_benchmarks.py
+++ /dev/null
@@ -1,239 +0,0 @@
-#
-# Simple benchmarks for the multiprocessing package
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
-import time
-import multiprocessing
-import threading
-import queue
-import gc
-
-_timer = time.perf_counter
-
-delta = 1
-
-
-#### TEST_QUEUESPEED
-
-def queuespeed_func(q, c, iterations):
- a = '0' * 256
- c.acquire()
- c.notify()
- c.release()
-
- for i in range(iterations):
- q.put(a)
-
- q.put('STOP')
-
-def test_queuespeed(Process, q, c):
- elapsed = 0
- iterations = 1
-
- while elapsed < delta:
- iterations *= 2
-
- p = Process(target=queuespeed_func, args=(q, c, iterations))
- c.acquire()
- p.start()
- c.wait()
- c.release()
-
- result = None
- t = _timer()
-
- while result != 'STOP':
- result = q.get()
-
- elapsed = _timer() - t
-
- p.join()
-
- print(iterations, 'objects passed through the queue in', elapsed, 'seconds')
- print('average number/sec:', iterations/elapsed)
-
-
-#### TEST_PIPESPEED
-
-def pipe_func(c, cond, iterations):
- a = '0' * 256
- cond.acquire()
- cond.notify()
- cond.release()
-
- for i in range(iterations):
- c.send(a)
-
- c.send('STOP')
-
-def test_pipespeed():
- c, d = multiprocessing.Pipe()
- cond = multiprocessing.Condition()
- elapsed = 0
- iterations = 1
-
- while elapsed < delta:
- iterations *= 2
-
- p = multiprocessing.Process(target=pipe_func,
- args=(d, cond, iterations))
- cond.acquire()
- p.start()
- cond.wait()
- cond.release()
-
- result = None
- t = _timer()
-
- while result != 'STOP':
- result = c.recv()
-
- elapsed = _timer() - t
- p.join()
-
- print(iterations, 'objects passed through connection in',elapsed,'seconds')
- print('average number/sec:', iterations/elapsed)
-
-
-#### TEST_SEQSPEED
-
-def test_seqspeed(seq):
- elapsed = 0
- iterations = 1
-
- while elapsed < delta:
- iterations *= 2
-
- t = _timer()
-
- for i in range(iterations):
- a = seq[5]
-
- elapsed = _timer() - t
-
- print(iterations, 'iterations in', elapsed, 'seconds')
- print('average number/sec:', iterations/elapsed)
-
-
-#### TEST_LOCK
-
-def test_lockspeed(l):
- elapsed = 0
- iterations = 1
-
- while elapsed < delta:
- iterations *= 2
-
- t = _timer()
-
- for i in range(iterations):
- l.acquire()
- l.release()
-
- elapsed = _timer() - t
-
- print(iterations, 'iterations in', elapsed, 'seconds')
- print('average number/sec:', iterations/elapsed)
-
-
-#### TEST_CONDITION
-
-def conditionspeed_func(c, N):
- c.acquire()
- c.notify()
-
- for i in range(N):
- c.wait()
- c.notify()
-
- c.release()
-
-def test_conditionspeed(Process, c):
- elapsed = 0
- iterations = 1
-
- while elapsed < delta:
- iterations *= 2
-
- c.acquire()
- p = Process(target=conditionspeed_func, args=(c, iterations))
- p.start()
-
- c.wait()
-
- t = _timer()
-
- for i in range(iterations):
- c.notify()
- c.wait()
-
- elapsed = _timer() - t
-
- c.release()
- p.join()
-
- print(iterations * 2, 'waits in', elapsed, 'seconds')
- print('average number/sec:', iterations * 2 / elapsed)
-
-####
-
-def test():
- manager = multiprocessing.Manager()
-
- gc.disable()
-
- print('\n\t######## testing Queue.Queue\n')
- test_queuespeed(threading.Thread, queue.Queue(),
- threading.Condition())
- print('\n\t######## testing multiprocessing.Queue\n')
- test_queuespeed(multiprocessing.Process, multiprocessing.Queue(),
- multiprocessing.Condition())
- print('\n\t######## testing Queue managed by server process\n')
- test_queuespeed(multiprocessing.Process, manager.Queue(),
- manager.Condition())
- print('\n\t######## testing multiprocessing.Pipe\n')
- test_pipespeed()
-
- print()
-
- print('\n\t######## testing list\n')
- test_seqspeed(list(range(10)))
- print('\n\t######## testing list managed by server process\n')
- test_seqspeed(manager.list(list(range(10))))
- print('\n\t######## testing Array("i", ..., lock=False)\n')
- test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=False))
- print('\n\t######## testing Array("i", ..., lock=True)\n')
- test_seqspeed(multiprocessing.Array('i', list(range(10)), lock=True))
-
- print()
-
- print('\n\t######## testing threading.Lock\n')
- test_lockspeed(threading.Lock())
- print('\n\t######## testing threading.RLock\n')
- test_lockspeed(threading.RLock())
- print('\n\t######## testing multiprocessing.Lock\n')
- test_lockspeed(multiprocessing.Lock())
- print('\n\t######## testing multiprocessing.RLock\n')
- test_lockspeed(multiprocessing.RLock())
- print('\n\t######## testing lock managed by server process\n')
- test_lockspeed(manager.Lock())
- print('\n\t######## testing rlock managed by server process\n')
- test_lockspeed(manager.RLock())
-
- print()
-
- print('\n\t######## testing threading.Condition\n')
- test_conditionspeed(threading.Thread, threading.Condition())
- print('\n\t######## testing multiprocessing.Condition\n')
- test_conditionspeed(multiprocessing.Process, multiprocessing.Condition())
- print('\n\t######## testing condition managed by a server process\n')
- test_conditionspeed(multiprocessing.Process, manager.Condition())
-
- gc.enable()
-
-if __name__ == '__main__':
- multiprocessing.freeze_support()
- test()
diff --git a/Doc/includes/mp_newtype.py b/Doc/includes/mp_newtype.py
index 7291743..1c796a5 100644
--- a/Doc/includes/mp_newtype.py
+++ b/Doc/includes/mp_newtype.py
@@ -1,11 +1,3 @@
-#
-# This module shows how to use arbitrary callables with a subclass of
-# `BaseManager`.
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
from multiprocessing import freeze_support
from multiprocessing.managers import BaseManager, BaseProxy
import operator
@@ -27,12 +19,10 @@ def baz():
# Proxy type for generator objects
class GeneratorProxy(BaseProxy):
- _exposed_ = ('next', '__next__')
+ _exposed_ = ['__next__']
def __iter__(self):
return self
def __next__(self):
- return self._callmethod('next')
- def __next__(self):
return self._callmethod('__next__')
# Function to return the operator module
@@ -90,8 +80,6 @@ def test():
op = manager.operator()
print('op.add(23, 45) =', op.add(23, 45))
print('op.pow(2, 94) =', op.pow(2, 94))
- print('op.getslice(range(10), 2, 6) =', op.getslice(list(range(10)), 2, 6))
- print('op.repeat(range(5), 3) =', op.repeat(list(range(5)), 3))
print('op._exposed_ =', op._exposed_)
##
diff --git a/Doc/includes/mp_pool.py b/Doc/includes/mp_pool.py
index 1578498..11101e1 100644
--- a/Doc/includes/mp_pool.py
+++ b/Doc/includes/mp_pool.py
@@ -1,10 +1,3 @@
-#
-# A test of `multiprocessing.Pool` class
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
import multiprocessing
import time
import random
@@ -46,269 +39,115 @@ def noop(x):
#
def test():
- print('cpu_count() = %d\n' % multiprocessing.cpu_count())
-
- #
- # Create pool
- #
-
PROCESSES = 4
print('Creating pool with %d processes\n' % PROCESSES)
- pool = multiprocessing.Pool(PROCESSES)
- print('pool = %s' % pool)
- print()
-
- #
- # Tests
- #
-
- TASKS = [(mul, (i, 7)) for i in range(10)] + \
- [(plus, (i, 8)) for i in range(10)]
-
- results = [pool.apply_async(calculate, t) for t in TASKS]
- imap_it = pool.imap(calculatestar, TASKS)
- imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)
-
- print('Ordered results using pool.apply_async():')
- for r in results:
- print('\t', r.get())
- print()
-
- print('Ordered results using pool.imap():')
- for x in imap_it:
- print('\t', x)
- print()
-
- print('Unordered results using pool.imap_unordered():')
- for x in imap_unordered_it:
- print('\t', x)
- print()
-
- print('Ordered results using pool.map() --- will block till complete:')
- for x in pool.map(calculatestar, TASKS):
- print('\t', x)
- print()
-
- #
- # Simple benchmarks
- #
-
- N = 100000
- print('def pow3(x): return x**3')
- t = time.time()
- A = list(map(pow3, range(N)))
- print('\tmap(pow3, range(%d)):\n\t\t%s seconds' % \
- (N, time.time() - t))
+ with multiprocessing.Pool(PROCESSES) as pool:
+ #
+ # Tests
+ #
- t = time.time()
- B = pool.map(pow3, range(N))
- print('\tpool.map(pow3, range(%d)):\n\t\t%s seconds' % \
- (N, time.time() - t))
+ TASKS = [(mul, (i, 7)) for i in range(10)] + \
+ [(plus, (i, 8)) for i in range(10)]
- t = time.time()
- C = list(pool.imap(pow3, range(N), chunksize=N//8))
- print('\tlist(pool.imap(pow3, range(%d), chunksize=%d)):\n\t\t%s' \
- ' seconds' % (N, N//8, time.time() - t))
+ results = [pool.apply_async(calculate, t) for t in TASKS]
+ imap_it = pool.imap(calculatestar, TASKS)
+ imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)
- assert A == B == C, (len(A), len(B), len(C))
- print()
+ print('Ordered results using pool.apply_async():')
+ for r in results:
+ print('\t', r.get())
+ print()
- L = [None] * 1000000
- print('def noop(x): pass')
- print('L = [None] * 1000000')
+ print('Ordered results using pool.imap():')
+ for x in imap_it:
+ print('\t', x)
+ print()
- t = time.time()
- A = list(map(noop, L))
- print('\tmap(noop, L):\n\t\t%s seconds' % \
- (time.time() - t))
+ print('Unordered results using pool.imap_unordered():')
+ for x in imap_unordered_it:
+ print('\t', x)
+ print()
- t = time.time()
- B = pool.map(noop, L)
- print('\tpool.map(noop, L):\n\t\t%s seconds' % \
- (time.time() - t))
+ print('Ordered results using pool.map() --- will block till complete:')
+ for x in pool.map(calculatestar, TASKS):
+ print('\t', x)
+ print()
- t = time.time()
- C = list(pool.imap(noop, L, chunksize=len(L)//8))
- print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
- (len(L)//8, time.time() - t))
+ #
+ # Test error handling
+ #
- assert A == B == C, (len(A), len(B), len(C))
- print()
+ print('Testing error handling:')
- del A, B, C, L
-
- #
- # Test error handling
- #
-
- print('Testing error handling:')
-
- try:
- print(pool.apply(f, (5,)))
- except ZeroDivisionError:
- print('\tGot ZeroDivisionError as expected from pool.apply()')
- else:
- raise AssertionError('expected ZeroDivisionError')
-
- try:
- print(pool.map(f, list(range(10))))
- except ZeroDivisionError:
- print('\tGot ZeroDivisionError as expected from pool.map()')
- else:
- raise AssertionError('expected ZeroDivisionError')
-
- try:
- print(list(pool.imap(f, list(range(10)))))
- except ZeroDivisionError:
- print('\tGot ZeroDivisionError as expected from list(pool.imap())')
- else:
- raise AssertionError('expected ZeroDivisionError')
-
- it = pool.imap(f, list(range(10)))
- for i in range(10):
try:
- x = next(it)
+ print(pool.apply(f, (5,)))
except ZeroDivisionError:
- if i == 5:
- pass
- except StopIteration:
- break
+ print('\tGot ZeroDivisionError as expected from pool.apply()')
else:
- if i == 5:
- raise AssertionError('expected ZeroDivisionError')
-
- assert i == 9
- print('\tGot ZeroDivisionError as expected from IMapIterator.next()')
- print()
+ raise AssertionError('expected ZeroDivisionError')
- #
- # Testing timeouts
- #
-
- print('Testing ApplyResult.get() with timeout:', end=' ')
- res = pool.apply_async(calculate, TASKS[0])
- while 1:
- sys.stdout.flush()
try:
- sys.stdout.write('\n\t%s' % res.get(0.02))
- break
- except multiprocessing.TimeoutError:
- sys.stdout.write('.')
- print()
- print()
+ print(pool.map(f, list(range(10))))
+ except ZeroDivisionError:
+ print('\tGot ZeroDivisionError as expected from pool.map()')
+ else:
+ raise AssertionError('expected ZeroDivisionError')
- print('Testing IMapIterator.next() with timeout:', end=' ')
- it = pool.imap(calculatestar, TASKS)
- while 1:
- sys.stdout.flush()
try:
- sys.stdout.write('\n\t%s' % it.next(0.02))
- except StopIteration:
- break
- except multiprocessing.TimeoutError:
- sys.stdout.write('.')
- print()
- print()
-
- #
- # Testing callback
- #
-
- print('Testing callback:')
-
- A = []
- B = [56, 0, 1, 8, 27, 64, 125, 216, 343, 512, 729]
-
- r = pool.apply_async(mul, (7, 8), callback=A.append)
- r.wait()
-
- r = pool.map_async(pow3, list(range(10)), callback=A.extend)
- r.wait()
-
- if A == B:
- print('\tcallbacks succeeded\n')
- else:
- print('\t*** callbacks failed\n\t\t%s != %s\n' % (A, B))
-
- #
- # Check there are no outstanding tasks
- #
-
- assert not pool._cache, 'cache = %r' % pool._cache
-
- #
- # Check close() methods
- #
-
- print('Testing close():')
-
- for worker in pool._pool:
- assert worker.is_alive()
-
- result = pool.apply_async(time.sleep, [0.5])
- pool.close()
- pool.join()
-
- assert result.get() is None
-
- for worker in pool._pool:
- assert not worker.is_alive()
-
- print('\tclose() succeeded\n')
-
- #
- # Check terminate() method
- #
-
- print('Testing terminate():')
-
- pool = multiprocessing.Pool(2)
- DELTA = 0.1
- ignore = pool.apply(pow3, [2])
- results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)]
- pool.terminate()
- pool.join()
-
- for worker in pool._pool:
- assert not worker.is_alive()
-
- print('\tterminate() succeeded\n')
-
- #
- # Check garbage collection
- #
-
- print('Testing garbage collection:')
-
- pool = multiprocessing.Pool(2)
- DELTA = 0.1
- processes = pool._pool
- ignore = pool.apply(pow3, [2])
- results = [pool.apply_async(time.sleep, [DELTA]) for i in range(100)]
-
- results = pool = None
-
- time.sleep(DELTA * 2)
-
- for worker in processes:
- assert not worker.is_alive()
-
- print('\tgarbage collection succeeded\n')
+ print(list(pool.imap(f, list(range(10)))))
+ except ZeroDivisionError:
+ print('\tGot ZeroDivisionError as expected from list(pool.imap())')
+ else:
+ raise AssertionError('expected ZeroDivisionError')
+
+ it = pool.imap(f, list(range(10)))
+ for i in range(10):
+ try:
+ x = next(it)
+ except ZeroDivisionError:
+ if i == 5:
+ pass
+ except StopIteration:
+ break
+ else:
+ if i == 5:
+ raise AssertionError('expected ZeroDivisionError')
+
+ assert i == 9
+ print('\tGot ZeroDivisionError as expected from IMapIterator.next()')
+ print()
+
+ #
+ # Testing timeouts
+ #
+
+ print('Testing ApplyResult.get() with timeout:', end=' ')
+ res = pool.apply_async(calculate, TASKS[0])
+ while 1:
+ sys.stdout.flush()
+ try:
+ sys.stdout.write('\n\t%s' % res.get(0.02))
+ break
+ except multiprocessing.TimeoutError:
+ sys.stdout.write('.')
+ print()
+ print()
+
+ print('Testing IMapIterator.next() with timeout:', end=' ')
+ it = pool.imap(calculatestar, TASKS)
+ while 1:
+ sys.stdout.flush()
+ try:
+ sys.stdout.write('\n\t%s' % it.next(0.02))
+ except StopIteration:
+ break
+ except multiprocessing.TimeoutError:
+ sys.stdout.write('.')
+ print()
+ print()
if __name__ == '__main__':
multiprocessing.freeze_support()
-
- assert len(sys.argv) in (1, 2)
-
- if len(sys.argv) == 1 or sys.argv[1] == 'processes':
- print(' Using processes '.center(79, '-'))
- elif sys.argv[1] == 'threads':
- print(' Using threads '.center(79, '-'))
- import multiprocessing.dummy as multiprocessing
- else:
- print('Usage:\n\t%s [processes | threads]' % sys.argv[0])
- raise SystemExit(2)
-
test()
diff --git a/Doc/includes/mp_synchronize.py b/Doc/includes/mp_synchronize.py
deleted file mode 100644
index 81dbc38..0000000
--- a/Doc/includes/mp_synchronize.py
+++ /dev/null
@@ -1,278 +0,0 @@
-#
-# A test file for the `multiprocessing` package
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
-import time
-import sys
-import random
-from queue import Empty
-
-import multiprocessing # may get overwritten
-
-
-#### TEST_VALUE
-
-def value_func(running, mutex):
- random.seed()
- time.sleep(random.random()*4)
-
- mutex.acquire()
- print('\n\t\t\t' + str(multiprocessing.current_process()) + ' has finished')
- running.value -= 1
- mutex.release()
-
-def test_value():
- TASKS = 10
- running = multiprocessing.Value('i', TASKS)
- mutex = multiprocessing.Lock()
-
- for i in range(TASKS):
- p = multiprocessing.Process(target=value_func, args=(running, mutex))
- p.start()
-
- while running.value > 0:
- time.sleep(0.08)
- mutex.acquire()
- print(running.value, end=' ')
- sys.stdout.flush()
- mutex.release()
-
- print()
- print('No more running processes')
-
-
-#### TEST_QUEUE
-
-def queue_func(queue):
- for i in range(30):
- time.sleep(0.5 * random.random())
- queue.put(i*i)
- queue.put('STOP')
-
-def test_queue():
- q = multiprocessing.Queue()
-
- p = multiprocessing.Process(target=queue_func, args=(q,))
- p.start()
-
- o = None
- while o != 'STOP':
- try:
- o = q.get(timeout=0.3)
- print(o, end=' ')
- sys.stdout.flush()
- except Empty:
- print('TIMEOUT')
-
- print()
-
-
-#### TEST_CONDITION
-
-def condition_func(cond):
- cond.acquire()
- print('\t' + str(cond))
- time.sleep(2)
- print('\tchild is notifying')
- print('\t' + str(cond))
- cond.notify()
- cond.release()
-
-def test_condition():
- cond = multiprocessing.Condition()
-
- p = multiprocessing.Process(target=condition_func, args=(cond,))
- print(cond)
-
- cond.acquire()
- print(cond)
- cond.acquire()
- print(cond)
-
- p.start()
-
- print('main is waiting')
- cond.wait()
- print('main has woken up')
-
- print(cond)
- cond.release()
- print(cond)
- cond.release()
-
- p.join()
- print(cond)
-
-
-#### TEST_SEMAPHORE
-
-def semaphore_func(sema, mutex, running):
- sema.acquire()
-
- mutex.acquire()
- running.value += 1
- print(running.value, 'tasks are running')
- mutex.release()
-
- random.seed()
- time.sleep(random.random()*2)
-
- mutex.acquire()
- running.value -= 1
- print('%s has finished' % multiprocessing.current_process())
- mutex.release()
-
- sema.release()
-
-def test_semaphore():
- sema = multiprocessing.Semaphore(3)
- mutex = multiprocessing.RLock()
- running = multiprocessing.Value('i', 0)
-
- processes = [
- multiprocessing.Process(target=semaphore_func,
- args=(sema, mutex, running))
- for i in range(10)
- ]
-
- for p in processes:
- p.start()
-
- for p in processes:
- p.join()
-
-
-#### TEST_JOIN_TIMEOUT
-
-def join_timeout_func():
- print('\tchild sleeping')
- time.sleep(5.5)
- print('\n\tchild terminating')
-
-def test_join_timeout():
- p = multiprocessing.Process(target=join_timeout_func)
- p.start()
-
- print('waiting for process to finish')
-
- while 1:
- p.join(timeout=1)
- if not p.is_alive():
- break
- print('.', end=' ')
- sys.stdout.flush()
-
-
-#### TEST_EVENT
-
-def event_func(event):
- print('\t%r is waiting' % multiprocessing.current_process())
- event.wait()
- print('\t%r has woken up' % multiprocessing.current_process())
-
-def test_event():
- event = multiprocessing.Event()
-
- processes = [multiprocessing.Process(target=event_func, args=(event,))
- for i in range(5)]
-
- for p in processes:
- p.start()
-
- print('main is sleeping')
- time.sleep(2)
-
- print('main is setting event')
- event.set()
-
- for p in processes:
- p.join()
-
-
-#### TEST_SHAREDVALUES
-
-def sharedvalues_func(values, arrays, shared_values, shared_arrays):
- for i in range(len(values)):
- v = values[i][1]
- sv = shared_values[i].value
- assert v == sv
-
- for i in range(len(values)):
- a = arrays[i][1]
- sa = list(shared_arrays[i][:])
- assert a == sa
-
- print('Tests passed')
-
-def test_sharedvalues():
- values = [
- ('i', 10),
- ('h', -2),
- ('d', 1.25)
- ]
- arrays = [
- ('i', list(range(100))),
- ('d', [0.25 * i for i in range(100)]),
- ('H', list(range(1000)))
- ]
-
- shared_values = [multiprocessing.Value(id, v) for id, v in values]
- shared_arrays = [multiprocessing.Array(id, a) for id, a in arrays]
-
- p = multiprocessing.Process(
- target=sharedvalues_func,
- args=(values, arrays, shared_values, shared_arrays)
- )
- p.start()
- p.join()
-
- assert p.exitcode == 0
-
-
-####
-
-def test(namespace=multiprocessing):
- global multiprocessing
-
- multiprocessing = namespace
-
- for func in [test_value, test_queue, test_condition,
- test_semaphore, test_join_timeout, test_event,
- test_sharedvalues]:
-
- print('\n\t######## %s\n' % func.__name__)
- func()
-
- ignore = multiprocessing.active_children() # cleanup any old processes
- if hasattr(multiprocessing, '_debug_info'):
- info = multiprocessing._debug_info()
- if info:
- print(info)
- raise ValueError('there should be no positive refcounts left')
-
-
-if __name__ == '__main__':
- multiprocessing.freeze_support()
-
- assert len(sys.argv) in (1, 2)
-
- if len(sys.argv) == 1 or sys.argv[1] == 'processes':
- print(' Using processes '.center(79, '-'))
- namespace = multiprocessing
- elif sys.argv[1] == 'manager':
- print(' Using processes and a manager '.center(79, '-'))
- namespace = multiprocessing.Manager()
- namespace.Process = multiprocessing.Process
- namespace.current_process = multiprocessing.current_process
- namespace.active_children = multiprocessing.active_children
- elif sys.argv[1] == 'threads':
- print(' Using threads '.center(79, '-'))
- import multiprocessing.dummy as namespace
- else:
- print('Usage:\n\t%s [processes | manager | threads]' % sys.argv[0])
- raise SystemExit(2)
-
- test(namespace)
diff --git a/Doc/includes/mp_webserver.py b/Doc/includes/mp_webserver.py
deleted file mode 100644
index 651024d..0000000
--- a/Doc/includes/mp_webserver.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#
-# Example where a pool of http servers share a single listening socket
-#
-# On Windows this module depends on the ability to pickle a socket
-# object so that the worker processes can inherit a copy of the server
-# object. (We import `multiprocessing.reduction` to enable this pickling.)
-#
-# Not sure if we should synchronize access to `socket.accept()` method by
-# using a process-shared lock -- does not seem to be necessary.
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
-import os
-import sys
-
-from multiprocessing import Process, current_process, freeze_support
-from http.server import HTTPServer
-from http.server import SimpleHTTPRequestHandler
-
-if sys.platform == 'win32':
- import multiprocessing.reduction # make sockets pickable/inheritable
-
-
-def note(format, *args):
- sys.stderr.write('[%s]\t%s\n' % (current_process().name, format % args))
-
-
-class RequestHandler(SimpleHTTPRequestHandler):
- # we override log_message() to show which process is handling the request
- def log_message(self, format, *args):
- note(format, *args)
-
-def serve_forever(server):
- note('starting server')
- try:
- server.serve_forever()
- except KeyboardInterrupt:
- pass
-
-
-def runpool(address, number_of_processes):
- # create a single server object -- children will each inherit a copy
- server = HTTPServer(address, RequestHandler)
-
- # create child processes to act as workers
- for i in range(number_of_processes - 1):
- Process(target=serve_forever, args=(server,)).start()
-
- # main process also acts as a worker
- serve_forever(server)
-
-
-def test():
- DIR = os.path.join(os.path.dirname(__file__), '..')
- ADDRESS = ('localhost', 8000)
- NUMBER_OF_PROCESSES = 4
-
- print('Serving at http://%s:%d using %d worker processes' % \
- (ADDRESS[0], ADDRESS[1], NUMBER_OF_PROCESSES))
- print('To exit press Ctrl-' + ['C', 'Break'][sys.platform=='win32'])
-
- os.chdir(DIR)
- runpool(ADDRESS, NUMBER_OF_PROCESSES)
-
-
-if __name__ == '__main__':
- freeze_support()
- test()
diff --git a/Doc/includes/mp_workers.py b/Doc/includes/mp_workers.py
index e66d97b..3b92269 100644
--- a/Doc/includes/mp_workers.py
+++ b/Doc/includes/mp_workers.py
@@ -1,16 +1,3 @@
-#
-# Simple example which uses a pool of workers to carry out some tasks.
-#
-# Notice that the results will probably not come out of the output
-# queue in the same in the same order as the corresponding tasks were
-# put on the input queue. If it is important to get the results back
-# in the original order then consider using `Pool.map()` or
-# `Pool.imap()` (which will save on the amount of code needed anyway).
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-
import time
import random
diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h
index 32647c0..fcb846a 100644
--- a/Doc/includes/typestruct.h
+++ b/Doc/includes/typestruct.h
@@ -70,4 +70,11 @@ typedef struct _typeobject {
PyObject *tp_subclasses;
PyObject *tp_weaklist;
+ destructor tp_del;
+
+ /* Type attribute cache version tag. Added in version 2.6 */
+ unsigned int tp_version_tag;
+
+ destructor tp_finalize;
+
} PyTypeObject;
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index 0c9a9f2..d0fea3d 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -2,9 +2,9 @@
.. _install-index:
-*****************************
- Installing Python Modules
-*****************************
+********************************************
+ Installing Python Modules (Legacy version)
+********************************************
:Author: Greg Ward
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
new file mode 100644
index 0000000..1284613
--- /dev/null
+++ b/Doc/installing/index.rst
@@ -0,0 +1,210 @@
+.. highlightlang:: none
+
+.. _installing-index:
+
+*****************************
+ Installing Python Modules
+*****************************
+
+:Email: distutils-sig@python.org
+
+As a popular open source development project, Python has an active
+supporting community of contributors and users that also make their software
+available for other Python developers to use under open source license terms.
+
+This allows Python users to share and collaborate effectively, benefiting
+from the solutions others have already created to common (and sometimes
+even rare!) problems, as well as potentially contributing their own
+solutions to the common pool.
+
+This guide covers the installation part of the process. For a guide to
+creating and sharing your own Python projects, refer to the
+:ref:`distribution guide <distributing-index>`.
+
+.. note::
+
+ For corporate and other institutional users, be aware that many
+ organisations have their own policies around using and contributing to
+ open source software. Please take such policies into account when making
+ use of the distribution and installation tools provided with Python.
+
+
+Key terms
+=========
+
+* ``pip`` is the preferred installer program. Starting with Python 3.4, it
+ is included by default with the Python binary installers.
+* a virtual environment is a semi-isolated Python environment that allows
+ packages to be installed for use by a particular application, rather than
+ being installed system wide
+* ``pyvenv`` is the standard tool for creating virtual environments, and has
+ been part of Python since Python 3.3. Starting with Python 3.4, it
+ defaults to installing ``pip`` into all created virtual environments
+* the `Python Package Index <https://pypi.python.org/pypi>`__ is a public
+ repository of open source licensed packages made available for use by
+ other Python users
+* the `Python Packaging Authority
+ <http://packaging.python.org/en/latest/future.html>`__ are the group of
+ developers and documentation authors responsible for the maintenance and
+ evolution of the standard packaging tools and the associated metadata and
+ file format standards. They maintain a variety of tools, documentation
+ and issue trackers on both `GitHub <https://github.com/pypa>`__ and
+ `BitBucket <https://bitbucket.org/pypa/>`__.
+* ``distutils`` is the original build and distribution system first added to
+ the Python standard library in 1998. While direct use of ``distutils`` is
+ being phased out, it still laid the foundation for the current packaging
+ and distribution infrastructure, and it not only remains part of the
+ standard library, but its name lives on in other ways (such as the name
+ of the mailing list used to coordinate Python packaging standards
+ development).
+
+
+Basic usage
+===========
+
+The standard packaging tools are all designed to be used from the command
+line. For Windows users, the examples below assume that the option to
+adjust the system PATH environment variable was selected when installing
+Python. For Linux users, the command to install into the system version of
+Python 3 is likely to be ``pip3`` rather than ``pip``.
+
+The following command will install the latest version of a module and its
+dependencies from the Python Package Index::
+
+ pip install SomePackage
+
+It's also possible to specify an exact or minimum version directly on the
+command line::
+
+ pip install SomePackage==1.0.4 # specific version
+ pip install 'SomePackage>=1.0.4' # minimum version
+
+Normally, if a suitable module is already installed, attempting to install
+it again will have no effect. Upgrading existing modules must be requested
+explicitly::
+
+ pip install --upgrade SomePackage
+
+More information and resources regarding ``pip`` and its capabilities can be
+found in the `Python Packaging User Guide <http://packaging.python.org>`__.
+
+``pyvenv`` has its own documentation at :ref:`scripts-pyvenv`. Installing
+into an active virtual environment uses the commands shown above.
+
+.. seealso::
+
+ `Python Packaging User Guide: Installing Python packages
+ <http://packaging.python.org/en/latest/tutorial.html#installing-python-packages>`__
+
+
+How do I ...?
+=============
+
+These are quick answers or links for some common tasks.
+
+... install ``pip`` in versions of Python prior to Python 3.4?
+--------------------------------------------------------------
+
+Python only started bundling ``pip`` with Python 3.4. For earlier versions,
+``pip`` needs to be "bootstrapped" as described in the Python Packaging
+User Guide.
+
+.. seealso::
+
+ `Python Packaging User Guide: Installing the Tools
+ <http://packaging.python.org/en/latest/tutorial.html#installing-the-tools>`__
+
+
+.. installing-per-user-installation:
+
+... install packages just for the current user?
+-----------------------------------------------
+
+Passing the ``--user`` option to ``pip install`` will install a package
+just for the current user, rather than for all users of the system.
+
+
+... install scientific Python packages?
+---------------------------------------
+
+A number of scientific Python packages have complex binary dependencies, and
+aren't currently easy to install using ``pip`` directly. At this point in
+time, it will often be easier for users to install these packages by
+`other means
+<http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__
+rather than attempting to install them with ``pip``.
+
+.. seealso::
+
+ `Python Packaging User Guide: Installing Scientific Packages
+ <http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__
+
+
+... work with multiple versions of Python installed in parallel?
+----------------------------------------------------------------
+
+On Linux, Mac OS X and other POSIX systems, use the versioned Python commands
+in combination with the ``-m`` switch to run the appropriate copy of
+``pip``::
+
+ python2 -m pip install SomePackage # default Python 2
+ python2.7 -m pip install SomePackage # specifically Python 2.7
+ python3 -m pip install SomePackage # default Python 3
+ python3.4 -m pip install SomePackage # specifically Python 3.4
+
+(appropriately versioned ``pip`` commands may also be available)
+
+On Windows, use the ``py`` Python launcher in combination with the ``-m``
+switch::
+
+ py -2 -m pip install SomePackage # default Python 2
+ py -2.7 -m pip install SomePackage # specifically Python 2.7
+ py -3 -m pip install SomePackage # default Python 3
+ py -3.4 -m pip install SomePackage # specifically Python 3.4
+
+.. other questions:
+
+ Once the Development & Deployment part of PPUG is fleshed out, some of
+ those sections should be linked from new questions here (most notably,
+ we should have a question about avoiding depending on PyPI that links to
+ http://packaging.python.org/en/latest/deployment.html#pypi-mirrors-and-caches)
+
+
+Common installation issues
+==========================
+
+Installing into the system Python on Linux
+------------------------------------------
+
+On Linux systems, a Python installation will typically be included as part
+of the distribution. Installing into this Python installation requires
+root access to the system, and may interfere with the operation of the
+system package manager and other components of the system if a component
+is unexpectedly upgraded using ``pip``.
+
+On such systems, it is often better to use a virtual environment or a
+per-user installation when installing packages with ``pip``.
+
+
+Installing binary extensions
+----------------------------
+
+Python has typically relied heavily on source based distribution, with end
+users being expected to compile extension modules from source as part of
+the installation process.
+
+With the introduction of support for the binary ``wheel`` format, and the
+ability to publish wheels for at least Windows and Mac OS X through the
+Python Package Index, this problem is expected to diminish over time,
+as users are more regularly able to install pre-built extensions rather
+than needing to build them themselves.
+
+Some of the solutions for installing `scientific software
+<http://packaging.python.org/en/latest/platforms.html#installing-scientific-packages>`__
+that is not yet available as pre-built ``wheel`` files may also help with
+obtaining other binary extensions without needing to build them locally.
+
+.. seealso::
+
+ `Python Packaging User Guide: Binary Extensions
+ <http://packaging.python.org/en/latest/extensions.html>`__
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
index 1e5f42d..1092e6b 100644
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -142,6 +142,39 @@ and off individually. They are described here in more detail.
Removes usage of :func:`apply`. For example ``apply(function, *args,
**kwargs)`` is converted to ``function(*args, **kwargs)``.
+.. 2to3fixer:: asserts
+
+ Replaces deprecated :mod:`unittest` method names with the correct ones.
+
+ ================================ ==========================================
+ From To
+ ================================ ==========================================
+ ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b)
+ <unittest.TestCase.assertEqual>`
+ ``assertEquals(a, b)`` :meth:`assertEqual(a, b)
+ <unittest.TestCase.assertEqual>`
+ ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b)
+ <unittest.TestCase.assertNotEqual>`
+ ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b)
+ <unittest.TestCase.assertNotEqual>`
+ ``failUnless(a)`` :meth:`assertTrue(a)
+ <unittest.TestCase.assertTrue>`
+ ``assert_(a)`` :meth:`assertTrue(a)
+ <unittest.TestCase.assertTrue>`
+ ``failIf(a)`` :meth:`assertFalse(a)
+ <unittest.TestCase.assertFalse>`
+ ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal)
+ <unittest.TestCase.assertRaises>`
+ ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b)
+ <unittest.TestCase.assertAlmostEqual>`
+ ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b)
+ <unittest.TestCase.assertAlmostEqual>`
+ ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b)
+ <unittest.TestCase.assertNotAlmostEqual>`
+ ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b)
+ <unittest.TestCase.assertNotAlmostEqual>`
+ ================================ ==========================================
+
.. 2to3fixer:: basestring
Converts :class:`basestring` to :class:`str`.
@@ -342,6 +375,10 @@ and off individually. They are described here in more detail.
Handles the move of :func:`reduce` to :func:`functools.reduce`.
+.. 2to3fixer:: reload
+
+ Converts :func:`reload` to :func:`imp.reload`.
+
.. 2to3fixer:: renames
Changes :data:`sys.maxint` to :data:`sys.maxsize`.
diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst
index 9299124..7a73704 100644
--- a/Doc/library/abc.rst
+++ b/Doc/library/abc.rst
@@ -12,9 +12,9 @@
--------------
This module provides the infrastructure for defining :term:`abstract base
-classes <abstract base class>` (ABCs) in Python, as outlined in :pep:`3119`; see the PEP for why this
-was added to Python. (See also :pep:`3141` and the :mod:`numbers` module
-regarding a type hierarchy for numbers based on ABCs.)
+classes <abstract base class>` (ABCs) in Python, as outlined in :pep:`3119`;
+see the PEP for why this was added to Python. (See also :pep:`3141` and the
+:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.)
The :mod:`collections` module has some concrete classes that derive from
ABCs; these can, of course, be further derived. In addition the
@@ -23,7 +23,7 @@ a class or instance provides a particular interface, for example, is it
hashable or a mapping.
-This module provides the following class:
+This module provides the following classes:
.. class:: ABCMeta
@@ -58,6 +58,10 @@ This module provides the following class:
.. versionchanged:: 3.3
Returns the registered subclass, to allow usage as a class decorator.
+ .. versionchanged:: 3.4
+ To detect calls to :meth:`register`, you can use the
+ :func:`get_cache_token` function.
+
You can also override this method in an abstract base class:
.. method:: __subclasshook__(subclass)
@@ -127,6 +131,19 @@ This module provides the following class:
available as a method of ``Foo``, so it is provided separately.
+.. class:: ABC
+
+ A helper class that has :class:`ABCMeta` as its metaclass. With this class,
+ an abstract base class can be created by simply deriving from :class:`ABC`,
+ avoiding sometimes confusing metaclass usage.
+
+ Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore
+ inheriting from :class:`ABC` requires the usual precautions regarding metaclass
+ usage, as multiple inheritance may lead to metaclass conflicts.
+
+ .. versionadded:: 3.4
+
+
The :mod:`abc` module also provides the following decorators:
.. decorator:: abstractmethod
@@ -295,6 +312,19 @@ The :mod:`abc` module also provides the following decorators:
:func:`abstractmethod`, making this decorator redundant.
+The :mod:`abc` module also provides the following functions:
+
+.. function:: get_cache_token()
+
+ Returns the current abstract base class cache token.
+
+ The token is an opaque object (that supports equality testing) identifying
+ the current version of the abstract base class cache for virtual subclasses.
+ The token changes with every call to :meth:`ABCMeta.register` on any ABC.
+
+ .. versionadded:: 3.4
+
+
.. rubric:: Footnotes
.. [#] C++ programmers should note that Python's virtual base class
diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst
index 8a3541b..6fbcf28 100644
--- a/Doc/library/aifc.rst
+++ b/Doc/library/aifc.rst
@@ -30,8 +30,8 @@ sampling rate or frame rate is the number of times per second the sound is
sampled. The number of channels indicate if the audio is mono, stereo, or
quadro. Each frame consists of one sample per channel. The sample size is the
size in bytes of each sample. Thus a frame consists of
-*nchannels*\*\ *samplesize* bytes, and a second's worth of audio consists of
-*nchannels*\*\ *samplesize*\*\ *framerate* bytes.
+``nchannels * samplesize`` bytes, and a second's worth of audio consists of
+``nchannels * samplesize * framerate`` bytes.
For example, CD quality audio has a sample size of two bytes (16 bits), uses two
channels (stereo) and has a frame rate of 44,100 frames/second. This gives a
@@ -51,6 +51,11 @@ Module :mod:`aifc` defines the following function:
used for writing, the file object should be seekable, unless you know ahead of
time how many samples you are going to write in total and use
:meth:`writeframesraw` and :meth:`setnframes`.
+ The :func:`.open` function may be used in a :keyword:`with` statement. When
+ the :keyword:`with` block completes, the :meth:`~aifc.close` method is called.
+
+ .. versionchanged:: 3.4
+ Support for the :keyword:`with` statement was added.
Objects returned by :func:`.open` when a file is opened for reading have the
following methods:
@@ -92,7 +97,9 @@ following methods:
.. method:: aifc.getparams()
- Return a tuple consisting of all of the above values in the above order.
+ Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth,
+ framerate, nframes, comptype, compname)``, equivalent to output of the
+ :meth:`get\*` methods.
.. method:: aifc.getmarkers()
@@ -218,12 +225,18 @@ number of frames must be filled in.
Write data to the output file. This method can only be called after the audio
file parameters have been set.
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
+
.. method:: aifc.writeframesraw(data)
Like :meth:`writeframes`, except that the header of the audio file is not
updated.
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
+
.. method:: aifc.close()
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index b686c44..b1b5135 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -977,9 +977,9 @@ See the section on the default_ keyword argument for information on when the
``type`` argument is applied to default arguments.
To ease the use of various types of files, the argparse module provides the
-factory FileType which takes the ``mode=`` and ``bufsize=`` arguments of the
-:func:`open` function. For example, ``FileType('w')`` can be used to create a
-writable file::
+factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and
+``errors=`` arguments of the :func:`open` function. For example,
+``FileType('w')`` can be used to create a writable file::
>>> parser = argparse.ArgumentParser()
>>> parser.add_argument('bar', type=argparse.FileType('w'))
@@ -1648,17 +1648,19 @@ Sub-commands
FileType objects
^^^^^^^^^^^^^^^^
-.. class:: FileType(mode='r', bufsize=None)
+.. class:: FileType(mode='r', bufsize=-1, encoding=None, errors=None)
The :class:`FileType` factory creates objects that can be passed to the type
argument of :meth:`ArgumentParser.add_argument`. Arguments that have
- :class:`FileType` objects as their type will open command-line arguments as files
- with the requested modes and buffer sizes::
+ :class:`FileType` objects as their type will open command-line arguments as
+ files with the requested modes, buffer sizes, encodings and error handling
+ (see the :func:`open` function for more details)::
>>> parser = argparse.ArgumentParser()
- >>> parser.add_argument('--output', type=argparse.FileType('wb', 0))
- >>> parser.parse_args(['--output', 'out'])
- Namespace(output=<_io.BufferedWriter name='out'>)
+ >>> parser.add_argument('--raw', type=argparse.FileType('wb', 0))
+ >>> parser.add_argument('out', type=argparse.FileType('w', encoding='UTF-8'))
+ >>> parser.parse_args(['--raw', 'raw.dat', 'file.txt'])
+ Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>)
FileType objects understand the pseudo-argument ``'-'`` and automatically
convert this into ``sys.stdin`` for readable :class:`FileType` objects and
@@ -1669,6 +1671,9 @@ FileType objects
>>> parser.parse_args(['-'])
Namespace(infile=<_io.TextIOWrapper name='<stdin>' encoding='UTF-8'>)
+ .. versionadded:: 3.4
+ The *encodings* and *errors* keyword arguments.
+
Argument groups
^^^^^^^^^^^^^^^
diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst
index adbd3be..c6fa061 100644
--- a/Doc/library/asynchat.rst
+++ b/Doc/library/asynchat.rst
@@ -10,6 +10,11 @@
--------------
+.. note::
+
+ This module exists for backwards compatibility only. For new code we
+ recommend using :mod:`asyncio`.
+
This module builds on the :mod:`asyncore` infrastructure, simplifying
asynchronous clients and servers and making it easier to handle protocols
whose elements are terminated by arbitrary strings, or are of variable length.
diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
new file mode 100644
index 0000000..9d6f054
--- /dev/null
+++ b/Doc/library/asyncio-dev.rst
@@ -0,0 +1,251 @@
+.. currentmodule:: asyncio
+
+.. _asyncio-dev:
+
+Develop with asyncio
+====================
+
+Asynchronous programming is different than classical "sequential" programming.
+This page lists common traps and explains how to avoid them.
+
+
+.. _asyncio-multithreading:
+
+Concurrency and multithreading
+------------------------------
+
+An event loop runs in a thread and executes all callbacks and tasks in the same
+thread. While a task is running in the event loop, no other task is running in
+the same thread. But when the task uses ``yield from``, the task is suspended
+and the event loop executes the next task.
+
+To schedule a callback from a different thread, the
+:meth:`BaseEventLoop.call_soon_threadsafe` method should be used. Example to
+schedule a coroutine from a different thread::
+
+ loop.call_soon_threadsafe(asyncio.async, coro_func())
+
+Most asyncio objects are not thread safe. You should only worry if you access
+objects outside the event loop. For example, to cancel a future, don't call
+directly its :meth:`Future.cancel` method, but::
+
+ loop.call_soon_threadsafe(fut.cancel)
+
+To handle signals and to execute subprocesses, the event loop must be run in
+the main thread.
+
+The :meth:`BaseEventLoop.run_in_executor` method can be used with a thread pool
+executor to execute a callback in different thread to not block the thread of
+the event loop.
+
+.. seealso::
+
+ See the :ref:`Synchronization primitives <asyncio-sync>` section to
+ synchronize tasks.
+
+
+.. _asyncio-handle-blocking:
+
+Handle blocking functions correctly
+-----------------------------------
+
+Blocking functions should not be called directly. For example, if a function
+blocks for 1 second, other tasks are delayed by 1 second which can have an
+important impact on reactivity.
+
+For networking and subprocesses, the :mod:`asyncio` module provides high-level
+APIs like :ref:`protocols <asyncio-protocol>`.
+
+An executor can be used to run a task in a different thread or even in a
+different process, to not block the thread of the event loop. See the
+:meth:`BaseEventLoop.run_in_executor` method.
+
+.. seealso::
+
+ The :ref:`Delayed calls <asyncio-delayed-calls>` section details how the
+ event loop handles time.
+
+
+.. _asyncio-logger:
+
+Logging
+-------
+
+The :mod:`asyncio` module logs information with the :mod:`logging` module in
+the logger ``'asyncio'``.
+
+
+.. _asyncio-coroutine-not-scheduled:
+
+Detect coroutine objects never scheduled
+----------------------------------------
+
+When a coroutine function is called but not passed to :func:`async` or to the
+:class:`Task` constructor, it is not scheduled and it is probably a bug.
+
+To detect such bug, set the environment variable :envvar:`PYTHONASYNCIODEBUG`
+to ``1``. When the coroutine object is destroyed by the garbage collector, a
+log will be emitted with the traceback where the coroutine function was called.
+See the :ref:`asyncio logger <asyncio-logger>`.
+
+The debug flag changes the behaviour of the :func:`coroutine` decorator. The
+debug flag value is only used when then coroutine function is defined, not when
+it is called. Coroutine functions defined before the debug flag is set to
+``True`` will not be tracked. For example, it is not possible to debug
+coroutines defined in the :mod:`asyncio` module, because the module must be
+imported before the flag value can be changed.
+
+Example with the bug::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def test():
+ print("never scheduled")
+
+ test()
+
+Output in debug mode::
+
+ Coroutine 'test' defined at test.py:4 was never yielded from
+
+The fix is to call the :func:`async` function or create a :class:`Task` object
+with this coroutine object.
+
+
+Detect exceptions not consumed
+------------------------------
+
+Python usually calls :func:`sys.displayhook` on unhandled exceptions. If
+:meth:`Future.set_exception` is called, but the exception is not consumed,
+:func:`sys.displayhook` is not called. Instead, a log is emitted when the
+future is deleted by the garbage collector, with the traceback where the
+exception was raised. See the :ref:`asyncio logger <asyncio-logger>`.
+
+Example of unhandled exception::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def bug():
+ raise Exception("not consumed")
+
+ loop = asyncio.get_event_loop()
+ asyncio.async(bug())
+ loop.run_forever()
+
+Output::
+
+ Future/Task exception was never retrieved:
+ Traceback (most recent call last):
+ File "/usr/lib/python3.4/asyncio/tasks.py", line 279, in _step
+ result = next(coro)
+ File "/usr/lib/python3.4/asyncio/tasks.py", line 80, in coro
+ res = func(*args, **kw)
+ File "test.py", line 5, in bug
+ raise Exception("not consumed")
+ Exception: not consumed
+
+There are different options to fix this issue. The first option is to chain to
+coroutine in another coroutine and use classic try/except::
+
+ @asyncio.coroutine
+ def handle_exception():
+ try:
+ yield from bug()
+ except Exception:
+ print("exception consumed")
+
+ loop = asyncio.get_event_loop()
+ asyncio.async(handle_exception())
+ loop.run_forever()
+
+Another option is to use the :meth:`BaseEventLoop.run_until_complete`
+function::
+
+ task = asyncio.async(bug())
+ try:
+ loop.run_until_complete(task)
+ except Exception:
+ print("exception consumed")
+
+See also the :meth:`Future.exception` method.
+
+
+Chain coroutines correctly
+--------------------------
+
+When a coroutine function calls other coroutine functions and tasks, they
+should be chained explicitly with ``yield from``. Otherwise, the execution is
+not guaranteed to be sequential.
+
+Example with different bugs using :func:`asyncio.sleep` to simulate slow
+operations::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def create():
+ yield from asyncio.sleep(3.0)
+ print("(1) create file")
+
+ @asyncio.coroutine
+ def write():
+ yield from asyncio.sleep(1.0)
+ print("(2) write into file")
+
+ @asyncio.coroutine
+ def close():
+ print("(3) close file")
+
+ @asyncio.coroutine
+ def test():
+ asyncio.async(create())
+ asyncio.async(write())
+ asyncio.async(close())
+ yield from asyncio.sleep(2.0)
+ loop.stop()
+
+ loop = asyncio.get_event_loop()
+ asyncio.async(test())
+ loop.run_forever()
+ print("Pending tasks at exit: %s" % asyncio.Task.all_tasks(loop))
+ loop.close()
+
+Expected output::
+
+ (1) create file
+ (2) write into file
+ (3) close file
+ Pending tasks at exit: set()
+
+Actual output::
+
+ (3) close file
+ (2) write into file
+ Pending tasks at exit: {Task(<create>)<PENDING>}
+
+The loop stopped before the ``create()`` finished, ``close()`` has been called
+before ``write()``, whereas coroutine functions were called in this order:
+``create()``, ``write()``, ``close()``.
+
+To fix the example, tasks must be marked with ``yield from``::
+
+ @asyncio.coroutine
+ def test():
+ yield from asyncio.async(create())
+ yield from asyncio.async(write())
+ yield from asyncio.async(close())
+ yield from asyncio.sleep(2.0)
+ loop.stop()
+
+Or without ``asyncio.async()``::
+
+ @asyncio.coroutine
+ def test():
+ yield from create()
+ yield from write()
+ yield from close()
+ yield from asyncio.sleep(2.0)
+ loop.stop()
+
diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst
new file mode 100644
index 0000000..835266f
--- /dev/null
+++ b/Doc/library/asyncio-eventloop.rst
@@ -0,0 +1,668 @@
+.. currentmodule:: asyncio
+
+.. _asyncio-event-loop:
+
+Event loops
+===========
+
+The event loop is the central execution device provided by :mod:`asyncio`.
+It provides multiple facilities, amongst which:
+
+* Registering, executing and cancelling delayed calls (timeouts).
+
+* Creating client and server :ref:`transports <asyncio-transport>` for various
+ kinds of communication.
+
+* Launching subprocesses and the associated :ref:`transports
+ <asyncio-transport>` for communication with an external program.
+
+* Delegating costly function calls to a pool of threads.
+
+Event loop policies and the default policy
+------------------------------------------
+
+Event loop management is abstracted with a *policy* pattern, to provide maximal
+flexibility for custom platforms and frameworks. Throughout the execution of a
+process, a single global policy object manages the event loops available to the
+process based on the calling context. A policy is an object implementing the
+:class:`AbstractEventLoopPolicy` interface.
+
+For most users of :mod:`asyncio`, policies never have to be dealt with
+explicitly, since the default global policy is sufficient.
+
+The default policy defines context as the current thread, and manages an event
+loop per thread that interacts with :mod:`asyncio`. The module-level functions
+:func:`get_event_loop` and :func:`set_event_loop` provide convenient access to
+event loops managed by the default policy.
+
+Event loop functions
+--------------------
+
+The following functions are convenient shortcuts to accessing the methods of the
+global policy. Note that this provides access to the default policy, unless an
+alternative policy was set by calling :func:`set_event_loop_policy` earlier in
+the execution of the process.
+
+.. function:: get_event_loop()
+
+ Equivalent to calling ``get_event_loop_policy().get_event_loop()``.
+
+.. function:: set_event_loop(loop)
+
+ Equivalent to calling ``get_event_loop_policy().set_event_loop(loop)``.
+
+.. function:: new_event_loop()
+
+ Equivalent to calling ``get_event_loop_policy().new_event_loop()``.
+
+Event loop policy interface
+---------------------------
+
+An event loop policy must implement the following interface:
+
+.. class:: AbstractEventLoopPolicy
+
+ .. method:: get_event_loop()
+
+ Get the event loop for current context. Returns an event loop object
+ implementing :class:`BaseEventLoop` interface, or raises an exception in case
+ no event loop has been set for the current context and the current policy
+ does not specify to create one. It should never return ``None``.
+
+ .. method:: set_event_loop(loop)
+
+ Set the event loop of the current context to *loop*.
+
+ .. method:: new_event_loop()
+
+ Create and return a new event loop object according to this policy's rules.
+ If there's need to set this loop as the event loop of the current context,
+ :meth:`set_event_loop` must be called explicitly.
+
+Access to the global loop policy
+--------------------------------
+
+.. function:: get_event_loop_policy()
+
+ Get the current event loop policy.
+
+.. function:: set_event_loop_policy(policy)
+
+ Set the current event loop policy. If *policy* is ``None``, the default
+ policy is restored.
+
+Run an event loop
+-----------------
+
+.. method:: BaseEventLoop.run_forever()
+
+ Run until :meth:`stop` is called.
+
+.. method:: BaseEventLoop.run_until_complete(future)
+
+ Run until the :class:`Future` is done.
+
+ If the argument is a :ref:`coroutine <coroutine>`, it is wrapped
+ in a :class:`Task`.
+
+ Return the Future's result, or raise its exception.
+
+.. method:: BaseEventLoop.is_running()
+
+ Returns running status of event loop.
+
+.. method:: BaseEventLoop.stop()
+
+ Stop running the event loop.
+
+ Every callback scheduled before :meth:`stop` is called will run.
+ Callback scheduled after :meth:`stop` is called won't. However, those
+ callbacks will run if :meth:`run_forever` is called again later.
+
+.. method:: BaseEventLoop.close()
+
+ Close the event loop. The loop should not be running.
+
+ This clears the queues and shuts down the executor, but does not wait for
+ the executor to finish.
+
+ This is idempotent and irreversible. No other methods should be called after
+ this one.
+
+
+Calls
+-----
+
+.. method:: BaseEventLoop.call_soon(callback, \*args)
+
+ Arrange for a callback to be called as soon as possible.
+
+ This operates as a FIFO queue, callbacks are called in the order in
+ which they are registered. Each callback will be called exactly once.
+
+ Any positional arguments after the callback will be passed to the
+ callback when it is called.
+
+ An instance of :class:`asyncio.Handle` is returned.
+
+.. method:: BaseEventLoop.call_soon_threadsafe(callback, \*args)
+
+ Like :meth:`call_soon`, but thread safe.
+
+
+.. _asyncio-delayed-calls:
+
+Delayed calls
+-------------
+
+The event loop has its own internal clock for computing timeouts.
+Which clock is used depends on the (platform-specific) event loop
+implementation; ideally it is a monotonic clock. This will generally be
+a different clock than :func:`time.time`.
+
+.. note::
+
+ Timeouts (relative *delay* or absolute *when*) should not exceed one day.
+
+
+.. method:: BaseEventLoop.call_later(delay, callback, *args)
+
+ Arrange for the *callback* to be called after the given *delay*
+ seconds (either an int or float).
+
+ An instance of :class:`asyncio.Handle` is returned.
+
+ *callback* will be called exactly once per call to :meth:`call_later`.
+ If two callbacks are scheduled for exactly the same time, it is
+ undefined which will be called first.
+
+ The optional positional *args* will be passed to the callback when it
+ is called. If you want the callback to be called with some named
+ arguments, use a closure or :func:`functools.partial`.
+
+.. method:: BaseEventLoop.call_at(when, callback, *args)
+
+ Arrange for the *callback* to be called at the given absolute timestamp
+ *when* (an int or float), using the same time reference as :meth:`time`.
+
+ This method's behavior is the same as :meth:`call_later`.
+
+.. method:: BaseEventLoop.time()
+
+ Return the current time, as a :class:`float` value, according to the
+ event loop's internal clock.
+
+.. seealso::
+
+ The :func:`asyncio.sleep` function.
+
+
+Creating connections
+--------------------
+
+.. method:: BaseEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None)
+
+ Create a streaming transport connection to a given Internet *host* and
+ *port*: socket family :py:data:`~socket.AF_INET` or
+ :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
+ socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a
+ callable returning a :ref:`protocol <asyncio-protocol>` instance.
+
+ This method is a :ref:`coroutine <coroutine>` which will try to
+ establish the connection in the background. When successful, the
+ coroutine returns a ``(transport, protocol)`` pair.
+
+ The chronological synopsis of the underlying operation is as follows:
+
+ #. The connection is established, and a :ref:`transport <asyncio-transport>`
+ is created to represent it.
+
+ #. *protocol_factory* is called without arguments and must return a
+ :ref:`protocol <asyncio-protocol>` instance.
+
+ #. The protocol instance is tied to the transport, and its
+ :meth:`connection_made` method is called.
+
+ #. The coroutine returns successfully with the ``(transport, protocol)``
+ pair.
+
+ The created transport is an implementation-dependent bidirectional stream.
+
+ .. note::
+ *protocol_factory* can be any kind of callable, not necessarily
+ a class. For example, if you want to use a pre-created
+ protocol instance, you can pass ``lambda: my_protocol``.
+
+ Options allowing to change how the connection is created:
+
+ * *ssl*: if given and not false, a SSL/TLS transport is created
+ (by default a plain TCP transport is created). If *ssl* is
+ a :class:`ssl.SSLContext` object, this context is used to create
+ the transport; if *ssl* is :const:`True`, a context with some
+ unspecified default settings is used.
+
+ .. seealso:: :ref:`SSL/TLS security considerations <ssl-security>`
+
+ * *server_hostname*, is only for use together with *ssl*,
+ and sets or overrides the hostname that the target server's certificate
+ will be matched against. By default the value of the *host* argument
+ is used. If *host* is empty, there is no default and you must pass a
+ value for *server_hostname*. If *server_hostname* is an empty
+ string, hostname matching is disabled (which is a serious security
+ risk, allowing for man-in-the-middle-attacks).
+
+ * *family*, *proto*, *flags* are the optional address family, protocol
+ and flags to be passed through to getaddrinfo() for *host* resolution.
+ If given, these should all be integers from the corresponding
+ :mod:`socket` module constants.
+
+ * *sock*, if given, should be an existing, already connected
+ :class:`socket.socket` object to be used by the transport.
+ If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags*
+ and *local_addr* should be specified.
+
+ * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used
+ to bind the socket to locally. The *local_host* and *local_port*
+ are looked up using getaddrinfo(), similarly to *host* and *port*.
+
+ .. seealso::
+
+ The :func:`open_connection` function can be used to get a pair of
+ (:class:`StreamReader`, :class:`StreamWriter`) instead of a protocol.
+
+
+.. method:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0)
+
+ Create datagram connection: socket family :py:data:`~socket.AF_INET` or
+ :py:data:`~socket.AF_INET6` depending on *host* (or *family* if specified),
+ socket type :py:data:`~socket.SOCK_DGRAM`.
+
+ This method is a :ref:`coroutine <coroutine>` which will try to
+ establish the connection in the background. When successful, the
+ coroutine returns a ``(transport, protocol)`` pair.
+
+ See the :meth:`BaseEventLoop.create_connection` method for parameters.
+
+
+.. method:: BaseEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None)
+
+ Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket
+ type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket
+ family is used to communicate between processes on the same machine
+ efficiently.
+
+ This method is a :ref:`coroutine <coroutine>` which will try to
+ establish the connection in the background. When successful, the
+ coroutine returns a ``(transport, protocol)`` pair.
+
+ See the :meth:`BaseEventLoop.create_connection` method for parameters.
+
+ Availability: UNIX.
+
+
+Creating listening connections
+------------------------------
+
+.. method:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None)
+
+ A :ref:`coroutine <coroutine>` method which creates a TCP server bound to
+ host and port.
+
+ The return value is a :class:`AbstractServer` object which can be used to stop
+ the service.
+
+ If *host* is an empty string or None all interfaces are assumed
+ and a list of multiple sockets will be returned (most likely
+ one for IPv4 and another one for IPv6).
+
+ *family* can be set to either :data:`~socket.AF_INET` or
+ :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set
+ it will be determined from host (defaults to :data:`~socket.AF_UNSPEC`).
+
+ *flags* is a bitmask for :meth:`getaddrinfo`.
+
+ *sock* can optionally be specified in order to use a preexisting
+ socket object.
+
+ *backlog* is the maximum number of queued connections passed to
+ :meth:`~socket.socket.listen` (defaults to 100).
+
+ ssl can be set to an :class:`~ssl.SSLContext` to enable SSL over the
+ accepted connections.
+
+ *reuse_address* tells the kernel to reuse a local socket in
+ TIME_WAIT state, without waiting for its natural timeout to
+ expire. If not specified will automatically be set to True on
+ UNIX.
+
+ .. seealso::
+
+ The function :func:`start_server` creates a (:class:`StreamReader`,
+ :class:`StreamWriter`) pair and calls back a function with this pair.
+
+
+.. method:: BaseEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None)
+
+ Similar to :meth:`BaseEventLoop.create_server`, but specific to the
+ socket family :py:data:`~socket.AF_UNIX`.
+
+ Availability: UNIX.
+
+
+
+Watch file descriptors
+----------------------
+
+.. method:: BaseEventLoop.add_reader(fd, callback, \*args)
+
+ Start watching the file descriptor for read availability and then call the
+ *callback* with specified arguments.
+
+.. method:: BaseEventLoop.remove_reader(fd)
+
+ Stop watching the file descriptor for read availability.
+
+.. method:: BaseEventLoop.add_writer(fd, callback, \*args)
+
+ Start watching the file descriptor for write availability and then call the
+ *callback* with specified arguments.
+
+.. method:: BaseEventLoop.remove_writer(fd)
+
+ Stop watching the file descriptor for write availability.
+
+
+Low-level socket operations
+---------------------------
+
+.. method:: BaseEventLoop.sock_recv(sock, nbytes)
+
+ Receive data from the socket. The return value is a bytes object
+ representing the data received. The maximum amount of data to be received
+ at once is specified by *nbytes*.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. seealso::
+
+ The :meth:`socket.socket.recv` method.
+
+.. method:: BaseEventLoop.sock_sendall(sock, data)
+
+ Send data to the socket. The socket must be connected to a remote socket.
+ This method continues to send data from *data* until either all data has
+ been sent or an error occurs. ``None`` is returned on success. On error,
+ an exception is raised, and there is no way to determine how much data, if
+ any, was successfully processed by the receiving end of the connection.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. seealso::
+
+ The :meth:`socket.socket.sendall` method.
+
+.. method:: BaseEventLoop.sock_connect(sock, address)
+
+ Connect to a remote socket at *address*.
+
+ The *address* must be already resolved to avoid the trap of hanging the
+ entire event loop when the address requires doing a DNS lookup. For
+ example, it must be an IP address, not an hostname, for
+ :py:data:`~socket.AF_INET` and :py:data:`~socket.AF_INET6` address families.
+ Use :meth:`getaddrinfo` to resolve the hostname asynchronously.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. seealso::
+
+ The :meth:`BaseEventLoop.create_connection` method, the
+ :func:`open_connection` function and the :meth:`socket.socket.connect`
+ method.
+
+
+.. method:: BaseEventLoop.sock_accept(sock)
+
+ Accept a connection. The socket must be bound to an address and listening
+ for connections. The return value is a pair ``(conn, address)`` where *conn*
+ is a *new* socket object usable to send and receive data on the connection,
+ and *address* is the address bound to the socket on the other end of the
+ connection.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. seealso::
+
+ The :meth:`BaseEventLoop.create_server` method, the :func:`start_server`
+ function and the :meth:`socket.socket.accept` method.
+
+
+Resolve host name
+-----------------
+
+.. method:: BaseEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0)
+
+ This method is a :ref:`coroutine <coroutine>`, similar to
+ :meth:`socket.getaddrinfo` function but non-blocking.
+
+.. method:: BaseEventLoop.getnameinfo(sockaddr, flags=0)
+
+ This method is a :ref:`coroutine <coroutine>`, similar to
+ :meth:`socket.getnameinfo` function but non-blocking.
+
+
+Connect pipes
+-------------
+
+.. method:: BaseEventLoop.connect_read_pipe(protocol_factory, pipe)
+
+ Register read pipe in eventloop.
+
+ *protocol_factory* should instantiate object with :class:`Protocol`
+ interface. pipe is file-like object already switched to nonblocking.
+ Return pair (transport, protocol), where transport support
+ :class:`ReadTransport` interface.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+.. method:: BaseEventLoop.connect_write_pipe(protocol_factory, pipe)
+
+ Register write pipe in eventloop.
+
+ *protocol_factory* should instantiate object with :class:`BaseProtocol`
+ interface. Pipe is file-like object already switched to nonblocking.
+ Return pair (transport, protocol), where transport support
+ :class:`WriteTransport` interface.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+.. seealso::
+
+ The :meth:`BaseEventLoop.subprocess_exec` and
+ :meth:`BaseEventLoop.subprocess_shell` methods.
+
+
+UNIX signals
+------------
+
+Availability: UNIX only.
+
+.. method:: BaseEventLoop.add_signal_handler(signum, callback, \*args)
+
+ Add a handler for a signal.
+
+ Raise :exc:`ValueError` if the signal number is invalid or uncatchable.
+ Raise :exc:`RuntimeError` if there is a problem setting up the handler.
+
+.. method:: BaseEventLoop.remove_signal_handler(sig)
+
+ Remove a handler for a signal.
+
+ Return ``True`` if a signal handler was removed, ``False`` if not.
+
+.. seealso::
+
+ The :mod:`signal` module.
+
+
+Executor
+--------
+
+Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or
+pool of processes). By default, an event loop uses a thread pool executor
+(:class:`~concurrent.futures.ThreadPoolExecutor`).
+
+.. method:: BaseEventLoop.run_in_executor(executor, callback, \*args)
+
+ Arrange for a callback to be called in the specified executor.
+
+ The *executor* argument should be an :class:`~concurrent.futures.Executor`
+ instance. The default executor is used if *executor* is ``None``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+.. method:: BaseEventLoop.set_default_executor(executor)
+
+ Set the default executor used by :meth:`run_in_executor`.
+
+
+Error Handling API
+------------------
+
+Allows to customize how exceptions are handled in the event loop.
+
+.. method:: BaseEventLoop.set_exception_handler(handler)
+
+ Set *handler* as the new event loop exception handler.
+
+ If *handler* is ``None``, the default exception handler will
+ be set.
+
+ If *handler* is a callable object, it should have a
+ matching signature to ``(loop, context)``, where ``loop``
+ will be a reference to the active event loop, ``context``
+ will be a ``dict`` object (see :meth:`call_exception_handler`
+ documentation for details about context).
+
+.. method:: BaseEventLoop.default_exception_handler(context)
+
+ Default exception handler.
+
+ This is called when an exception occurs and no exception
+ handler is set, and can be called by a custom exception
+ handler that wants to defer to the default behavior.
+
+ *context* parameter has the same meaning as in
+ :meth:`call_exception_handler`.
+
+.. method:: BaseEventLoop.call_exception_handler(context)
+
+ Call the current event loop exception handler.
+
+ *context* is a ``dict`` object containing the following keys
+ (new keys may be introduced later):
+
+ * 'message': Error message;
+ * 'exception' (optional): Exception object;
+ * 'future' (optional): :class:`asyncio.Future` instance;
+ * 'handle' (optional): :class:`asyncio.Handle` instance;
+ * 'protocol' (optional): :ref:`Protocol <asyncio-protocol>` instance;
+ * 'transport' (optional): :ref:`Transport <asyncio-transport>` instance;
+ * 'socket' (optional): :class:`socket.socket` instance.
+
+ .. note::
+
+ Note: this method should not be overloaded in subclassed
+ event loops. For any custom exception handling, use
+ :meth:`set_exception_handler()` method.
+
+Debug mode
+----------
+
+.. method:: BaseEventLoop.get_debug()
+
+ Get the debug mode (:class:`bool`) of the event loop, ``False`` by default.
+
+.. method:: BaseEventLoop.set_debug(enabled: bool)
+
+ Set the debug mode of the event loop.
+
+.. seealso::
+
+ The :ref:`Develop with asyncio <asyncio-dev>` section.
+
+
+Server
+------
+
+.. class:: AbstractServer
+
+ Abstract server returned by :func:`BaseEventLoop.create_server`.
+
+ .. method:: close()
+
+ Stop serving. This leaves existing connections open.
+
+ .. method:: wait_closed()
+
+ A :ref:`coroutine <coroutine>` to wait until service is closed.
+
+
+Handle
+------
+
+.. class:: Handle
+
+ A callback wrapper object returned by :func:`BaseEventLoop.call_soon`,
+ :func:`BaseEventLoop.call_soon_threadsafe`, :func:`BaseEventLoop.call_later`,
+ and :func:`BaseEventLoop.call_at`.
+
+ .. method:: cancel()
+
+ Cancel the call.
+
+
+.. _asyncio-hello-world-callback:
+
+Example: Hello World (callback)
+-------------------------------
+
+Print ``Hello World`` every two seconds, using a callback::
+
+ import asyncio
+
+ def print_and_repeat(loop):
+ print('Hello World')
+ loop.call_later(2, print_and_repeat, loop)
+
+ loop = asyncio.get_event_loop()
+ loop.call_soon(print_and_repeat, loop)
+ loop.run_forever()
+
+.. seealso::
+
+ :ref:`Hello World example using a coroutine <asyncio-hello-world-coroutine>`.
+
+
+Example: Set signal handlers for SIGINT and SIGTERM
+---------------------------------------------------
+
+Register handlers for signals :py:data:`SIGINT` and :py:data:`SIGTERM`::
+
+ import asyncio
+ import functools
+ import os
+ import signal
+
+ def ask_exit(signame):
+ print("got signal %s: exit" % signame)
+ loop.stop()
+
+ loop = asyncio.get_event_loop()
+ for signame in ('SIGINT', 'SIGTERM'):
+ loop.add_signal_handler(getattr(signal, signame),
+ functools.partial(ask_exit, signame))
+
+ print("Event loop running forever, press CTRL+c to interrupt.")
+ print("pid %s: send SIGINT or SIGTERM to exit." % os.getpid())
+ loop.run_forever()
+
diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst
new file mode 100644
index 0000000..58c94ae
--- /dev/null
+++ b/Doc/library/asyncio-protocol.rst
@@ -0,0 +1,502 @@
+.. currentmodule:: asyncio
+
++++++++++++++++++++++++++++++++++++++++++
+Transports and protocols (low-level API)
++++++++++++++++++++++++++++++++++++++++++
+
+.. _asyncio-transport:
+
+Transports
+==========
+
+Transports are classes provided by :mod:`asyncio` in order to abstract
+various kinds of communication channels. You generally won't instantiate
+a transport yourself; instead, you will call a :class:`BaseEventLoop` method
+which will create the transport and try to initiate the underlying
+communication channel, calling you back when it succeeds.
+
+Once the communication channel is established, a transport is always
+paired with a :ref:`protocol <asyncio-protocol>` instance. The protocol can
+then call the transport's methods for various purposes.
+
+:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and
+subprocess pipes. The methods available on a transport depend on
+the transport's kind.
+
+
+BaseTransport
+-------------
+
+.. class:: BaseTransport
+
+ Base class for transports.
+
+ .. method:: close(self)
+
+ Close the transport. If the transport has a buffer for outgoing
+ data, buffered data will be flushed asynchronously. No more data
+ will be received. After all buffered data is flushed, the
+ protocol's :meth:`connection_lost` method will be called with
+ :const:`None` as its argument.
+
+
+ .. method:: get_extra_info(name, default=None)
+
+ Return optional transport information. *name* is a string representing
+ the piece of transport-specific information to get, *default* is the
+ value to return if the information doesn't exist.
+
+ This method allows transport implementations to easily expose
+ channel-specific information.
+
+ * socket:
+
+ - ``'peername'``: the remote address to which the socket is connected,
+ result of :meth:`socket.socket.getpeername` (``None`` on error)
+ - ``'socket'``: :class:`socket.socket` instance
+ - ``'sockname'``: the socket's own address,
+ result of :meth:`socket.socket.getsockname`
+
+ * SSL socket:
+
+ - ``'compression'``: the compression algorithm being used as a string,
+ or ``None`` if the connection isn't compressed; result of
+ :meth:`ssl.SSLSocket.compression`
+ - ``'cipher'``: a three-value tuple containing the name of the cipher
+ being used, the version of the SSL protocol that defines its use, and
+ the number of secret bits being used; result of
+ :meth:`ssl.SSLSocket.cipher`
+ - ``'peercert'``: peer certificate; result of
+ :meth:`ssl.SSLSocket.getpeercert`
+ - ``'sslcontext'``: :class:`ssl.SSLContext` instance
+
+ * pipe:
+
+ - ``'pipe'``: pipe object
+
+ * subprocess:
+
+ - ``'subprocess'``: :class:`subprocess.Popen` instance
+
+
+ReadTransport
+-------------
+
+.. class:: ReadTransport
+
+ Interface for read-only transports.
+
+ .. method:: pause_reading()
+
+ Pause the receiving end of the transport. No data will be passed to
+ the protocol's :meth:`data_received` method until :meth:`resume_reading`
+ is called.
+
+ .. method:: resume_reading()
+
+ Resume the receiving end. The protocol's :meth:`data_received` method
+ will be called once again if some data is available for reading.
+
+
+WriteTransport
+--------------
+
+.. class:: WriteTransport
+
+ Interface for write-only transports.
+
+ .. method:: abort()
+
+ Close the transport immediately, without waiting for pending operations
+ to complete. Buffered data will be lost. No more data will be received.
+ The protocol's :meth:`connection_lost` method will eventually be
+ called with :const:`None` as its argument.
+
+ .. method:: can_write_eof()
+
+ Return :const:`True` if the transport supports :meth:`write_eof`,
+ :const:`False` if not.
+
+ .. method:: get_write_buffer_size()
+
+ Return the current size of the output buffer used by the transport.
+
+ .. method:: set_write_buffer_limits(high=None, low=None)
+
+ Set the *high*- and *low*-water limits for write flow control.
+
+ These two values control when call the protocol's
+ :meth:`pause_writing` and :meth:`resume_writing` methods are called.
+ If specified, the low-water limit must be less than or equal to the
+ high-water limit. Neither *high* nor *low* can be negative.
+
+ The defaults are implementation-specific. If only the
+ high-water limit is given, the low-water limit defaults to a
+ implementation-specific value less than or equal to the
+ high-water limit. Setting *high* to zero forces *low* to zero as
+ well, and causes :meth:`pause_writing` to be called whenever the
+ buffer becomes non-empty. Setting *low* to zero causes
+ :meth:`resume_writing` to be called only once the buffer is empty.
+ Use of zero for either limit is generally sub-optimal as it
+ reduces opportunities for doing I/O and computation
+ concurrently.
+
+ .. method:: write(data)
+
+ Write some *data* bytes to the transport.
+
+ This method does not block; it buffers the data and arranges for it
+ to be sent out asynchronously.
+
+ .. method:: writelines(list_of_data)
+
+ Write a list (or any iterable) of data bytes to the transport.
+ This is functionally equivalent to calling :meth:`write` on each
+ element yielded by the iterable, but may be implemented more efficiently.
+
+ .. method:: write_eof()
+
+ Close the write end of the transport after flushing buffered data.
+ Data may still be received.
+
+ This method can raise :exc:`NotImplementedError` if the transport
+ (e.g. SSL) doesn't support half-closes.
+
+
+DatagramTransport
+-----------------
+
+.. method:: DatagramTransport.sendto(data, addr=None)
+
+ Send the *data* bytes to the remote peer given by *addr* (a
+ transport-dependent target address). If *addr* is :const:`None`, the
+ data is sent to the target address given on transport creation.
+
+ This method does not block; it buffers the data and arranges for it
+ to be sent out asynchronously.
+
+.. method:: DatagramTransport.abort()
+
+ Close the transport immediately, without waiting for pending operations
+ to complete. Buffered data will be lost. No more data will be received.
+ The protocol's :meth:`connection_lost` method will eventually be
+ called with :const:`None` as its argument.
+
+
+BaseSubprocessTransport
+-----------------------
+
+.. class:: BaseSubprocessTransport
+
+ .. method:: get_pid()
+
+ Return the subprocess process id as an integer.
+
+ .. method:: get_pipe_transport(fd)
+
+ Return the transport for the communication pipe corresponding to the
+ integer file descriptor *fd*. The return value can be a readable or
+ writable streaming transport, depending on the *fd*. If *fd* doesn't
+ correspond to a pipe belonging to this transport, :const:`None` is
+ returned.
+
+ .. method:: get_returncode()
+
+ Return the subprocess returncode as an integer or :const:`None`
+ if it hasn't returned, similarly to the
+ :attr:`subprocess.Popen.returncode` attribute.
+
+ .. method:: kill(self)
+
+ Kill the subprocess, as in :meth:`subprocess.Popen.kill`
+
+ On POSIX systems, the function sends SIGKILL to the subprocess.
+ On Windows, this method is an alias for :meth:`terminate`.
+
+ .. method:: send_signal(signal)
+
+ Send the *signal* number to the subprocess, as in
+ :meth:`subprocess.Popen.send_signal`.
+
+ .. method:: terminate()
+
+ Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`.
+ This method is an alias for the :meth:`close` method.
+
+ On POSIX systems, this method sends SIGTERM to the subprocess.
+ On Windows, the Windows API function TerminateProcess() is called to
+ stop the subprocess.
+
+
+.. _asyncio-protocol:
+
+Protocols
+=========
+
+:mod:`asyncio` provides base classes that you can subclass to implement
+your network protocols. Those classes are used in conjunction with
+:ref:`transports <asyncio-transport>` (see below): the protocol parses incoming
+data and asks for the writing of outgoing data, while the transport is
+responsible for the actual I/O and buffering.
+
+When subclassing a protocol class, it is recommended you override certain
+methods. Those methods are callbacks: they will be called by the transport
+on certain events (for example when some data is received); you shouldn't
+call them yourself, unless you are implementing a transport.
+
+.. note::
+ All callbacks have default implementations, which are empty. Therefore,
+ you only need to implement the callbacks for the events in which you
+ are interested.
+
+
+Protocol classes
+----------------
+
+.. class:: Protocol
+
+ The base class for implementing streaming protocols (for use with
+ e.g. TCP and SSL transports).
+
+.. class:: DatagramProtocol
+
+ The base class for implementing datagram protocols (for use with
+ e.g. UDP transports).
+
+.. class:: SubprocessProtocol
+
+ The base class for implementing protocols communicating with child
+ processes (through a set of unidirectional pipes).
+
+
+Connection callbacks
+--------------------
+
+These callbacks may be called on :class:`Protocol` and
+:class:`SubprocessProtocol` instances:
+
+.. method:: BaseProtocol.connection_made(transport)
+
+ Called when a connection is made.
+
+ The *transport* argument is the transport representing the
+ connection. You are responsible for storing it somewhere
+ (e.g. as an attribute) if you need to.
+
+.. method:: BaseProtocol.connection_lost(exc)
+
+ Called when the connection is lost or closed.
+
+ The argument is either an exception object or :const:`None`.
+ The latter means a regular EOF is received, or the connection was
+ aborted or closed by this side of the connection.
+
+:meth:`connection_made` and :meth:`connection_lost` are called exactly once
+per successful connection. All other callbacks will be called between those
+two methods, which allows for easier resource management in your protocol
+implementation.
+
+The following callbacks may be called only on :class:`SubprocessProtocol`
+instances:
+
+.. method:: SubprocessProtocol.pipe_data_received(fd, data)
+
+ Called when the child process writes data into its stdout or stderr pipe.
+ *fd* is the integer file descriptor of the pipe. *data* is a non-empty
+ bytes object containing the data.
+
+.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc)
+
+ Called when one of the pipes communicating with the child process
+ is closed. *fd* is the integer file descriptor that was closed.
+
+.. method:: SubprocessProtocol.process_exited()
+
+ Called when the child process has exited.
+
+
+Streaming protocols
+-------------------
+
+The following callbacks are called on :class:`Protocol` instances:
+
+.. method:: Protocol.data_received(data)
+
+ Called when some data is received. *data* is a non-empty bytes object
+ containing the incoming data.
+
+ .. note::
+ Whether the data is buffered, chunked or reassembled depends on
+ the transport. In general, you shouldn't rely on specific semantics
+ and instead make your parsing generic and flexible enough. However,
+ data is always received in the correct order.
+
+.. method:: Protocol.eof_received()
+
+ Calls when the other end signals it won't send any more data
+ (for example by calling :meth:`write_eof`, if the other end also uses
+ asyncio).
+
+ This method may return a false value (including None), in which case
+ the transport will close itself. Conversely, if this method returns a
+ true value, closing the transport is up to the protocol. Since the
+ default implementation returns None, it implicitly closes the connection.
+
+ .. note::
+ Some transports such as SSL don't support half-closed connections,
+ in which case returning true from this method will not prevent closing
+ the connection.
+
+:meth:`data_received` can be called an arbitrary number of times during
+a connection. However, :meth:`eof_received` is called at most once
+and, if called, :meth:`data_received` won't be called after it.
+
+Datagram protocols
+------------------
+
+The following callbacks are called on :class:`DatagramProtocol` instances.
+
+.. method:: DatagramProtocol.datagram_received(data, addr)
+
+ Called when a datagram is received. *data* is a bytes object containing
+ the incoming data. *addr* is the address of the peer sending the data;
+ the exact format depends on the transport.
+
+.. method:: DatagramProtocol.error_received(exc)
+
+ Called when a previous send or receive operation raises an
+ :class:`OSError`. *exc* is the :class:`OSError` instance.
+
+ This method is called in rare conditions, when the transport (e.g. UDP)
+ detects that a datagram couldn't be delivered to its recipient.
+ In many conditions though, undeliverable datagrams will be silently
+ dropped.
+
+
+Flow control callbacks
+----------------------
+
+These callbacks may be called on :class:`Protocol`,
+:class:`DatagramProtocol` and :class:`SubprocessProtocol` instances:
+
+.. method:: BaseProtocol.pause_writing()
+
+ Called when the transport's buffer goes over the high-water mark.
+
+.. method:: BaseProtocol.resume_writing()
+
+ Called when the transport's buffer drains below the low-water mark.
+
+
+:meth:`pause_writing` and :meth:`resume_writing` calls are paired --
+:meth:`pause_writing` is called once when the buffer goes strictly over
+the high-water mark (even if subsequent writes increases the buffer size
+even more), and eventually :meth:`resume_writing` is called once when the
+buffer size reaches the low-water mark.
+
+.. note::
+ If the buffer size equals the high-water mark,
+ :meth:`pause_writing` is not called -- it must go strictly over.
+ Conversely, :meth:`resume_writing` is called when the buffer size is
+ equal or lower than the low-water mark. These end conditions
+ are important to ensure that things go as expected when either
+ mark is zero.
+
+.. note::
+ On BSD systems (OS X, FreeBSD, etc.) flow control is not supported
+ for :class:`DatagramProtocol`, because send failures caused by
+ writing too many packets cannot be detected easily. The socket
+ always appears 'ready' and excess packets are dropped; an
+ :class:`OSError` with errno set to :const:`errno.ENOBUFS` may or
+ may not be raised; if it is raised, it will be reported to
+ :meth:`DatagramProtocol.error_received` but otherwise ignored.
+
+
+Coroutines and protocols
+------------------------
+
+Coroutines can be scheduled in a protocol method using :func:`async`, but there
+is not guarantee on the execution order. Protocols are not aware of coroutines
+created in protocol methods and so will not wait for them.
+
+To have a reliable execution order, use :ref:`stream objects <asyncio-streams>` in a
+coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain`
+coroutine can be used to wait until the write buffer is flushed.
+
+
+Protocol example: TCP echo server and client
+============================================
+
+Echo client
+-----------
+
+TCP echo client example, send data and wait until the connection is closed::
+
+ import asyncio
+
+ class EchoClient(asyncio.Protocol):
+ message = 'This is the message. It will be echoed.'
+
+ def connection_made(self, transport):
+ transport.write(self.message.encode())
+ print('data sent: {}'.format(self.message))
+
+ def data_received(self, data):
+ print('data received: {}'.format(data.decode()))
+
+ def connection_lost(self, exc):
+ print('server closed the connection')
+ asyncio.get_event_loop().stop()
+
+ loop = asyncio.get_event_loop()
+ coro = loop.create_connection(EchoClient, '127.0.0.1', 8888)
+ loop.run_until_complete(coro)
+ loop.run_forever()
+ loop.close()
+
+The event loop is running twice. The
+:meth:`~BaseEventLoop.run_until_complete` method is preferred in this short
+example to raise an exception if the server is not listening, instead of
+having to write a short coroutine to handle the exception and stop the
+running loop. At :meth:`~BaseEventLoop.run_until_complete` exit, the loop is
+no more running, so there is no need to stop the loop in case of an error.
+
+Echo server
+-----------
+
+TCP echo server example, send back received data and close the connection::
+
+ import asyncio
+
+ class EchoServer(asyncio.Protocol):
+ def connection_made(self, transport):
+ peername = transport.get_extra_info('peername')
+ print('connection from {}'.format(peername))
+ self.transport = transport
+
+ def data_received(self, data):
+ print('data received: {}'.format(data.decode()))
+ self.transport.write(data)
+
+ # close the socket
+ self.transport.close()
+
+ loop = asyncio.get_event_loop()
+ coro = loop.create_server(EchoServer, '127.0.0.1', 8888)
+ server = loop.run_until_complete(coro)
+ print('serving on {}'.format(server.sockets[0].getsockname()))
+
+ try:
+ loop.run_forever()
+ except KeyboardInterrupt:
+ print("exit")
+ finally:
+ server.close()
+ loop.close()
+
+:meth:`Transport.close` can be called immediately after
+:meth:`WriteTransport.write` even if data are not sent yet on the socket: both
+methods are asynchronous. ``yield from`` is not needed because these transport
+methods are not coroutines.
+
+
diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst
new file mode 100644
index 0000000..4543af4
--- /dev/null
+++ b/Doc/library/asyncio-stream.rst
@@ -0,0 +1,278 @@
+.. currentmodule:: asyncio
+
+.. _asyncio-streams:
+
+++++++++++++++++++++++++
+Streams (high-level API)
+++++++++++++++++++++++++
+
+Stream functions
+================
+
+.. function:: open_connection(host=None, port=None, \*, loop=None, limit=None, **kwds)
+
+ A wrapper for :meth:`~BaseEventLoop.create_connection()` returning a (reader,
+ writer) pair.
+
+ The reader returned is a :class:`StreamReader` instance; the writer is
+ a :class:`StreamWriter` instance.
+
+ The arguments are all the usual arguments to
+ :meth:`BaseEventLoop.create_connection` except *protocol_factory*; most
+ common are positional host and port, with various optional keyword arguments
+ following.
+
+ Additional optional keyword arguments are *loop* (to set the event loop
+ instance to use) and *limit* (to set the buffer limit passed to the
+ :class:`StreamReader`).
+
+ (If you want to customize the :class:`StreamReader` and/or
+ :class:`StreamReaderProtocol` classes, just copy the code -- there's really
+ nothing special here except some convenience.)
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+.. function:: start_server(client_connected_cb, host=None, port=None, \*, loop=None, limit=None, **kwds)
+
+ Start a socket server, with a callback for each client connected.
+
+ The first parameter, *client_connected_cb*, takes two parameters:
+ *client_reader*, *client_writer*. *client_reader* is a
+ :class:`StreamReader` object, while *client_writer* is a
+ :class:`StreamWriter` object. This parameter can either be a plain callback
+ function or a :ref:`coroutine function <coroutine>`; if it is a coroutine
+ function, it will be automatically converted into a :class:`Task`.
+
+ The rest of the arguments are all the usual arguments to
+ :meth:`~BaseEventLoop.create_server()` except *protocol_factory*; most
+ common are positional host and port, with various optional keyword arguments
+ following. The return value is the same as
+ :meth:`~BaseEventLoop.create_server()`.
+
+ Additional optional keyword arguments are *loop* (to set the event loop
+ instance to use) and *limit* (to set the buffer limit passed to the
+ :class:`StreamReader`).
+
+ The return value is the same as :meth:`~BaseEventLoop.create_server()`, i.e.
+ a :class:`AbstractServer` object which can be used to stop the service.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+.. function:: open_unix_connection(path=None, \*, loop=None, limit=None, **kwds)
+
+ A wrapper for :meth:`~BaseEventLoop.create_unix_connection()` returning
+ a (reader, writer) pair.
+
+ See :func:`open_connection` for information about return value and other
+ details.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+ Availability: UNIX.
+
+.. function:: start_unix_server(client_connected_cb, path=None, \*, loop=None, limit=None, **kwds)
+
+ Start a UNIX Domain Socket server, with a callback for each client connected.
+
+ See :func:`start_server` for information about return value and other
+ details.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+ Availability: UNIX.
+
+
+StreamReader
+============
+
+.. class:: StreamReader(limit=None, loop=None)
+
+ .. method:: exception()
+
+ Get the exception.
+
+ .. method:: feed_eof()
+
+ Acknowledge the EOF.
+
+ .. method:: feed_data(data)
+
+ Feed *data* bytes in the internal buffer. Any operations waiting
+ for the data will be resumed.
+
+ .. method:: set_exception(exc)
+
+ Set the exception.
+
+ .. method:: set_transport(transport)
+
+ Set the transport.
+
+ .. method:: read(n=-1)
+
+ Read up to *n* bytes. If *n* is not provided, or set to ``-1``,
+ read until EOF and return all read bytes.
+
+ If the EOF was received and the internal buffer is empty,
+ return an empty ``bytes`` object.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: readline()
+
+ Read one line, where "line" is a sequence of bytes ending with ``\n``.
+
+ If EOF is received, and ``\n`` was not found, the method will
+ return the partial read bytes.
+
+ If the EOF was received and the internal buffer is empty,
+ return an empty ``bytes`` object.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: readexactly(n)
+
+ Read exactly *n* bytes. Raise an :exc:`IncompleteReadError` if the end of
+ the stream is reached before *n* can be read, the
+ :attr:`IncompleteReadError.partial` attribute of the exception contains
+ the partial read bytes.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: at_eof()
+
+ Return ``True`` if the buffer is empty and :meth:`feed_eof` was called.
+
+
+StreamWriter
+============
+
+.. class:: StreamWriter(transport, protocol, reader, loop)
+
+ Wraps a Transport.
+
+ This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`,
+ :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds
+ :meth:`drain` which returns an optional :class:`Future` on which you can
+ wait for flow control. It also adds a transport attribute which references
+ the :class:`Transport` directly.
+
+ .. attribute:: transport
+
+ Transport.
+
+ .. method:: can_write_eof()
+
+ Return :const:`True` if the transport supports :meth:`write_eof`,
+ :const:`False` if not. See :meth:`WriteTransport.can_write_eof`.
+
+ .. method:: close()
+
+ Close the transport: see :meth:`BaseTransport.close`.
+
+ .. method:: drain()
+
+ Wait until the write buffer of the underlying transport is flushed.
+
+ This method has an unusual return value. The intended use is to write::
+
+ w.write(data)
+ yield from w.drain()
+
+ When there's nothing to wait for, :meth:`drain()` returns ``()``, and the
+ yield-from continues immediately. When the transport buffer is full (the
+ protocol is paused), :meth:`drain` creates and returns a
+ :class:`Future` and the yield-from will block until
+ that Future is completed, which will happen when the buffer is
+ (partially) drained and the protocol is resumed.
+
+ .. method:: get_extra_info(name, default=None)
+
+ Return optional transport information: see
+ :meth:`BaseTransport.get_extra_info`.
+
+ .. method:: write(data)
+
+ Write some *data* bytes to the transport: see
+ :meth:`WriteTransport.write`.
+
+ .. method:: writelines(data)
+
+ Write a list (or any iterable) of data bytes to the transport:
+ see :meth:`WriteTransport.writelines`.
+
+ .. method:: write_eof()
+
+ Close the write end of the transport after flushing buffered data:
+ see :meth:`WriteTransport.write_eof`.
+
+
+StreamReaderProtocol
+====================
+
+.. class:: StreamReaderProtocol(stream_reader, client_connected_cb=None, loop=None)
+
+ Trivial helper class to adapt between :class:`Protocol` and
+ :class:`StreamReader`. Sublclass of :class:`Protocol`.
+
+ *stream_reader* is a :class:`StreamReader` instance, *client_connected_cb*
+ is an optional function called with (stream_reader, stream_writer) when a
+ connection is made, *loop* is the event loop instance to use.
+
+ (This is a helper class instead of making :class:`StreamReader` itself a
+ :class:`Protocol` subclass, because the :class:`StreamReader` has other
+ potential uses, and to prevent the user of the :class:`StreamReader` to
+ accidentally call inappropriate methods of the protocol.)
+
+
+IncompleteReadError
+===================
+
+.. exception:: IncompleteReadError
+
+ Incomplete read error, subclass of :exc:`EOFError`.
+
+ .. attribute:: expected
+
+ Total number of expected bytes (:class:`int`).
+
+ .. attribute:: partial
+
+ Read bytes string before the end of stream was reached (:class:`bytes`).
+
+
+Example
+=======
+
+Simple example querying HTTP headers of the URL passed on the command line::
+
+ import asyncio
+ import urllib.parse
+ import sys
+
+ @asyncio.coroutine
+ def print_http_headers(url):
+ url = urllib.parse.urlsplit(url)
+ reader, writer = yield from asyncio.open_connection(url.hostname, 80)
+ query = ('HEAD {url.path} HTTP/1.0\r\n'
+ 'Host: {url.hostname}\r\n'
+ '\r\n').format(url=url)
+ writer.write(query.encode('latin-1'))
+ while True:
+ line = yield from reader.readline()
+ if not line:
+ break
+ line = line.decode('latin1').rstrip()
+ if line:
+ print('HTTP header> %s' % line)
+
+ url = sys.argv[1]
+ loop = asyncio.get_event_loop()
+ task = asyncio.async(print_http_headers(url))
+ loop.run_until_complete(task)
+ loop.close()
+
+Usage::
+
+ python example.py http://example.com/path/page.html
+
diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst
new file mode 100644
index 0000000..b387543
--- /dev/null
+++ b/Doc/library/asyncio-subprocess.rst
@@ -0,0 +1,270 @@
+.. currentmodule:: asyncio
+
+Subprocess
+==========
+
+Operating system support
+------------------------
+
+On Windows, the default event loop uses :class:`selectors.SelectSelector`
+which only supports sockets. The :class:`ProactorEventLoop` should be used to
+support subprocesses.
+
+On Mac OS X older than 10.9 (Mavericks), :class:`selectors.KqueueSelector`
+does not support character devices like PTY, whereas it is used by the
+default event loop. The :class:`SelectorEventLoop` can be used with
+:class:`SelectSelector` or :class:`PollSelector` to handle character
+devices on Mac OS X 10.6 (Snow Leopard) and later.
+
+
+Create a subprocess: high-level API using Process
+-------------------------------------------------
+
+.. function:: create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds)
+
+ Run the shell command *cmd* given as a string. Return a :class:`~asyncio.subprocess.Process`
+ instance.
+
+ The optional *limit* parameter sets the buffer limit passed to the
+ :class:`StreamReader`.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+.. function:: create_subprocess_exec(\*args, stdin=None, stdout=None, stderr=None, loop=None, limit=None, \*\*kwds)
+
+ Create a subprocess. Return a :class:`~asyncio.subprocess.Process` instance.
+
+ The optional *limit* parameter sets the buffer limit passed to the
+ :class:`StreamReader`.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+Use the :meth:`BaseEventLoop.connect_read_pipe` and
+:meth:`BaseEventLoop.connect_write_pipe` methods to connect pipes.
+
+
+Create a subprocess: low-level API using subprocess.Popen
+---------------------------------------------------------
+
+Run subprocesses asynchronously using the :mod:`subprocess` module.
+
+.. method:: BaseEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs)
+
+ Create a subprocess from one or more string arguments, where the first string
+ specifies the program to execute, and the remaining strings specify the
+ program's arguments. (Thus, together the string arguments form the
+ ``sys.argv`` value of the program, assuming it is a Python script.) This is
+ similar to the standard library :class:`subprocess.Popen` class called with
+ shell=False and the list of strings passed as the first argument;
+ however, where :class:`~subprocess.Popen` takes a single argument which is
+ list of strings, :func:`subprocess_exec` takes multiple string arguments.
+
+ Other parameters:
+
+ * *stdin*: Either a file-like object representing the pipe to be connected
+ to the subprocess's standard input stream using
+ :meth:`~BaseEventLoop.connect_write_pipe`, or the constant
+ :const:`subprocess.PIPE` (the default). By default a new pipe will be
+ created and connected.
+
+ * *stdout*: Either a file-like object representing the pipe to be connected
+ to the subprocess's standard output stream using
+ :meth:`~BaseEventLoop.connect_read_pipe`, or the constant
+ :const:`subprocess.PIPE` (the default). By default a new pipe will be
+ created and connected.
+
+ * *stderr*: Either a file-like object representing the pipe to be connected
+ to the subprocess's standard error stream using
+ :meth:`~BaseEventLoop.connect_read_pipe`, or one of the constants
+ :const:`subprocess.PIPE` (the default) or :const:`subprocess.STDOUT`.
+ By default a new pipe will be created and connected. When
+ :const:`subprocess.STDOUT` is specified, the subprocess's standard error
+ stream will be connected to the same pipe as the standard output stream.
+
+ * All other keyword arguments are passed to :class:`subprocess.Popen`
+ without interpretation, except for *bufsize*, *universal_newlines* and
+ *shell*, which should not be specified at all.
+
+ Returns a pair of ``(transport, protocol)``, where *transport* is an
+ instance of :class:`BaseSubprocessTransport`.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ See the constructor of the :class:`subprocess.Popen` class for parameters.
+
+.. method:: BaseEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \*\*kwargs)
+
+ Create a subprocess from *cmd*, which is a string using the platform's
+ "shell" syntax. This is similar to the standard library
+ :class:`subprocess.Popen` class called with ``shell=True``.
+
+ See :meth:`~BaseEventLoop.subprocess_exec` for more details about
+ the remaining arguments.
+
+ Returns a pair of ``(transport, protocol)``, where *transport* is an
+ instance of :class:`BaseSubprocessTransport`.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ See the constructor of the :class:`subprocess.Popen` class for parameters.
+
+.. seealso::
+
+ The :meth:`BaseEventLoop.connect_read_pipe` and
+ :meth:`BaseEventLoop.connect_write_pipe` methods.
+
+
+Constants
+---------
+
+.. data:: asyncio.subprocess.PIPE
+
+ Special value that can be used as the *stdin*, *stdout* or *stderr* argument
+ to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and
+ indicates that a pipe to the standard stream should be opened.
+
+.. data:: asyncio.subprocess.STDOUT
+
+ Special value that can be used as the *stderr* argument to
+ :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and
+ indicates that standard error should go into the same handle as standard
+ output.
+
+.. data:: asyncio.subprocess.DEVNULL
+
+ Special value that can be used as the *stdin*, *stdout* or *stderr* argument
+ to :func:`create_subprocess_shell` and :func:`create_subprocess_exec` and
+ indicates that the special file :data:`os.devnull` will be used.
+
+
+Process
+-------
+
+.. class:: asyncio.subprocess.Process
+
+ .. attribute:: pid
+
+ The identifier of the process.
+
+ Note that if you set the *shell* argument to ``True``, this is the
+ process identifier of the spawned shell.
+
+ .. attribute:: returncode
+
+ Return code of the process when it exited. A ``None`` value indicates
+ that the process has not terminated yet.
+
+ A negative value ``-N`` indicates that the child was terminated by signal
+ ``N`` (Unix only).
+
+ .. attribute:: stdin
+
+ Standard input stream (write), ``None`` if the process was created with
+ ``stdin=None``.
+
+ .. attribute:: stdout
+
+ Standard output stream (read), ``None`` if the process was created with
+ ``stdout=None``.
+
+ .. attribute:: stderr
+
+ Standard error stream (read), ``None`` if the process was created with
+ ``stderr=None``.
+
+ .. method:: communicate(input=None)
+
+ Interact with process: Send data to stdin. Read data from stdout and
+ stderr, until end-of-file is reached. Wait for process to terminate.
+ The optional *input* argument should be data to be sent to the child
+ process, or ``None``, if no data should be sent to the child. The type
+ of *input* must be bytes.
+
+ :meth:`communicate` returns a tuple ``(stdoutdata, stderrdata)``.
+
+ Note that if you want to send data to the process's stdin, you need to
+ create the Process object with ``stdin=PIPE``. Similarly, to get anything
+ other than ``None`` in the result tuple, you need to give ``stdout=PIPE``
+ and/or ``stderr=PIPE`` too.
+
+ .. note::
+
+ The data read is buffered in memory, so do not use this method if the
+ data size is large or unlimited.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: kill()
+
+ Kills the child. On Posix OSs the function sends :py:data:`SIGKILL` to
+ the child. On Windows :meth:`kill` is an alias for :meth:`terminate`.
+
+ .. method:: send_signal(signal)
+
+ Sends the signal *signal* to the child process.
+
+ .. note::
+
+ On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`.
+ ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes
+ started with a *creationflags* parameter which includes
+ ``CREATE_NEW_PROCESS_GROUP``.
+
+ .. method:: terminate()
+
+ Stop the child. On Posix OSs the method sends :py:data:`signal.SIGTERM`
+ to the child. On Windows the Win32 API function
+ :c:func:`TerminateProcess` is called to stop the child.
+
+ .. method:: wait():
+
+ Wait for child process to terminate. Set and return :attr:`returncode`
+ attribute.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+
+Example
+-------
+
+Implement a function similar to :func:`subprocess.getstatusoutput`, except that
+it does not use a shell. Get the output of the "python -m platform" command and
+display the output::
+
+ import asyncio
+ import os
+ import sys
+ from asyncio import subprocess
+
+ @asyncio.coroutine
+ def getstatusoutput(*args):
+ proc = yield from asyncio.create_subprocess_exec(
+ *args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ try:
+ stdout, _ = yield from proc.communicate()
+ except:
+ proc.kill()
+ yield from proc.wait()
+ raise
+ exitcode = yield from proc.wait()
+ return (exitcode, stdout)
+
+ if os.name == 'nt':
+ loop = asyncio.ProactorEventLoop()
+ asyncio.set_event_loop(loop)
+ else:
+ loop = asyncio.get_event_loop()
+ coro = getstatusoutput(sys.executable, '-m', 'platform')
+ exitcode, stdout = loop.run_until_complete(coro)
+ if not exitcode:
+ stdout = stdout.decode('ascii').rstrip()
+ print("Platform: %s" % stdout)
+ else:
+ print("Python failed with exit code %s:" % exitcode)
+ sys.stdout.flush()
+ sys.stdout.buffer.flush()
+ sys.stdout.buffer.write(stdout)
+ sys.stdout.buffer.flush()
+ loop.close()
diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst
new file mode 100644
index 0000000..a299f09
--- /dev/null
+++ b/Doc/library/asyncio-sync.rst
@@ -0,0 +1,410 @@
+.. currentmodule:: asyncio
+.. _asyncio-sync:
+
+Synchronization primitives
+==========================
+
+Locks
+-----
+
+Lock
+^^^^
+
+.. class:: Lock(\*, loop=None)
+
+ Primitive lock objects.
+
+ A primitive lock is a synchronization primitive that is not owned by a
+ particular coroutine when locked. A primitive lock is in one of two states,
+ 'locked' or 'unlocked'.
+
+ It is created in the unlocked state. It has two basic methods, :meth:`acquire`
+ and :meth:`release`. When the state is unlocked, acquire() changes the state to
+ locked and returns immediately. When the state is locked, acquire() blocks
+ until a call to release() in another coroutine changes it to unlocked, then
+ the acquire() call resets it to locked and returns. The release() method
+ should only be called in the locked state; it changes the state to unlocked
+ and returns immediately. If an attempt is made to release an unlocked lock,
+ a :exc:`RuntimeError` will be raised.
+
+ When more than one coroutine is blocked in acquire() waiting for the state
+ to turn to unlocked, only one coroutine proceeds when a release() call
+ resets the state to unlocked; first coroutine which is blocked in acquire()
+ is being processed.
+
+ :meth:`acquire` is a coroutine and should be called with ``yield from``.
+
+ Locks also support the context manager protocol. ``(yield from lock)``
+ should be used as context manager expression.
+
+ Usage::
+
+ lock = Lock()
+ ...
+ yield from lock
+ try:
+ ...
+ finally:
+ lock.release()
+
+ Context manager usage::
+
+ lock = Lock()
+ ...
+ with (yield from lock):
+ ...
+
+ Lock objects can be tested for locking state::
+
+ if not lock.locked():
+ yield from lock
+ else:
+ # lock is acquired
+ ...
+
+ .. method:: locked()
+
+ Return ``True`` if the lock is acquired.
+
+ .. method:: acquire()
+
+ Acquire a lock.
+
+ This method blocks until the lock is unlocked, then sets it to locked and
+ returns ``True``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: release()
+
+ Release a lock.
+
+ When the lock is locked, reset it to unlocked, and return. If any other
+ coroutines are blocked waiting for the lock to become unlocked, allow
+ exactly one of them to proceed.
+
+ When invoked on an unlocked lock, a :exc:`RuntimeError` is raised.
+
+ There is no return value.
+
+
+Event
+^^^^^
+
+.. class:: Event(\*, loop=None)
+
+ An Event implementation, asynchronous equivalent to :class:`threading.Event`.
+
+ Class implementing event objects. An event manages a flag that can be set to
+ true with the :meth:`set` method and reset to false with the :meth:`clear`
+ method. The :meth:`wait` method blocks until the flag is true. The flag is
+ initially false.
+
+ .. method:: clear()
+
+ Reset the internal flag to false. Subsequently, coroutines calling
+ :meth:`wait` will block until :meth:`set` is called to set the internal
+ flag to true again.
+
+ .. method:: is_set()
+
+ Return ``True`` if and only if the internal flag is true.
+
+ .. method:: set()
+
+ Set the internal flag to true. All coroutines waiting for it to become
+ true are awakened. Coroutine that call :meth:`wait` once the flag is true
+ will not block at all.
+
+ .. method:: wait()
+
+ Block until the internal flag is true.
+
+ If the internal flag is true on entry, return ``True`` immediately.
+ Otherwise, block until another coroutine calls :meth:`set` to set the
+ flag to true, then return ``True``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+
+Condition
+^^^^^^^^^
+
+.. class:: Condition(\*, loop=None)
+
+ A Condition implementation, asynchronous equivalent to
+ :class:`threading.Condition`.
+
+ This class implements condition variable objects. A condition variable
+ allows one or more coroutines to wait until they are notified by another
+ coroutine.
+
+ A new :class:`Lock` object is created and used as the underlying lock.
+
+ .. method:: acquire()
+
+ Acquire the underlying lock.
+
+ This method blocks until the lock is unlocked, then sets it to locked and
+ returns ``True``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: notify(n=1)
+
+ By default, wake up one coroutine waiting on this condition, if any.
+ If the calling coroutine has not acquired the lock when this method is
+ called, a :exc:`RuntimeError` is raised.
+
+ This method wakes up at most *n* of the coroutines waiting for the
+ condition variable; it is a no-op if no coroutines are waiting.
+
+ .. note::
+
+ An awakened coroutine does not actually return from its :meth:`wait`
+ call until it can reacquire the lock. Since :meth:`notify` does not
+ release the lock, its caller should.
+
+ .. method:: locked()
+
+ Return ``True`` if the underlying lock is acquired.
+
+ .. method:: notify_all()
+
+ Wake up all threads waiting on this condition. This method acts like
+ :meth:`notify`, but wakes up all waiting threads instead of one. If the
+ calling thread has not acquired the lock when this method is called, a
+ :exc:`RuntimeError` is raised.
+
+ .. method:: release()
+
+ Release the underlying lock.
+
+ When the lock is locked, reset it to unlocked, and return. If any other
+ coroutines are blocked waiting for the lock to become unlocked, allow
+ exactly one of them to proceed.
+
+ When invoked on an unlocked lock, a :exc:`RuntimeError` is raised.
+
+ There is no return value.
+
+ .. method:: wait()
+
+ Wait until notified.
+
+ If the calling coroutine has not acquired the lock when this method is
+ called, a :exc:`RuntimeError` is raised.
+
+ This method releases the underlying lock, and then blocks until it is
+ awakened by a :meth:`notify` or :meth:`notify_all` call for the same
+ condition variable in another coroutine. Once awakened, it re-acquires
+ the lock and returns ``True``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: wait_for(predicate)
+
+ Wait until a predicate becomes true.
+
+ The predicate should be a callable which result will be interpreted as a
+ boolean value. The final predicate value is the return value.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+
+Semaphores
+----------
+
+Semaphore
+^^^^^^^^^
+
+.. class:: Semaphore(value=1, \*, loop=None)
+
+ A Semaphore implementation.
+
+ A semaphore manages an internal counter which is decremented by each
+ :meth:`acquire` call and incremented by each :meth:`release` call. The
+ counter can never go below zero; when :meth:`acquire` finds that it is zero,
+ it blocks, waiting until some other thread calls :meth:`release`.
+
+ Semaphores also support the context manager protocol.
+
+ The optional argument gives the initial value for the internal counter; it
+ defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError`
+ is raised.
+
+ .. method:: acquire()
+
+ Acquire a semaphore.
+
+ If the internal counter is larger than zero on entry, decrement it by one
+ and return ``True`` immediately. If it is zero on entry, block, waiting
+ until some other coroutine has called :meth:`release` to make it larger
+ than ``0``, and then return ``True``.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: locked()
+
+ Returns ``True`` if semaphore can not be acquired immediately.
+
+ .. method:: release()
+
+ Release a semaphore, incrementing the internal counter by one. When it
+ was zero on entry and another coroutine is waiting for it to become
+ larger than zero again, wake up that coroutine.
+
+
+BoundedSemaphore
+^^^^^^^^^^^^^^^^
+
+.. class:: BoundedSemaphore(value=1, \*, loop=None)
+
+ A bounded semaphore implementation. Inherit from :class:`Semaphore`.
+
+ This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would
+ increase the value above the initial value.
+
+
+Queues
+------
+
+Queue
+^^^^^
+
+.. class:: Queue(maxsize=0, \*, loop=None)
+
+ A queue, useful for coordinating producer and consumer coroutines.
+
+ If *maxsize* is less than or equal to zero, the queue size is infinite. If
+ it is an integer greater than ``0``, then ``yield from put()`` will block
+ when the queue reaches *maxsize*, until an item is removed by :meth:`get`.
+
+ Unlike the standard library :mod:`queue`, you can reliably know this Queue's
+ size with :meth:`qsize`, since your single-threaded asyncio application won't
+ be interrupted between calling :meth:`qsize` and doing an operation on the
+ Queue.
+
+ .. method:: empty()
+
+ Return ``True`` if the queue is empty, ``False`` otherwise.
+
+ .. method:: full()
+
+ Return ``True`` if there are maxsize items in the queue.
+
+ .. note::
+
+ If the Queue was initialized with ``maxsize=0`` (the default), then
+ :meth:`full()` is never ``True``.
+
+ .. method:: get()
+
+ Remove and return an item from the queue.
+
+ If you yield from :meth:`get()`, wait until a item is available.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: get_nowait()
+
+ Remove and return an item from the queue.
+
+ Return an item if one is immediately available, else raise
+ :exc:`QueueEmpty`.
+
+ .. method:: put(item)
+
+ Put an item into the queue.
+
+ If you yield from ``put()``, wait until a free slot is available before
+ adding item.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: put_nowait(item)
+
+ Put an item into the queue without blocking.
+
+ If no free slot is immediately available, raise :exc:`QueueFull`.
+
+ .. method:: qsize()
+
+ Number of items in the queue.
+
+ .. attribute:: maxsize
+
+ Number of items allowed in the queue.
+
+
+PriorityQueue
+^^^^^^^^^^^^^
+
+.. class:: PriorityQueue
+
+ A subclass of :class:`Queue`; retrieves entries in priority order (lowest
+ first).
+
+ Entries are typically tuples of the form: (priority number, data).
+
+
+LifoQueue
+^^^^^^^^^
+
+.. class:: LifoQueue
+
+ A subclass of :class:`Queue` that retrieves most recently added entries
+ first.
+
+
+JoinableQueue
+^^^^^^^^^^^^^
+
+.. class:: JoinableQueue
+
+ A subclass of :class:`Queue` with :meth:`task_done` and :meth:`join`
+ methods.
+
+ .. method:: join()
+
+ Block until all items in the queue have been gotten and processed.
+
+ The count of unfinished tasks goes up whenever an item is added to the
+ queue. The count goes down whenever a consumer thread calls
+ :meth:`task_done` to indicate that the item was retrieved and all work on
+ it is complete. When the count of unfinished tasks drops to zero,
+ :meth:`join` unblocks.
+
+ This method is a :ref:`coroutine <coroutine>`.
+
+ .. method:: task_done()
+
+ Indicate that a formerly enqueued task is complete.
+
+ Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a
+ subsequent call to :meth:`task_done` tells the queue that the processing
+ on the task is complete.
+
+ If a :meth:`join` is currently blocking, it will resume when all items
+ have been processed (meaning that a :meth:`task_done` call was received
+ for every item that had been :meth:`~Queue.put` into the queue).
+
+ Raises :exc:`ValueError` if called more times than there were items
+ placed in the queue.
+
+
+Exceptions
+^^^^^^^^^^
+
+.. exception:: QueueEmpty
+
+ Exception raised when non-blocking :meth:`~Queue.get` (or
+ :meth:`~Queue.get_nowait`) is called
+ on a :class:`Queue` object which is empty.
+
+
+.. exception:: QueueFull
+
+ Exception raised when non-blocking :meth:`~Queue.put` (or
+ :meth:`~Queue.put_nowait`) is called
+ on a :class:`Queue` object which is full.
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
new file mode 100644
index 0000000..4e5526e
--- /dev/null
+++ b/Doc/library/asyncio-task.rst
@@ -0,0 +1,545 @@
+.. currentmodule:: asyncio
+
+Tasks and coroutines
+====================
+
+.. _coroutine:
+
+Coroutines
+----------
+
+A coroutine is a generator that follows certain conventions. For
+documentation purposes, all coroutines should be decorated with
+``@asyncio.coroutine``, but this cannot be strictly enforced.
+
+Coroutines use the ``yield from`` syntax introduced in :pep:`380`,
+instead of the original ``yield`` syntax.
+
+The word "coroutine", like the word "generator", is used for two
+different (though related) concepts:
+
+- The function that defines a coroutine (a function definition
+ decorated with ``@asyncio.coroutine``). If disambiguation is needed
+ we will call this a *coroutine function* (:func:`iscoroutinefunction`
+ returns ``True``).
+
+- The object obtained by calling a coroutine function. This object
+ represents a computation or an I/O operation (usually a combination)
+ that will complete eventually. If disambiguation is needed we will
+ call it a *coroutine object* (:func:`iscoroutine` returns ``True``).
+
+Things a coroutine can do:
+
+- ``result = yield from future`` -- suspends the coroutine until the
+ future is done, then returns the future's result, or raises an
+ exception, which will be propagated. (If the future is cancelled,
+ it will raise a ``CancelledError`` exception.) Note that tasks are
+ futures, and everything said about futures also applies to tasks.
+
+- ``result = yield from coroutine`` -- wait for another coroutine to
+ produce a result (or raise an exception, which will be propagated).
+ The ``coroutine`` expression must be a *call* to another coroutine.
+
+- ``return expression`` -- produce a result to the coroutine that is
+ waiting for this one using ``yield from``.
+
+- ``raise exception`` -- raise an exception in the coroutine that is
+ waiting for this one using ``yield from``.
+
+Calling a coroutine does not start its code running -- it is just a
+generator, and the coroutine object returned by the call is really a
+generator object, which doesn't do anything until you iterate over it.
+In the case of a coroutine object, there are two basic ways to start
+it running: call ``yield from coroutine`` from another coroutine
+(assuming the other coroutine is already running!), or convert it to a
+:class:`Task`.
+
+Coroutines (and tasks) can only run when the event loop is running.
+
+.. decorator:: coroutine
+
+ Decorator to mark coroutines.
+
+ If the coroutine is not yielded from before it is destroyed, an error
+ message is logged. See :ref:`Detect coroutines never scheduled
+ <asyncio-coroutine-not-scheduled>`.
+
+.. note::
+
+ In this documentation, some methods are documented as coroutines,
+ even if they are plain Python functions returning a :class:`Future`.
+ This is intentional to have a freedom of tweaking the implementation
+ of these functions in the future. If such a function is needed to be
+ used in a callback-style code, wrap its result with :func:`async`.
+
+
+.. _asyncio-hello-world-coroutine:
+
+Example: "Hello World" coroutine
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Print ``"Hello World"`` every two seconds using a coroutine::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def greet_every_two_seconds():
+ while True:
+ print('Hello World')
+ yield from asyncio.sleep(2)
+
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(greet_every_two_seconds())
+
+.. seealso::
+
+ :ref:`Hello World example using a callback <asyncio-hello-world-callback>`.
+
+
+Example: Chain coroutines
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example chaining coroutines::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def compute(x, y):
+ print("Compute %s + %s ..." % (x, y))
+ yield from asyncio.sleep(1.0)
+ return x + y
+
+ @asyncio.coroutine
+ def print_sum(x, y):
+ result = yield from compute(x, y)
+ print("%s + %s = %s" % (x, y, result))
+
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(print_sum(1, 2))
+ loop.close()
+
+``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits
+until ``compute()`` is completed before returning its result.
+
+Sequence diagram of the example:
+
+.. image:: tulip_coro.png
+ :align: center
+
+The "Task" is created by the :meth:`BaseEventLoop.run_until_complete` method
+when it gets a coroutine object instead of a task.
+
+The diagram shows the control flow, it does not describe exactly how things
+work internally. For example, the sleep coroutine creates an internal future
+which uses :meth:`BaseEventLoop.call_later` to wake up the task in 1 second.
+
+
+InvalidStateError
+-----------------
+
+.. exception:: InvalidStateError
+
+ The operation is not allowed in this state.
+
+
+Future
+------
+
+.. class:: Future(\*, loop=None)
+
+ This class is *almost* compatible with :class:`concurrent.futures.Future`.
+
+ Differences:
+
+ - :meth:`result` and :meth:`exception` do not take a timeout argument and
+ raise an exception when the future isn't done yet.
+
+ - Callbacks registered with :meth:`add_done_callback` are always called
+ via the event loop's :meth:`~BaseEventLoop.call_soon_threadsafe`.
+
+ - This class is not compatible with the :func:`~concurrent.futures.wait` and
+ :func:`~concurrent.futures.as_completed` functions in the
+ :mod:`concurrent.futures` package.
+
+ .. method:: cancel()
+
+ Cancel the future and schedule callbacks.
+
+ If the future is already done or cancelled, return ``False``. Otherwise,
+ change the future's state to cancelled, schedule the callbacks and return
+ ``True``.
+
+ .. method:: cancelled()
+
+ Return ``True`` if the future was cancelled.
+
+ .. method:: done()
+
+ Return True if the future is done.
+
+ Done means either that a result / exception are available, or that the
+ future was cancelled.
+
+ .. method:: result()
+
+ Return the result this future represents.
+
+ If the future has been cancelled, raises :exc:`CancelledError`. If the
+ future's result isn't yet available, raises :exc:`InvalidStateError`. If
+ the future is done and has an exception set, this exception is raised.
+
+ .. method:: exception()
+
+ Return the exception that was set on this future.
+
+ The exception (or ``None`` if no exception was set) is returned only if
+ the future is done. If the future has been cancelled, raises
+ :exc:`CancelledError`. If the future isn't done yet, raises
+ :exc:`InvalidStateError`.
+
+ .. method:: add_done_callback(fn)
+
+ Add a callback to be run when the future becomes done.
+
+ The callback is called with a single argument - the future object. If the
+ future is already done when this is called, the callback is scheduled
+ with :meth:`~BaseEventLoop.call_soon`.
+
+ .. method:: remove_done_callback(fn)
+
+ Remove all instances of a callback from the "call when done" list.
+
+ Returns the number of callbacks removed.
+
+ .. method:: set_result(result)
+
+ Mark the future done and set its result.
+
+ If the future is already done when this method is called, raises
+ :exc:`InvalidStateError`.
+
+ .. method:: set_exception(exception)
+
+ Mark the future done and set an exception.
+
+ If the future is already done when this method is called, raises
+ :exc:`InvalidStateError`.
+
+
+Example: Future with run_until_complete()
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example combining a :class:`Future` and a :ref:`coroutine function
+<coroutine>`::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def slow_operation(future):
+ yield from asyncio.sleep(1)
+ future.set_result('Future is done!')
+
+ loop = asyncio.get_event_loop()
+ future = asyncio.Future()
+ asyncio.Task(slow_operation(future))
+ loop.run_until_complete(future)
+ print(future.result())
+ loop.close()
+
+The coroutine function is responsible of the computation (which takes 1 second)
+and it stores the result into the future. The
+:meth:`~BaseEventLoop.run_until_complete` method waits for the completion of
+the future.
+
+.. note::
+ The :meth:`~BaseEventLoop.run_until_complete` method uses internally the
+ :meth:`~Future.add_done_callback` method to be notified when the future is
+ done.
+
+
+Example: Future with run_forever()
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The previous example can be written differently using the
+:meth:`Future.add_done_callback` method to describe explicitly the control
+flow::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def slow_operation(future):
+ yield from asyncio.sleep(1)
+ future.set_result('Future is done!')
+
+ def got_result(future):
+ print(future.result())
+ loop.stop()
+
+ loop = asyncio.get_event_loop()
+ future = asyncio.Future()
+ asyncio.Task(slow_operation(future))
+ future.add_done_callback(got_result)
+ try:
+ loop.run_forever()
+ finally:
+ loop.close()
+
+In this example, the future is responsible to display the result and to stop
+the loop.
+
+.. note::
+ The "slow_operation" coroutine object is only executed when the event loop
+ starts running, so it is possible to add a "done callback" to the future
+ after creating the task scheduling the coroutine object.
+
+
+
+Task
+----
+
+.. class:: Task(coro, \*, loop=None)
+
+ A coroutine object wrapped in a :class:`Future`. Subclass of :class:`Future`.
+
+ .. classmethod:: all_tasks(loop=None)
+
+ Return a set of all tasks for an event loop.
+
+ By default all tasks for the current event loop are returned.
+
+ .. classmethod:: current_task(loop=None)
+
+ Return the currently running task in an event loop or ``None``.
+
+ By default the current task for the current event loop is returned.
+
+ ``None`` is returned when called not in the context of a :class:`Task`.
+
+ .. method:: get_stack(self, \*, limit=None)
+
+ Return the list of stack frames for this task's coroutine.
+
+ If the coroutine is active, this returns the stack where it is suspended.
+ If the coroutine has completed successfully or was cancelled, this
+ returns an empty list. If the coroutine was terminated by an exception,
+ this returns the list of traceback frames.
+
+ The frames are always ordered from oldest to newest.
+
+ The optional limit gives the maximum number of frames to return; by
+ default all available frames are returned. Its meaning differs depending
+ on whether a stack or a traceback is returned: the newest frames of a
+ stack are returned, but the oldest frames of a traceback are returned.
+ (This matches the behavior of the traceback module.)
+
+ For reasons beyond our control, only one stack frame is returned for a
+ suspended coroutine.
+
+ .. method:: print_stack(\*, limit=None, file=None)
+
+ Print the stack or traceback for this task's coroutine.
+
+ This produces output similar to that of the traceback module, for the
+ frames retrieved by get_stack(). The limit argument is passed to
+ get_stack(). The file argument is an I/O stream to which the output
+ goes; by default it goes to sys.stderr.
+
+
+Example: Parallel execution of tasks
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example executing 3 tasks (A, B, C) in parallel::
+
+ import asyncio
+
+ @asyncio.coroutine
+ def factorial(name, number):
+ f = 1
+ for i in range(2, number+1):
+ print("Task %s: Compute factorial(%s)..." % (name, i))
+ yield from asyncio.sleep(1)
+ f *= i
+ print("Task %s: factorial(%s) = %s" % (name, number, f))
+
+ tasks = [
+ asyncio.Task(factorial("A", 2)),
+ asyncio.Task(factorial("B", 3)),
+ asyncio.Task(factorial("C", 4))]
+
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(asyncio.wait(tasks))
+ loop.close()
+
+Output::
+
+ Task A: Compute factorial(2)...
+ Task B: Compute factorial(2)...
+ Task C: Compute factorial(2)...
+ Task A: factorial(2) = 2
+ Task B: Compute factorial(3)...
+ Task C: Compute factorial(3)...
+ Task B: factorial(3) = 6
+ Task C: Compute factorial(4)...
+ Task C: factorial(4) = 24
+
+A task is automatically scheduled for execution when it is created. The event
+loop stops when all tasks are done.
+
+
+Task functions
+--------------
+
+.. note::
+
+ In the functions below, the optional *loop* argument allows to explicitly set
+ the event loop object used by the underlying task or coroutine. If it's
+ not provided, the default event loop is used.
+
+.. function:: as_completed(fs, \*, loop=None, timeout=None)
+
+ Return an iterator whose values, when waited for, are :class:`Future`
+ instances.
+
+ Raises :exc:`TimeoutError` if the timeout occurs before all Futures are done.
+
+ Example::
+
+ for f in as_completed(fs):
+ result = yield from f # The 'yield from' may raise
+ # Use result
+
+ .. note::
+
+ The futures ``f`` are not necessarily members of fs.
+
+.. function:: async(coro_or_future, \*, loop=None)
+
+ Wrap a :ref:`coroutine object <coroutine>` in a future.
+
+ If the argument is a :class:`Future`, it is returned directly.
+
+.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False)
+
+ Return a future aggregating results from the given coroutine objects or
+ futures.
+
+ All futures must share the same event loop. If all the tasks are done
+ successfully, the returned future's result is the list of results (in the
+ order of the original sequence, not necessarily the order of results
+ arrival). If *return_exceptions* is True, exceptions in the tasks are
+ treated the same as successful results, and gathered in the result list;
+ otherwise, the first raised exception will be immediately propagated to the
+ returned future.
+
+ Cancellation: if the outer Future is cancelled, all children (that have not
+ completed yet) are also cancelled. If any child is cancelled, this is
+ treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the
+ outer Future is *not* cancelled in this case. (This is to prevent the
+ cancellation of one child to cause other children to be cancelled.)
+
+.. function:: iscoroutine(obj)
+
+ Return ``True`` if *obj* is a :ref:`coroutine object <coroutine>`.
+
+.. function:: iscoroutinefunction(obj)
+
+ Return ``True`` if *func* is a decorated :ref:`coroutine function
+ <coroutine>`.
+
+.. function:: sleep(delay, result=None, \*, loop=None)
+
+ Create a :ref:`coroutine <coroutine>` that completes after a given
+ time (in seconds). If *result* is provided, it is produced to the caller
+ when the coroutine completes.
+
+ The resolution of the sleep depends on the :ref:`granularity of the event
+ loop <asyncio-delayed-calls>`.
+
+.. function:: shield(arg, \*, loop=None)
+
+ Wait for a future, shielding it from cancellation.
+
+ The statement::
+
+ res = yield from shield(something())
+
+ is exactly equivalent to the statement::
+
+ res = yield from something()
+
+ *except* that if the coroutine containing it is cancelled, the task running
+ in ``something()`` is not cancelled. From the point of view of
+ ``something()``, the cancellation did not happen. But its caller is still
+ cancelled, so the yield-from expression still raises
+ :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is
+ cancelled by other means this will still cancel ``shield()``.
+
+ If you want to completely ignore cancellation (not recommended) you can
+ combine ``shield()`` with a try/except clause, as follows::
+
+ try:
+ res = yield from shield(something())
+ except CancelledError:
+ res = None
+
+.. function:: wait(futures, \*, loop=None, timeout=None, return_when=ALL_COMPLETED)
+
+ Wait for the Futures and coroutine objects given by the sequence *futures*
+ to complete. Coroutines will be wrapped in Tasks. Returns two sets of
+ :class:`Future`: (done, pending).
+
+ *timeout* can be used to control the maximum number of seconds to wait before
+ returning. *timeout* can be an int or float. If *timeout* is not specified
+ or ``None``, there is no limit to the wait time.
+
+ *return_when* indicates when this function should return. It must be one of
+ the following constants of the :mod:`concurrent.futures` module:
+
+ .. tabularcolumns:: |l|L|
+
+ +-----------------------------+----------------------------------------+
+ | Constant | Description |
+ +=============================+========================================+
+ | :const:`FIRST_COMPLETED` | The function will return when any |
+ | | future finishes or is cancelled. |
+ +-----------------------------+----------------------------------------+
+ | :const:`FIRST_EXCEPTION` | The function will return when any |
+ | | future finishes by raising an |
+ | | exception. If no future raises an |
+ | | exception then it is equivalent to |
+ | | :const:`ALL_COMPLETED`. |
+ +-----------------------------+----------------------------------------+
+ | :const:`ALL_COMPLETED` | The function will return when all |
+ | | futures finish or are cancelled. |
+ +-----------------------------+----------------------------------------+
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+ Usage::
+
+ done, pending = yield from asyncio.wait(fs)
+
+ .. note::
+
+ This does not raise :exc:`TimeoutError`! Futures that aren't done when
+ the timeout occurs are returned in the second set.
+
+
+.. function:: wait_for(fut, timeout, \*, loop=None)
+
+ Wait for the single :class:`Future` or :ref:`coroutine object <coroutine>`
+ to complete, with timeout. If *timeout* is ``None``, block until the future
+ completes.
+
+ Coroutine will be wrapped in :class:`Task`.
+
+ Returns result of the Future or coroutine. When a timeout occurs, it
+ cancels the task and raises :exc:`TimeoutError`. To avoid the task
+ cancellation, wrap it in :func:`shield`.
+
+ This function is a :ref:`coroutine <coroutine>`.
+
+ Usage::
+
+ result = yield from asyncio.wait_for(fut, 60.0)
+
diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst
new file mode 100644
index 0000000..23731b1
--- /dev/null
+++ b/Doc/library/asyncio.rst
@@ -0,0 +1,59 @@
+:mod:`asyncio` -- Asynchronous I/O, event loop, coroutines and tasks
+====================================================================
+
+.. module:: asyncio
+ :synopsis: Asynchronous I/O, event loop, coroutines and tasks.
+
+.. versionadded:: 3.4
+
+**Source code:** :source:`Lib/asyncio/`
+
+--------------
+
+This module provides infrastructure for writing single-threaded concurrent
+code using coroutines, multiplexing I/O access over sockets and other
+resources, running network clients and servers, and other related primitives.
+Here is a more detailed list of the package contents:
+
+* a pluggable :ref:`event loop <asyncio-event-loop>` with various system-specific
+ implementations;
+
+* :ref:`transport <asyncio-transport>` and :ref:`protocol <asyncio-protocol>` abstractions
+ (similar to those in `Twisted <http://twistedmatrix.com/>`_);
+
+* concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and
+ others (some may be system-dependent);
+
+* a :class:`Future` class that mimics the one in the :mod:`concurrent.futures`
+ module, but adapted for use with the event loop;
+
+* coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write
+ concurrent code in a sequential fashion;
+
+* cancellation support for :class:`Future`\s and coroutines;
+
+* :ref:`synchronization primitives <asyncio-sync>` for use between coroutines in
+ a single thread, mimicking those in the :mod:`threading` module;
+
+* an interface for passing work off to a threadpool, for times when
+ you absolutely, positively have to use a library that makes blocking
+ I/O calls.
+
+Table of content:
+
+.. toctree::
+ :maxdepth: 3
+
+ asyncio-eventloop.rst
+ asyncio-task.rst
+ asyncio-protocol.rst
+ asyncio-stream.rst
+ asyncio-subprocess.rst
+ asyncio-sync.rst
+ asyncio-dev.rst
+
+.. seealso::
+
+ The :mod:`asyncio` module was designed in the :PEP:`3156`. For a
+ motivational primer on transports and protocols, see :PEP:`3153`.
+
diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst
index 1521e72..0adf8d9 100644
--- a/Doc/library/asyncore.rst
+++ b/Doc/library/asyncore.rst
@@ -13,6 +13,11 @@
--------------
+.. note::
+
+ This module exists for backwards compatibility only. For new code we
+ recommend using :mod:`asyncio`.
+
This module provides the basic infrastructure for writing asynchronous socket
service clients and servers.
diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst
index edb3870..ce127aa 100644
--- a/Doc/library/audioop.rst
+++ b/Doc/library/audioop.rst
@@ -6,9 +6,14 @@
The :mod:`audioop` module contains some useful operations on sound fragments.
-It operates on sound fragments consisting of signed integer samples 8, 16 or 32
-bits wide, stored in bytes objects. All scalar items are integers, unless
-specified otherwise.
+It operates on sound fragments consisting of signed integer samples 8, 16, 24
+or 32 bits wide, stored in :term:`bytes-like object`\ s. All scalar items are
+integers, unless specified otherwise.
+
+.. versionchanged:: 3.4
+ Support for 24-bit samples was added.
+ All functions now accept any :term:`bytes-like object`.
+ String input now results in an immediate error.
.. index::
single: Intel/DVI ADPCM
@@ -35,7 +40,7 @@ The module defines the following variables and functions:
.. function:: add(fragment1, fragment2, width)
Return a fragment which is the addition of the two samples passed as parameters.
- *width* is the sample width in bytes, either ``1``, ``2`` or ``4``. Both
+ *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both
fragments should have the same length. Samples are truncated in case of overflow.
@@ -70,6 +75,14 @@ The module defines the following variables and functions:
sample. Samples wrap around in case of overflow.
+.. function:: byteswap(fragment, width)
+
+ "Byteswap" all samples in a fragment and returns the modified fragment.
+ Converts big-endian samples to little-endian and vice versa.
+
+ .. versionadded:: 3.4
+
+
.. function:: cross(fragment, width)
Return the number of zero crossings in the fragment passed as an argument.
@@ -133,19 +146,19 @@ The module defines the following variables and functions:
.. function:: lin2lin(fragment, width, newwidth)
- Convert samples between 1-, 2- and 4-byte formats.
+ Convert samples between 1-, 2-, 3- and 4-byte formats.
.. note::
- In some audio formats, such as .WAV files, 16 and 32 bit samples are
+ In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are
signed, but 8 bit samples are unsigned. So when converting to 8 bit wide
samples for these formats, you need to also add 128 to the result::
new_frames = audioop.lin2lin(frames, old_width, 1)
new_frames = audioop.bias(new_frames, 1, 128)
- The same, in reverse, has to be applied when converting from 8 to 16 or 32
- bit width samples.
+ The same, in reverse, has to be applied when converting from 8 to 16, 24
+ or 32 bit width samples.
.. function:: lin2ulaw(fragment, width)
diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst
index f0d11b0..02b4d7b 100644
--- a/Doc/library/base64.rst
+++ b/Doc/library/base64.rst
@@ -1,32 +1,42 @@
-:mod:`base64` --- RFC 3548: Base16, Base32, Base64 Data Encodings
-=================================================================
+:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings
+===============================================================
.. module:: base64
- :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings
+ :synopsis: RFC 3548: Base16, Base32, Base64 Data Encodings;
+ Base85 and Ascii85
.. index::
pair: base64; encoding
single: MIME; base64 encoding
-This module provides data encoding and decoding as specified in :rfc:`3548`.
-This standard defines the Base16, Base32, and Base64 algorithms for encoding
-and decoding arbitrary binary strings into ASCII-only byte strings that can be
+This module provides functions for encoding binary data to printable
+ASCII characters and decoding such encodings back to binary data.
+It provides encoding and decoding functions for the encodings specified in
+in :rfc:`3548`, which defines the Base16, Base32, and Base64 algorithms,
+and for the de-facto standard Ascii85 and Base85 encodings.
+
+The :rfc:`3548` encodings are suitable for encoding binary data so that it can
safely sent by email, used as parts of URLs, or included as part of an HTTP
POST request. The encoding algorithm is not the same as the
:program:`uuencode` program.
-There are two interfaces provided by this module. The modern interface
-supports encoding and decoding ASCII byte string objects using all three
-alphabets. Additionally, the decoding functions of the modern interface also
-accept Unicode strings containing only ASCII characters. The legacy interface
-provides for encoding and decoding to and from file-like objects as well as
-byte strings, but only using the Base64 standard alphabet.
+There are two :rfc:`3548` interfaces provided by this module. The modern
+interface supports encoding and decoding ASCII byte string objects using all
+three :rfc:`3548` defined alphabets (normal, URL-safe, and filesystem-safe).
+Additionally, the decoding functions of the modern interface also accept
+Unicode strings containing only ASCII characters. The legacy interface provides
+for encoding and decoding to and from file-like objects as well as byte
+strings, but only using the Base64 standard alphabet.
.. versionchanged:: 3.3
ASCII-only Unicode strings are now accepted by the decoding functions of
the modern interface.
+.. versionchanged:: 3.4
+ Any :term:`bytes-like object`\ s are now accepted by all
+ encoding and decoding functions in this module. Ascii85/Base85 support added.
+
The modern interface provides:
.. function:: b64encode(s, altchars=None)
@@ -128,6 +138,76 @@ The modern interface provides:
string.
+.. function:: a85encode(s, *, foldspaces=False, wrapcol=0, pad=False, adobe=False)
+
+ Encode a byte string using Ascii85.
+
+ *s* is the string to encode. The encoded byte string is returned.
+
+ *foldspaces* is an optional flag that uses the special short sequence 'y'
+ instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This
+ feature is not supported by the "standard" Ascii85 encoding.
+
+ *wrapcol* controls whether the output should have newline ('\n')
+ characters added to it. If this is non-zero, each output line will be
+ at most this many characters long.
+
+ *pad* controls whether the input string is padded to a multiple of 4
+ before encoding. Note that the ``btoa`` implementation always pads.
+
+ *adobe* controls whether the encoded byte sequence is framed with ``<~``
+ and ``~>``, which is used by the Adobe implementation.
+
+ .. versionadded:: 3.4
+
+
+.. function:: a85decode(s, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v')
+
+ Decode an Ascii85 encoded byte string.
+
+ *s* is the byte string to decode.
+
+ *foldspaces* is a flag that specifies whether the 'y' short sequence
+ should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20).
+ This feature is not supported by the "standard" Ascii85 encoding.
+
+ *adobe* controls whether the input sequence is in Adobe Ascii85 format
+ (i.e. is framed with <~ and ~>).
+
+ *ignorechars* should be a byte string containing characters to ignore
+ from the input. This should only contain whitespace characters, and by
+ default contains all whitespace characters in ASCII.
+
+ .. versionadded:: 3.4
+
+
+.. function:: b85encode(s, pad=False)
+
+ Encode a byte string using base85, as used in e.g. git-style binary
+ diffs.
+
+ If *pad* is true, the input is padded with "\\0" so its length is a
+ multiple of 4 characters before encoding.
+
+ .. versionadded:: 3.4
+
+
+.. function:: b85decode(b)
+
+ Decode base85-encoded byte string. Padding is implicitly removed, if
+ necessary.
+
+ .. versionadded:: 3.4
+
+
+.. note::
+ Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or
+ Ascii85 characters can encode 4 binary bytes), while the better-known
+ Base64 has an expansion factor of 6 to 4. They are therefore more
+ efficient when space expensive. They differ by details such as the
+ character map used for encoding.
+
+
The legacy interface:
.. function:: decode(input, output)
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index b79bccd..488cda5 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -37,8 +37,8 @@ All of the classes in this module may safely be accessed from multiple threads.
file object to read from or write to.
The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``,
- ``'a'``, or ``'ab'`` for binary mode, or ``'rt'``, ``'wt'``, or ``'at'`` for
- text mode. The default is ``'rb'``.
+ ``'x'``, ``'xb'``, ``'a'`` or ``'ab'`` for binary mode, or ``'rt'``,
+ ``'wt'``, ``'xt'``, or ``'at'`` for text mode. The default is ``'rb'``.
The *compresslevel* argument is an integer from 1 to 9, as for the
:class:`BZ2File` constructor.
@@ -54,6 +54,9 @@ All of the classes in this module may safely be accessed from multiple threads.
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ The ``'x'`` (exclusive creation) mode was added.
+
.. class:: BZ2File(filename, mode='r', buffering=None, compresslevel=9)
@@ -64,8 +67,9 @@ All of the classes in this module may safely be accessed from multiple threads.
be used to read or write the compressed data.
The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for
- overwriting, or ``'a'`` for appending. These can equivalently be given as
- ``'rb'``, ``'wb'``, and ``'ab'`` respectively.
+ overwriting, ``'x'`` for exclusive creation, or ``'a'`` for appending. These
+ can equivalently be given as ``'rb'``, ``'wb'``, ``'xb'`` and ``'ab'``
+ respectively.
If *filename* is a file object (rather than an actual file name), a mode of
``'w'`` does not truncate the file, and is instead equivalent to ``'a'``.
@@ -113,6 +117,9 @@ All of the classes in this module may safely be accessed from multiple threads.
The ``'a'`` (append) mode was added, along with support for reading
multi-stream files.
+ .. versionchanged:: 3.4
+ The ``'x'`` (exclusive creation) mode was added.
+
Incremental (de)compression
---------------------------
diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst
index c4e7c60..fa13145 100644
--- a/Doc/library/cgi.rst
+++ b/Doc/library/cgi.rst
@@ -142,9 +142,11 @@ If a field represents an uploaded file, accessing the value via the
method reads the entire file in memory as bytes. This may not be what you
want. You can test for an uploaded file by testing either the
:attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file`
-attribute. You can then read the data at leisure from the :attr:`!file`
-attribute (the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline`
-methods will return bytes)::
+attribute. You can then read the data from the :attr:`!file`
+attribute before it is automatically closed as part of the garbage collection of
+the :class:`FieldStorage` instance
+(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will
+return bytes)::
fileitem = form["userfile"]
if fileitem.file:
@@ -176,6 +178,11 @@ actually be instances of the class :class:`MiniFieldStorage`. In this case, the
A form submitted via POST that also has a query string will contain both
:class:`FieldStorage` and :class:`MiniFieldStorage` items.
+.. versionchanged:: 3.4
+ The :attr:`~FieldStorage.file` attribute is automatically closed upon the
+ garbage collection of the creating :class:`FieldStorage` instance.
+
+
Higher Level Interface
----------------------
diff --git a/Doc/library/code.rst b/Doc/library/code.rst
index e869004..5b5d7cc 100644
--- a/Doc/library/code.rst
+++ b/Doc/library/code.rst
@@ -132,12 +132,15 @@ interpreter objects as well as the following additions.
.. method:: InteractiveConsole.interact(banner=None)
- Closely emulate the interactive Python console. The optional banner argument
+ Closely emulate the interactive Python console. The optional *banner* argument
specify the banner to print before the first interaction; by default it prints a
banner similar to the one printed by the standard Python interpreter, followed
by the class name of the console object in parentheses (so as not to confuse
this with the real interpreter -- since it's so close!).
+ .. versionchanged:: 3.4
+ To suppress printing any banner, pass an empty string.
+
.. method:: InteractiveConsole.push(line)
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 009ae26..3729dac 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -365,18 +365,23 @@ and implemented by all standard Python codecs:
| | in :pep:`383`. |
+-------------------------+-----------------------------------------------+
-In addition, the following error handlers are specific to a single codec:
+In addition, the following error handlers are specific to Unicode encoding
+schemes:
-+-------------------+---------+-------------------------------------------+
-| Value | Codec | Meaning |
-+===================+=========+===========================================+
-|``'surrogatepass'``| utf-8 | Allow encoding and decoding of surrogate |
-| | | codes in UTF-8. |
-+-------------------+---------+-------------------------------------------+
++-------------------+------------------------+-------------------------------------------+
+| Value | Codec | Meaning |
++===================+========================+===========================================+
+|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding of surrogate |
+| | utf-16-be, utf-16-le, | codes in all the Unicode encoding schemes.|
+| | utf-32-be, utf-32-le | |
++-------------------+------------------------+-------------------------------------------+
.. versionadded:: 3.1
The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers.
+.. versionchanged:: 3.4
+ The ``'surrogatepass'`` error handlers now works with utf-16\* and utf-32\* codecs.
+
The set of allowed values can be extended via :meth:`register_error`.
@@ -966,6 +971,10 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| cp037 | IBM037, IBM039 | English |
+-----------------+--------------------------------+--------------------------------+
+| cp273 | 273, IBM273, csIBM273 | German |
+| | | |
+| | | .. versionadded:: 3.4 |
++-----------------+--------------------------------+--------------------------------+
| cp424 | EBCDIC-CP-HE, IBM424 | Hebrew |
+-----------------+--------------------------------+--------------------------------+
| cp437 | 437, IBM437 | English |
@@ -1022,6 +1031,10 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| cp1026 | ibm1026 | Turkish |
+-----------------+--------------------------------+--------------------------------+
+| cp1125 | 1125, ibm1125, cp866u, ruscii | Ukrainian |
+| | | |
+| | | .. versionadded:: 3.4 |
++-----------------+--------------------------------+--------------------------------+
| cp1140 | ibm1140 | Western Europe |
+-----------------+--------------------------------+--------------------------------+
| cp1250 | windows-1250 | Central and Eastern Europe |
@@ -1167,6 +1180,12 @@ particular, the following variants typically exist:
| utf_8_sig | | all languages |
+-----------------+--------------------------------+--------------------------------+
+.. versionchanged:: 3.4
+ The utf-16\* and utf-32\* encoders no longer allow surrogate code points
+ (U+D800--U+DFFF) to be encoded. The utf-32\* decoders no longer decode
+ byte sequences that correspond to surrogate code points.
+
+
Python Specific Encodings
-------------------------
@@ -1177,6 +1196,9 @@ common use case for codecs, the underlying codec infrastructure supports
arbitrary data transforms rather than just text encodings). For asymmetric
codecs, the stated purpose describes the encoding direction.
+Text Encodings
+^^^^^^^^^^^^^^
+
The following codecs provide :class:`str` to :class:`bytes` encoding and
:term:`bytes-like object` to :class:`str` decoding, similar to the Unicode text
encodings.
@@ -1223,57 +1245,83 @@ encodings.
| | | .. deprecated:: 3.3 |
+--------------------+---------+---------------------------+
-The following codecs provide :term:`bytes-like object` to :class:`bytes`
-mappings.
-
-
-.. tabularcolumns:: |l|L|L|
-
-+----------------------+---------------------------+------------------------------+
-| Codec | Purpose | Encoder/decoder |
-+======================+===========================+==============================+
-| base64_codec [#b64]_ | Convert operand to MIME | :meth:`base64.b64encode`, |
-| | base64 (the result always | :meth:`base64.b64decode` |
-| | includes a trailing | |
-| | ``'\n'``) | |
-+----------------------+---------------------------+------------------------------+
-| bz2_codec | Compress the operand | :meth:`bz2.compress`, |
-| | using bz2 | :meth:`bz2.decompress` |
-+----------------------+---------------------------+------------------------------+
-| hex_codec | Convert operand to | :meth:`base64.b16encode`, |
-| | hexadecimal | :meth:`base64.b16decode` |
-| | representation, with two | |
-| | digits per byte | |
-+----------------------+---------------------------+------------------------------+
-| quopri_codec | Convert operand to MIME | :meth:`quopri.encodestring`, |
-| | quoted printable | :meth:`quopri.decodestring` |
-+----------------------+---------------------------+------------------------------+
-| uu_codec | Convert the operand using | :meth:`uu.encode`, |
-| | uuencode | :meth:`uu.decode` |
-+----------------------+---------------------------+------------------------------+
-| zlib_codec | Compress the operand | :meth:`zlib.compress`, |
-| | using gzip | :meth:`zlib.decompress` |
-+----------------------+---------------------------+------------------------------+
-
-.. [#b64] Rather than accepting any :term:`bytes-like object`,
- ``'base64_codec'`` accepts only :class:`bytes` and :class:`bytearray` for
- encoding and only :class:`bytes`, :class:`bytearray`, and ASCII-only
- instances of :class:`str` for decoding
-
-
-The following codecs provide :class:`str` to :class:`str` mappings.
+.. _binary-transforms:
+
+Binary Transforms
+^^^^^^^^^^^^^^^^^
+
+The following codecs provide binary transforms: :term:`bytes-like object`
+to :class:`bytes` mappings.
+
+
+.. tabularcolumns:: |l|L|L|L|
+
++----------------------+------------------+------------------------------+------------------------------+
+| Codec | Aliases | Purpose | Encoder / decoder |
++======================+==================+==============================+==============================+
+| base64_codec [#b64]_ | base64, base_64 | Convert operand to MIME | :meth:`base64.b64encode` / |
+| | | base64 (the result always | :meth:`base64.b64decode` |
+| | | includes a trailing | |
+| | | ``'\n'``) | |
+| | | | |
+| | | .. versionchanged:: 3.4 | |
+| | | accepts any | |
+| | | :term:`bytes-like object` | |
+| | | as input for encoding and | |
+| | | decoding | |
++----------------------+------------------+------------------------------+------------------------------+
+| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress` / |
+| | | using bz2 | :meth:`bz2.decompress` |
++----------------------+------------------+------------------------------+------------------------------+
+| hex_codec | hex | Convert operand to | :meth:`base64.b16encode` / |
+| | | hexadecimal | :meth:`base64.b16decode` |
+| | | representation, with two | |
+| | | digits per byte | |
++----------------------+------------------+------------------------------+------------------------------+
+| quopri_codec | quopri, | Convert operand to MIME | :meth:`quopri.encodestring` /|
+| | quotedprintable, | quoted printable | :meth:`quopri.decodestring` |
+| | quoted_printable | | |
++----------------------+------------------+------------------------------+------------------------------+
+| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / |
+| | | uuencode | :meth:`uu.decode` |
++----------------------+------------------+------------------------------+------------------------------+
+| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress` / |
+| | | using gzip | :meth:`zlib.decompress` |
++----------------------+------------------+------------------------------+------------------------------+
+
+.. [#b64] In addition to :term:`bytes-like objects <bytes-like object>`,
+ ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for
+ decoding
-.. tabularcolumns:: |l|L|
+.. versionadded:: 3.2
+ Restoration of the binary transforms.
+
+.. versionchanged:: 3.4
+ Restoration of the aliases for the binary transforms.
-+--------------------+---------------------------+
-| Codec | Purpose |
-+====================+===========================+
-| rot_13 | Returns the Caesar-cypher |
-| | encryption of the operand |
-+--------------------+---------------------------+
+
+.. _text-transforms:
+
+Text Transforms
+^^^^^^^^^^^^^^^
+
+The following codec provides a text transform: a :class:`str` to :class:`str`
+mapping.
+
+.. tabularcolumns:: |l|l|L|
+
++--------------------+---------+---------------------------+
+| Codec | Aliases | Purpose |
++====================+=========+===========================+
+| rot_13 | rot13 | Returns the Caesar-cypher |
+| | | encryption of the operand |
++--------------------+---------+---------------------------+
.. versionadded:: 3.2
- bytes-to-bytes and str-to-str codecs.
+ Restoration of the ``rot_13`` text transform.
+
+.. versionchanged:: 3.4
+ Restoration of the ``rot13`` alias.
:mod:`encodings.idna` --- Internationalized Domain Names in Applications
diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst
index 4b21932..356f473 100644
--- a/Doc/library/collections.abc.rst
+++ b/Doc/library/collections.abc.rst
@@ -15,7 +15,7 @@
import itertools
__name__ = '<doctest>'
-**Source code:** :source:`Lib/collections/abc.py`
+**Source code:** :source:`Lib/_collections_abc.py`
--------------
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 4def1dc..f5fe12a 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -76,14 +76,19 @@ The class can be used to simulate nested scopes and is useful in templating.
be modified to change which mappings are searched. The list should
always contain at least one mapping.
- .. method:: new_child()
+ .. method:: new_child(m=None)
- Returns a new :class:`ChainMap` containing a new :class:`dict` followed by
- all of the maps in the current instance. A call to ``d.new_child()`` is
- equivalent to: ``ChainMap({}, *d.maps)``. This method is used for
+ Returns a new :class:`ChainMap` containing a new map followed by
+ all of the maps in the current instance. If ``m`` is specified,
+ it becomes the new map at the front of the list of mappings; if not
+ specified, an empty dict is used, so that a call to ``d.new_child()``
+ is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for
creating subcontexts that can be updated without altering values in any
of the parent mappings.
+ .. versionchanged:: 3.4
+ The optional ``m`` parameter was added.
+
.. attribute:: parents
Property returning a new :class:`ChainMap` containing all of the maps in
@@ -369,10 +374,6 @@ or subtracting from an empty counter.
.. seealso::
- * `Counter class <http://code.activestate.com/recipes/576611/>`_
- adapted for Python 2.5 and an early `Bag recipe
- <http://code.activestate.com/recipes/259174/>`_ for Python 2.4.
-
* `Bag class <http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html>`_
in Smalltalk.
@@ -915,11 +916,6 @@ and more efficient to use a simple class declaration:
>>> class Status:
open, pending, closed = range(3)
-.. seealso::
-
- * `Named tuple recipe <http://code.activestate.com/recipes/500261/>`_
- adapted for Python 2.4.
-
* `Recipe for named tuple abstract base class with a metaclass mix-in
<http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/>`_
by Jan Kaliszewski. Besides providing an :term:`abstract base class` for
@@ -982,10 +978,6 @@ The :class:`OrderedDict` constructor and :meth:`update` method both accept
keyword arguments, but their order is lost because Python's function call
semantics pass-in keyword arguments using a regular unordered dictionary.
-.. seealso::
-
- `Equivalent OrderedDict recipe <http://code.activestate.com/recipes/576693/>`_
- that runs on Python 2.4 or later.
:class:`OrderedDict` Examples and Recipes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index b12c217..41e9e1b 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -20,7 +20,8 @@ compile Python sources.
.. program:: compileall
-.. cmdoption:: [directory|file]...
+.. cmdoption:: directory ...
+ file ...
Positional arguments are files to compile or directories that contain
source files, traversed recursively. If no argument is given, behave as if
diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst
index f2a512f..0de281b 100644
--- a/Doc/library/concurrency.rst
+++ b/Doc/library/concurrency.rst
@@ -20,7 +20,6 @@ multitasking). Here's an overview:
subprocess.rst
sched.rst
queue.rst
- select.rst
The following are support modules for some of the above services:
diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst
index 575b1ea..0495737 100644
--- a/Doc/library/concurrent.futures.rst
+++ b/Doc/library/concurrent.futures.rst
@@ -164,6 +164,9 @@ uses a pool of processes to execute calls asynchronously.
allows it to side-step the :term:`Global Interpreter Lock` but also means that
only picklable objects can be executed and returned.
+The ``__main__`` module must be importable by worker subprocesses. This means
+that :class:`ProcessPoolExecutor` will not work in the interactive interpreter.
+
Calling :class:`Executor` or :class:`Future` methods from a callable submitted
to a :class:`ProcessPoolExecutor` will result in deadlock.
diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst
index fba48f4..82efd0c 100644
--- a/Doc/library/contextlib.rst
+++ b/Doc/library/contextlib.rst
@@ -95,6 +95,83 @@ Functions and classes provided:
``page.close()`` will be called when the :keyword:`with` block is exited.
+.. function:: suppress(*exceptions)
+
+ Return a context manager that suppresses any of the specified exceptions
+ if they occur in the body of a with statement and then resumes execution
+ with the first statement following the end of the with statement.
+
+ As with any other mechanism that completely suppresses exceptions, this
+ context manager should be used only to cover very specific errors where
+ silently continuing with program execution is known to be the right
+ thing to do.
+
+ For example::
+
+ from contextlib import suppress
+
+ with suppress(FileNotFoundError):
+ os.remove('somefile.tmp')
+
+ with suppress(FileNotFoundError):
+ os.remove('someotherfile.tmp')
+
+ This code is equivalent to::
+
+ try:
+ os.remove('somefile.tmp')
+ except FileNotFoundError:
+ pass
+
+ try:
+ os.remove('someotherfile.tmp')
+ except FileNotFoundError:
+ pass
+
+ This context manager is :ref:`reentrant <reentrant-cms>`.
+
+ .. versionadded:: 3.4
+
+
+.. function:: redirect_stdout(new_target)
+
+ Context manager for temporarily redirecting :data:`sys.stdout` to
+ another file or file-like object.
+
+ This tool adds flexibility to existing functions or classes whose output
+ is hardwired to stdout.
+
+ For example, the output of :func:`help` normally is sent to *sys.stdout*.
+ You can capture that output in a string by redirecting the output to a
+ :class:`io.StringIO` object::
+
+ f = io.StringIO()
+ with redirect_stdout(f):
+ help(pow)
+ s = f.getvalue()
+
+ To send the output of :func:`help` to a file on disk, redirect the output
+ to a regular file::
+
+ with open('help.txt', 'w') as f:
+ with redirect_stdout(f):
+ help(pow)
+
+ To send the output of :func:`help` to *sys.stderr*::
+
+ with redirect_stdout(sys.stderr):
+ help(pow)
+
+ Note that the global side effect on :data:`sys.stdout` means that this
+ context manager is not suitable for use in library code and most threaded
+ applications. It also has no effect on the output of subprocesses.
+ However, it is still a useful approach for many utility scripts.
+
+ This context manager is :ref:`reusable but not reentrant <reusable-cms>`.
+
+ .. versionadded:: 3.4
+
+
.. class:: ContextDecorator()
A base class that enables a context manager to also be used as a decorator.
@@ -520,3 +597,153 @@ an explicit ``with`` statement.
The specification, background, and examples for the Python :keyword:`with`
statement.
+.. _single-use-reusable-and-reentrant-cms:
+
+Single use, reusable and reentrant context managers
+---------------------------------------------------
+
+Most context managers are written in a way that means they can only be
+used effectively in a :keyword:`with` statement once. These single use
+context managers must be created afresh each time they're used -
+attempting to use them a second time will trigger an exception or
+otherwise not work correctly.
+
+This common limitation means that it is generally advisable to create
+context managers directly in the header of the :keyword:`with` statement
+where they are used (as shown in all of the usage examples above).
+
+Files are an example of effectively single use context managers, since
+the first :keyword:`with` statement will close the file, preventing any
+further IO operations using that file object.
+
+Context managers created using :func:`contextmanager` are also single use
+context managers, and will complain about the underlying generator failing
+to yield if an attempt is made to use them a second time::
+
+ >>> from contextlib import contextmanager
+ >>> @contextmanager
+ ... def singleuse():
+ ... print("Before")
+ ... yield
+ ... print("After")
+ ...
+ >>> cm = singleuse()
+ >>> with cm:
+ ... pass
+ ...
+ Before
+ After
+ >>> with cm:
+ ... pass
+ ...
+ Traceback (most recent call last):
+ ...
+ RuntimeError: generator didn't yield
+
+
+.. _reentrant-cms:
+
+Reentrant context managers
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+More sophisticated context managers may be "reentrant". These context
+managers can not only be used in multiple :keyword:`with` statements,
+but may also be used *inside* a :keyword:`with` statement that is already
+using the same context manager.
+
+:class:`threading.RLock` is an example of a reentrant context manager, as are
+:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of
+reentrant use::
+
+ >>> from contextlib import redirect_stdout
+ >>> from io import StringIO
+ >>> stream = StringIO()
+ >>> write_to_stream = redirect_stdout(stream)
+ >>> with write_to_stream:
+ ... print("This is written to the stream rather than stdout")
+ ... with write_to_stream:
+ ... print("This is also written to the stream")
+ ...
+ >>> print("This is written directly to stdout")
+ This is written directly to stdout
+ >>> print(stream.getvalue())
+ This is written to the stream rather than stdout
+ This is also written to the stream
+
+Real world examples of reentrancy are more likely to involve multiple
+functions calling each other and hence be far more complicated than this
+example.
+
+Note also that being reentrant is *not* the same thing as being thread safe.
+:func:`redirect_stdout`, for example, is definitely not thread safe, as it
+makes a global modification to the system state by binding :data:`sys.stdout`
+to a different stream.
+
+
+.. _reusable-cms:
+
+Reusable context managers
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Distinct from both single use and reentrant context managers are "reusable"
+context managers (or, to be completely explicit, "reusable, but not
+reentrant" context managers, since reentrant context managers are also
+reusable). These context managers support being used multiple times, but
+will fail (or otherwise not work correctly) if the specific context manager
+instance has already been used in a containing with statement.
+
+:class:`threading.Lock` is an example of a reusable, but not reentrant,
+context manager (for a reentrant lock, it is necessary to use
+:class:`threading.RLock` instead).
+
+Another example of a reusable, but not reentrant, context manager is
+:class:`ExitStack`, as it invokes *all* currently registered callbacks
+when leaving any with statement, regardless of where those callbacks
+were added::
+
+ >>> from contextlib import ExitStack
+ >>> stack = ExitStack()
+ >>> with stack:
+ ... stack.callback(print, "Callback: from first context")
+ ... print("Leaving first context")
+ ...
+ Leaving first context
+ Callback: from first context
+ >>> with stack:
+ ... stack.callback(print, "Callback: from second context")
+ ... print("Leaving second context")
+ ...
+ Leaving second context
+ Callback: from second context
+ >>> with stack:
+ ... stack.callback(print, "Callback: from outer context")
+ ... with stack:
+ ... stack.callback(print, "Callback: from inner context")
+ ... print("Leaving inner context")
+ ... print("Leaving outer context")
+ ...
+ Leaving inner context
+ Callback: from inner context
+ Callback: from outer context
+ Leaving outer context
+
+As the output from the example shows, reusing a single stack object across
+multiple with statements works correctly, but attempting to nest them
+will cause the stack to be cleared at the end of the innermost with
+statement, which is unlikely to be desirable behaviour.
+
+Using separate :class:`ExitStack` instances instead of reusing a single
+instance avoids that problem::
+
+ >>> from contextlib import ExitStack
+ >>> with ExitStack() as outer_stack:
+ ... outer_stack.callback(print, "Callback: from outer context")
+ ... with ExitStack() as inner_stack:
+ ... inner_stack.callback(print, "Callback: from inner context")
+ ... print("Leaving inner context")
+ ... print("Leaving outer context")
+ ...
+ Leaving inner context
+ Callback: from inner context
+ Leaving outer context
+ Callback: from outer context
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index 314636e..f3e60b4 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -12,7 +12,7 @@ The :mod:`curses` module provides an interface to the curses library, the
de-facto standard for portable advanced terminal handling.
While curses is most widely used in the Unix environment, versions are available
-for DOS, OS/2, and possibly other systems as well. This extension module is
+for Windows, DOS, and possibly other systems as well. This extension module is
designed to match the API of ncurses, an open-source curses library hosted on
Linux and the BSD variants of Unix.
diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst
index d0382e0..48af082 100644
--- a/Doc/library/datatypes.rst
+++ b/Doc/library/datatypes.rst
@@ -30,3 +30,4 @@ The following modules are documented in this chapter:
copy.rst
pprint.rst
reprlib.rst
+ enum.rst
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index 065e850..e4f1eb2 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -170,10 +170,12 @@ dates or times.
* ``0 <= seconds < 3600*24`` (the number of seconds in one day)
* ``-999999999 <= days <= 999999999``
- If any argument is a float and there are fractional microseconds, the fractional
- microseconds left over from all arguments are combined and their sum is rounded
- to the nearest microsecond. If no argument is a float, the conversion and
- normalization processes are exact (no information is lost).
+ If any argument is a float and there are fractional microseconds,
+ the fractional microseconds left over from all arguments are
+ combined and their sum is rounded to the nearest microsecond using
+ round-half-to-even tiebreaker. If no argument is a float, the
+ conversion and normalization processes are exact (no information is
+ lost).
If the normalized value of days lies outside the indicated range,
:exc:`OverflowError` is raised.
diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst
index 81a05c7..f5496d5 100644
--- a/Doc/library/dbm.rst
+++ b/Doc/library/dbm.rst
@@ -73,33 +73,39 @@ Key and values are always stored as bytes. This means that when
strings are used they are implicitly converted to the default encoding before
being stored.
+These objects also support being used in a :keyword:`with` statement, which
+will automatically close them when done.
+
+.. versionchanged:: 3.4
+ Added native support for the context management protocol to the objects
+ returned by :func:`.open`.
+
The following example records some hostnames and a corresponding title, and
then prints out the contents of the database::
import dbm
# Open database, creating it if necessary.
- db = dbm.open('cache', 'c')
+ with dbm.open('cache', 'c') as db:
- # Record some values
- db[b'hello'] = b'there'
- db['www.python.org'] = 'Python Website'
- db['www.cnn.com'] = 'Cable News Network'
+ # Record some values
+ db[b'hello'] = b'there'
+ db['www.python.org'] = 'Python Website'
+ db['www.cnn.com'] = 'Cable News Network'
- # Note that the keys are considered bytes now.
- assert db[b'www.python.org'] == b'Python Website'
- # Notice how the value is now in bytes.
- assert db['www.cnn.com'] == b'Cable News Network'
+ # Note that the keys are considered bytes now.
+ assert db[b'www.python.org'] == b'Python Website'
+ # Notice how the value is now in bytes.
+ assert db['www.cnn.com'] == b'Cable News Network'
- # Often-used methods of the dict interface work too.
- print(db.get('python.org', b'not present'))
+ # Often-used methods of the dict interface work too.
+ print(db.get('python.org', b'not present'))
- # Storing a non-string key or value will raise an exception (most
- # likely a TypeError).
- db['www.yahoo.com'] = 4
+ # Storing a non-string key or value will raise an exception (most
+ # likely a TypeError).
+ db['www.yahoo.com'] = 4
- # Close when done.
- db.close()
+ # db is automatically closed when leaving the with statement.
.. seealso::
diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst
index c69fb1c..88a2fa6 100644
--- a/Doc/library/debug.rst
+++ b/Doc/library/debug.rst
@@ -15,3 +15,4 @@ allowing you to identify bottlenecks in your programs.
profile.rst
timeit.rst
trace.rst
+ tracemalloc.rst
diff --git a/Doc/library/development.rst b/Doc/library/development.rst
index 2368769..06e7048 100644
--- a/Doc/library/development.rst
+++ b/Doc/library/development.rst
@@ -23,4 +23,3 @@ The list of modules described in this chapter is:
unittest.mock-examples.rst
2to3.rst
test.rst
- venv.rst
diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst
index ec7112d..d86550f 100644
--- a/Doc/library/dis.rst
+++ b/Doc/library/dis.rst
@@ -26,7 +26,8 @@ Example: Given the function :func:`myfunc`::
def myfunc(alist):
return len(alist)
-the following command can be used to get the disassembly of :func:`myfunc`::
+the following command can be used to display the disassembly of
+:func:`myfunc`::
>>> dis.dis(myfunc)
2 0 LOAD_GLOBAL 0 (len)
@@ -36,8 +37,77 @@ the following command can be used to get the disassembly of :func:`myfunc`::
(The "2" is a line number).
-The :mod:`dis` module defines the following functions and constants:
+Bytecode analysis
+-----------------
+.. versionadded:: 3.4
+
+The bytecode analysis API allows pieces of Python code to be wrapped in a
+:class:`Bytecode` object that provides easy access to details of the
+compiled code.
+
+.. class:: Bytecode(x, *, first_line=None, current_offset=None)
+
+ Analyse the bytecode corresponding to a function, method, string of
+ source code, or a code object (as returned by :func:`compile`).
+
+ This is a convenience wrapper around many of the functions listed below,
+ most notably :func:`get_instructions`, as iterating over a
+ :class:`Bytecode` instance yields the bytecode operations as
+ :class:`Instruction` instances.
+
+ If *first_line* is not None, it indicates the line number that should
+ be reported for the first source line in the disassembled code.
+ Otherwise, the source line information (if any) is taken directly from
+ the disassembled code object.
+
+ If *current_offset* is not None, it refers to an instruction offset
+ in the disassembled code. Setting this means :meth:`dis` will display
+ a "current instruction" marker against the specified opcode.
+
+ .. classmethod:: from_traceback(tb)
+
+ Construct a :class:`Bytecode` instance from the given traceback,
+ setting *current_offset* to the instruction responsible for the
+ exception.
+
+ .. data:: codeobj
+
+ The compiled code object.
+
+ .. data:: first_line
+
+ The first source line of the code object (if available)
+
+ .. method:: dis()
+
+ Return a formatted view of the bytecode operations (the same as
+ printed by :func:`dis`, but returned as a multi-line string).
+
+ .. method:: info()
+
+ Return a formatted multi-line string with detailed information about the
+ code object, like :func:`code_info`.
+
+Example::
+
+ >>> bytecode = dis.Bytecode(myfunc)
+ >>> for instr in bytecode:
+ ... print(instr.opname)
+ ...
+ LOAD_GLOBAL
+ LOAD_FAST
+ CALL_FUNCTION
+ RETURN_VALUE
+
+
+Analysis functions
+------------------
+
+The :mod:`dis` module also defines the following analysis functions that
+convert the input directly to the desired output. They can be useful if
+only a single operation is being performed, so the intermediate analysis
+object isn't useful:
.. function:: code_info(x)
@@ -51,17 +121,22 @@ The :mod:`dis` module defines the following functions and constants:
.. versionadded:: 3.2
-.. function:: show_code(x)
+.. function:: show_code(x, *, file=None)
Print detailed code object information for the supplied function, method,
- source code string or code object to stdout.
+ source code string or code object to *file* (or ``sys.stdout`` if *file*
+ is not specified).
- This is a convenient shorthand for ``print(code_info(x))``, intended for
- interactive exploration at the interpreter prompt.
+ This is a convenient shorthand for ``print(code_info(x), file=file)``,
+ intended for interactive exploration at the interpreter prompt.
.. versionadded:: 3.2
-.. function:: dis(x=None)
+ .. versionchanged:: 3.4
+ Added ``file`` parameter
+
+
+.. function:: dis(x=None, *, file=None)
Disassemble the *x* object. *x* can denote either a module, a class, a
method, a function, a code object, a string of source code or a byte sequence
@@ -72,16 +147,28 @@ The :mod:`dis` module defines the following functions and constants:
disassembled. If no object is provided, this function disassembles the last
traceback.
+ The disassembly is written as text to the supplied ``file`` argument if
+ provided and to ``sys.stdout`` otherwise.
-.. function:: distb(tb=None)
+ .. versionchanged:: 3.4
+ Added ``file`` parameter
+
+
+.. function:: distb(tb=None, *, file=None)
Disassemble the top-of-stack function of a traceback, using the last
traceback if none was passed. The instruction causing the exception is
indicated.
+ The disassembly is written as text to the supplied ``file`` argument if
+ provided and to ``sys.stdout`` otherwise.
+
+ .. versionchanged:: 3.4
+ Added ``file`` parameter
+
-.. function:: disassemble(code, lasti=-1)
- disco(code, lasti=-1)
+.. function:: disassemble(code, lasti=-1, *, file=None)
+ disco(code, lasti=-1, *, file=None)
Disassemble a code object, indicating the last instruction if *lasti* was
provided. The output is divided in the following columns:
@@ -97,6 +184,28 @@ The :mod:`dis` module defines the following functions and constants:
The parameter interpretation recognizes local and global variable names,
constant values, branch targets, and compare operators.
+ The disassembly is written as text to the supplied ``file`` argument if
+ provided and to ``sys.stdout`` otherwise.
+
+ .. versionchanged:: 3.4
+ Added ``file`` parameter
+
+
+.. function:: get_instructions(x, *, first_line=None)
+
+ Return an iterator over the instructions in the supplied function, method,
+ source code string or code object.
+
+ The iterator generates a series of :class:`Instruction` named tuples
+ giving the details of each operation in the supplied code.
+
+ If *first_line* is not None, it indicates the line number that should
+ be reported for the first source line in the disassembled code.
+ Otherwise, the source line information (if any) is taken directly from
+ the disassembled code object.
+
+ .. versionadded:: 3.4
+
.. function:: findlinestarts(code)
@@ -111,60 +220,66 @@ The :mod:`dis` module defines the following functions and constants:
return a list of these offsets.
-.. data:: opname
+.. function:: stack_effect(opcode, [oparg])
- Sequence of operation names, indexable using the bytecode.
+ Compute the stack effect of *opcode* with argument *oparg*.
+ .. versionadded:: 3.4
-.. data:: opmap
+.. _bytecodes:
- Dictionary mapping operation names to bytecodes.
+Python Bytecode Instructions
+----------------------------
+The :func:`get_instructions` function and :class:`Bytecode` class provide
+details of bytecode instructions as :class:`Instruction` instances:
-.. data:: cmp_op
+.. class:: Instruction
- Sequence of all compare operation names.
+ Details for a bytecode operation
+ .. data:: opcode
-.. data:: hasconst
+ numeric code for operation, corresponding to the opcode values listed
+ below and the bytecode values in the :ref:`opcode_collections`.
- Sequence of bytecodes that have a constant parameter.
+ .. data:: opname
-.. data:: hasfree
+ human readable name for operation
- Sequence of bytecodes that access a free variable.
+ .. data:: arg
-.. data:: hasname
+ numeric argument to operation (if any), otherwise None
- Sequence of bytecodes that access an attribute by name.
+ .. data:: argval
-.. data:: hasjrel
+ resolved arg value (if known), otherwise same as arg
- Sequence of bytecodes that have a relative jump target.
+ .. data:: argrepr
-.. data:: hasjabs
+ human readable description of operation argument
- Sequence of bytecodes that have an absolute jump target.
+ .. data:: offset
-.. data:: haslocal
+ start index of operation within bytecode sequence
- Sequence of bytecodes that access a local variable.
+ .. data:: starts_line
-.. data:: hascompare
+ line started by this opcode (if any), otherwise None
- Sequence of bytecodes of Boolean operations.
+ .. data:: is_jump_target
-.. _bytecodes:
+ ``True`` if other code jumps to here, otherwise ``False``
+
+ .. versionadded:: 3.4
-Python Bytecode Instructions
-----------------------------
The Python compiler currently generates the following bytecode instructions.
@@ -506,12 +621,6 @@ the stack so that it is available for further iterations of the loop.
.. XXX explain the WHY stuff!
-.. opcode:: STORE_LOCALS
-
- Pops TOS from the stack and stores it as the current frame's ``f_locals``.
- This is used in class construction.
-
-
All of the following opcodes expect arguments. An argument is two bytes, with
the more significant byte last.
@@ -722,6 +831,13 @@ the more significant byte last.
Pushes a reference to the object the cell contains on the stack.
+.. opcode:: LOAD_CLASSDEREF (i)
+
+ Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before
+ consulting the cell. This is used for loading free variables in class
+ bodies.
+
+
.. opcode:: STORE_DEREF (i)
Stores TOS into the cell contained in slot *i* of the cell and free variable
@@ -820,3 +936,62 @@ the more significant byte last.
which don't take arguments ``< HAVE_ARGUMENT`` and those which do ``>=
HAVE_ARGUMENT``.
+.. _opcode_collections:
+
+Opcode collections
+------------------
+
+These collections are provided for automatic introspection of bytecode
+instructions:
+
+.. data:: opname
+
+ Sequence of operation names, indexable using the bytecode.
+
+
+.. data:: opmap
+
+ Dictionary mapping operation names to bytecodes.
+
+
+.. data:: cmp_op
+
+ Sequence of all compare operation names.
+
+
+.. data:: hasconst
+
+ Sequence of bytecodes that have a constant parameter.
+
+
+.. data:: hasfree
+
+ Sequence of bytecodes that access a free variable (note that 'free' in
+ this context refers to names in the current scope that are referenced by
+ inner scopes or names in outer scopes that are referenced from this scope.
+ It does *not* include references to global or builtin scopes).
+
+
+.. data:: hasname
+
+ Sequence of bytecodes that access an attribute by name.
+
+
+.. data:: hasjrel
+
+ Sequence of bytecodes that have a relative jump target.
+
+
+.. data:: hasjabs
+
+ Sequence of bytecodes that have an absolute jump target.
+
+
+.. data:: haslocal
+
+ Sequence of bytecodes that access a local variable.
+
+
+.. data:: hascompare
+
+ Sequence of bytecodes of Boolean operations.
diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst
new file mode 100644
index 0000000..fb3f5df
--- /dev/null
+++ b/Doc/library/distribution.rst
@@ -0,0 +1,14 @@
+***********************************
+Software Packaging and Distribution
+***********************************
+
+These libraries help you with publishing and installing Python software.
+While these modules are designed to work in conjunction with the
+`Python Package Index <https://pypi.python.org>`__, they can also be used
+with a local index server, or without any index server at all.
+
+.. toctree::
+
+ distutils.rst
+ ensurepip.rst
+ venv.rst
diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst
index 6666a9b..8e3732b 100644
--- a/Doc/library/distutils.rst
+++ b/Doc/library/distutils.rst
@@ -12,14 +12,15 @@ additional modules into a Python installation. The new modules may be either
100%-pure Python, or may be extension modules written in C, or may be
collections of Python packages which include modules coded in both Python and C.
+Most Python users will *not* want to use this module directly, but instead
+use the cross-version tools maintained by the Python Packaging Authority.
+Refer to the `Python Packaging User Guide <http://packaging.python.org>`_
+for more information.
-User documentation and API reference are provided in another document:
+For the benefits of packaging tool authors and users seeking a deeper
+understanding of the details of the current packaging and distribution
+system, the legacy :mod:`distutils` based user documentation and API
+reference remain available:
-.. seealso::
-
- :ref:`distutils-index`
- The manual for developers and packagers of Python modules. This describes
- how to prepare :mod:`distutils`\ -based packages so that they may be
- easily installed into an existing Python installation. It also contains
- instructions for end-users wanting to install a distutils-based package,
- :ref:`install-index`.
+* :ref:`install-index`
+* :ref:`distutils-index`
diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst
index 222c719..50626e9 100644
--- a/Doc/library/doctest.rst
+++ b/Doc/library/doctest.rst
@@ -278,6 +278,10 @@ strings are treated as if they were docstrings. In output, a key ``K`` in
Any classes found are recursively searched similarly, to test docstrings in
their contained methods and nested classes.
+.. impl-detail::
+ Prior to version 3.4, extension modules written in C were not fully
+ searched by doctest.
+
.. _doctest-finding-examples:
@@ -495,7 +499,11 @@ Option Flags
A number of option flags control various aspects of doctest's behavior.
Symbolic names for the flags are supplied as module constants, which can be
or'ed together and passed to various functions. The names can also be used in
-:ref:`doctest directives <doctest-directives>`.
+:ref:`doctest directives <doctest-directives>`, and may be passed to the
+doctest command line interface via the ``-o`` option.
+
+.. versionadded:: 3.4
+ The ``-o`` command line option.
The first group of options define test semantics, controlling aspects of how
doctest decides whether actual output matches an example's expected output:
@@ -633,6 +641,19 @@ The second group of options controls how test failures are reported:
the output is suppressed.
+.. data:: FAIL_FAST
+
+ When specified, exit after the first failing example and don't attempt to run
+ the remaining examples. Thus, the number of failures reported will be at most
+ 1. This flag may be useful during debugging, since examples after the first
+ failure won't even produce debugging output.
+
+ The doctest command line accepts the option ``-f`` as a shorthand for ``-o
+ FAIL_FAST``.
+
+ .. versionadded:: 3.4
+
+
.. data:: REPORTING_FLAGS
A bitmask or'ing together all the reporting flags above.
@@ -1269,9 +1290,8 @@ DocTestFinder objects
A processing class used to extract the :class:`DocTest`\ s that are relevant to
a given object, from its docstring and the docstrings of its contained objects.
- :class:`DocTest`\ s can currently be extracted from the following object types:
- modules, functions, classes, methods, staticmethods, classmethods, and
- properties.
+ :class:`DocTest`\ s can be extracted from modules, classes, functions,
+ methods, staticmethods, classmethods, and properties.
The optional argument *verbose* can be used to display the objects searched by
the finder. It defaults to ``False`` (no output).
diff --git a/Doc/library/email-examples.rst b/Doc/library/email-examples.rst
index 32cecf3..cbbcb78 100644
--- a/Doc/library/email-examples.rst
+++ b/Doc/library/email-examples.rst
@@ -40,6 +40,36 @@ text version: [2]_
.. literalinclude:: ../includes/email-alternative.py
+.. _email-contentmanager-api-examples:
+
+Examples using the Provisional API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Here is a reworking of the last example using the provisional API. To make
+things a bit more interesting, we include a related image in the html part, and
+we save a copy of what we are going to send to disk, as well as sending it.
+
+This example also shows how easy it is to include non-ASCII, and simplifies the
+sending of the message using the :meth:`.send_message` method of the
+:mod:`smtplib` module.
+
+.. literalinclude:: ../includes/email-alternative-new-api.py
+
+If we were instead sent the message from the last example, here is one
+way we could process it:
+
+.. literalinclude:: ../includes/email-read-alternative-new-api.py
+
+Up to the prompt, the output from the above is::
+
+ To: Penelope Pussycat <"penelope@example.com">, Fabrette Pussycat <"fabrette@example.com">
+ From: Pepé Le Pew <pepe@example.com>
+ Subject: Ayons asperges pour le déjeuner
+
+ Salut!
+
+ Cela ressemble à un excellent recipie[1] déjeuner.
+
+
.. rubric:: Footnotes
.. [1] Thanks to Matthew Dixon Cowles for the original inspiration and examples.
diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst
new file mode 100644
index 0000000..8f0bfdb
--- /dev/null
+++ b/Doc/library/email.contentmanager.rst
@@ -0,0 +1,430 @@
+:mod:`email.contentmanager`: Managing MIME Content
+--------------------------------------------------
+
+.. module:: email.contentmanager
+ :synopsis: Storing and Retrieving Content from MIME Parts
+
+.. moduleauthor:: R. David Murray <rdmurray@bitdance.com>
+.. sectionauthor:: R. David Murray <rdmurray@bitdance.com>
+
+
+.. note::
+
+ The contentmanager module has been included in the standard library on a
+ :term:`provisional basis <provisional package>`. Backwards incompatible
+ changes (up to and including removal of the module) may occur if deemed
+ necessary by the core developers.
+
+.. versionadded:: 3.4
+ as a :term:`provisional module <provisional package>`.
+
+The :mod:`~email.message` module provides a class that can represent an
+arbitrary email message. That basic message model has a useful and flexible
+API, but it provides only a lower-level API for interacting with the generic
+parts of a message (the headers, generic header parameters, and the payload,
+which may be a list of sub-parts). This module provides classes and tools
+that provide an enhanced and extensible API for dealing with various specific
+types of content, including the ability to retrieve the content of the message
+as a specialized object type rather than as a simple bytes object. The module
+automatically takes care of the RFC-specified MIME details (required headers
+and parameters, etc.) for the certain common content types content properties,
+and support for additional types can be added by an application using the
+extension mechanisms.
+
+This module defines the eponymous "Content Manager" classes. The base
+:class:`.ContentManager` class defines an API for registering content
+management functions which extract data from ``Message`` objects or insert data
+and headers into ``Message`` objects, thus providing a way of converting
+between ``Message`` objects containing data and other representations of that
+data (Python data types, specialized Python objects, external files, etc). The
+module also defines one concrete content manager: :data:`raw_data_manager`
+converts between MIME content types and ``str`` or ``bytes`` data. It also
+provides a convenient API for managing the MIME parameters when inserting
+content into ``Message``\ s. It also handles inserting and extracting
+``Message`` objects when dealing with the ``message/rfc822`` content type.
+
+Another part of the enhanced interface is subclasses of
+:class:`~email.message.Message` that provide new convenience API functions,
+including convenience methods for calling the Content Managers derived from
+this module.
+
+.. note::
+
+ Although :class:`.EmailMessage` and :class:`.MIMEPart` are currently
+ documented in this module because of the provisional nature of the code, the
+ implementation lives in the :mod:`email.message` module.
+
+.. currentmodule:: email.message
+
+.. class:: EmailMessage(policy=default)
+
+ If *policy* is specified (it must be an instance of a :mod:`~email.policy`
+ class) use the rules it specifies to udpate and serialize the representation
+ of the message. If *policy* is not set, use the
+ :class:`~email.policy.default` policy, which follows the rules of the email
+ RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses
+ the Python standard ``\n`` line endings). For more information see the
+ :mod:`~email.policy` documentation.
+
+ This class is a subclass of :class:`~email.message.Message`. It adds
+ the following methods:
+
+
+ .. attribute:: is_attachment
+
+ Set to ``True`` if there is a :mailheader:`Content-Disposition` header
+ and its (case insensitive) value is ``attachment``, ``False`` otherwise.
+
+
+ .. method:: get_body(preferencelist=('related', 'html', 'plain'))
+
+ Return the MIME part that is the best candidate to be the "body" of the
+ message.
+
+ *preferencelist* must be a sequence of strings from the set ``related``,
+ ``html``, and ``plain``, and indicates the order of preference for the
+ content type of the part returned.
+
+ Start looking for candidate matches with the object on which the
+ ``get_body`` method is called.
+
+ If ``related`` is not included in *preferencelist*, consider the root
+ part (or subpart of the root part) of any related encountered as a
+ candidate if the (sub-)part matches a preference.
+
+ When encountering a ``multipart/related``, check the ``start`` parameter
+ and if a part with a matching :mailheader:`Content-ID` is found, consider
+ only it when looking for candidate matches. Otherwise consider only the
+ first (default root) part of the ``multipart/related``.
+
+ If a part has a :mailheader:`Content-Disposition` header, only consider
+ the part a candidate match if the value of the header is ``inline``.
+
+ If none of the candidates matches any of the preferences in
+ *preferneclist*, return ``None``.
+
+ Notes: (1) For most applications the only *preferencelist* combinations
+ that really make sense are ``('plain',)``, ``('html', 'plain')``, and the
+ default, ``('related', 'html', 'plain')``. (2) Because matching starts
+ with the object on which ``get_body`` is called, calling ``get_body`` on
+ a ``multipart/related`` will return the object itself unless
+ *preferencelist* has a non-default value. (3) Messages (or message parts)
+ that do not specify a :mailheader:`Content-Type` or whose
+ :mailheader:`Content-Type` header is invalid will be treated as if they
+ are of type ``text/plain``, which may occasionally cause ``get_body`` to
+ return unexpected results.
+
+
+ .. method:: iter_attachments()
+
+ Return an iterator over all of the parts of the message that are not
+ candidate "body" parts. That is, skip the first occurrence of each of
+ ``text/plain``, ``text/html``, ``multipart/related``, or
+ ``multipart/alternative`` (unless they are explicitly marked as
+ attachments via :mailheader:`Content-Disposition: attachment`), and
+ return all remaining parts. When applied directly to a
+ ``multipart/related``, return an iterator over the all the related parts
+ except the root part (ie: the part pointed to by the ``start`` parameter,
+ or the first part if there is no ``start`` parameter or the ``start``
+ parameter doesn't match the :mailheader:`Content-ID` of any of the
+ parts). When applied directly to a ``multipart/alternative`` or a
+ non-``multipart``, return an empty iterator.
+
+
+ .. method:: iter_parts()
+
+ Return an iterator over all of the immediate sub-parts of the message,
+ which will be empty for a non-``multipart``. (See also
+ :meth:`~email.message.walk`.)
+
+
+ .. method:: get_content(*args, content_manager=None, **kw)
+
+ Call the ``get_content`` method of the *content_manager*, passing self
+ as the message object, and passing along any other arguments or keywords
+ as additional arguments. If *content_manager* is not specified, use
+ the ``content_manager`` specified by the current :mod:`~email.policy`.
+
+
+ .. method:: set_content(*args, content_manager=None, **kw)
+
+ Call the ``set_content`` method of the *content_manager*, passing self
+ as the message object, and passing along any other arguments or keywords
+ as additional arguments. If *content_manager* is not specified, use
+ the ``content_manager`` specified by the current :mod:`~email.policy`.
+
+
+ .. method:: make_related(boundary=None)
+
+ Convert a non-``multipart`` message into a ``multipart/related`` message,
+ moving any existing :mailheader:`Content-` headers and payload into a
+ (new) first part of the ``multipart``. If *boundary* is specified, use
+ it as the boundary string in the multipart, otherwise leave the boundary
+ to be automatically created when it is needed (for example, when the
+ message is serialized).
+
+
+ .. method:: make_alternative(boundary=None)
+
+ Convert a non-``multipart`` or a ``multipart/related`` into a
+ ``multipart/alternative``, moving any existing :mailheader:`Content-`
+ headers and payload into a (new) first part of the ``multipart``. If
+ *boundary* is specified, use it as the boundary string in the multipart,
+ otherwise leave the boundary to be automatically created when it is
+ needed (for example, when the message is serialized).
+
+
+ .. method:: make_mixed(boundary=None)
+
+ Convert a non-``multipart``, a ``multipart/related``, or a
+ ``multipart-alternative`` into a ``multipart/mixed``, moving any existing
+ :mailheader:`Content-` headers and payload into a (new) first part of the
+ ``multipart``. If *boundary* is specified, use it as the boundary string
+ in the multipart, otherwise leave the boundary to be automatically
+ created when it is needed (for example, when the message is serialized).
+
+
+ .. method:: add_related(*args, content_manager=None, **kw)
+
+ If the message is a ``multipart/related``, create a new message
+ object, pass all of the arguments to its :meth:`set_content` method,
+ and :meth:`~email.message.Message.attach` it to the ``multipart``. If
+ the message is a non-``multipart``, call :meth:`make_related` and then
+ proceed as above. If the message is any other type of ``multipart``,
+ raise a :exc:`TypeError`. If *content_manager* is not specified, use
+ the ``content_manager`` specified by the current :mod:`~email.policy`.
+ If the added part has no :mailheader:`Content-Disposition` header,
+ add one with the value ``inline``.
+
+
+ .. method:: add_alternative(*args, content_manager=None, **kw)
+
+ If the message is a ``multipart/alternative``, create a new message
+ object, pass all of the arguments to its :meth:`set_content` method, and
+ :meth:`~email.message.Message.attach` it to the ``multipart``. If the
+ message is a non-``multipart`` or ``multipart/related``, call
+ :meth:`make_alternative` and then proceed as above. If the message is
+ any other type of ``multipart``, raise a :exc:`TypeError`. If
+ *content_manager* is not specified, use the ``content_manager`` specified
+ by the current :mod:`~email.policy`.
+
+
+ .. method:: add_attachment(*args, content_manager=None, **kw)
+
+ If the message is a ``multipart/mixed``, create a new message object,
+ pass all of the arguments to its :meth:`set_content` method, and
+ :meth:`~email.message.Message.attach` it to the ``multipart``. If the
+ message is a non-``multipart``, ``multipart/related``, or
+ ``multipart/alternative``, call :meth:`make_mixed` and then proceed as
+ above. If *content_manager* is not specified, use the ``content_manager``
+ specified by the current :mod:`~email.policy`. If the added part
+ has no :mailheader:`Content-Disposition` header, add one with the value
+ ``attachment``. This method can be used both for explicit attachments
+ (:mailheader:`Content-Disposition: attachment` and ``inline`` attachments
+ (:mailheader:`Content-Disposition: inline`), by passing appropriate
+ options to the ``content_manager``.
+
+
+ .. method:: clear()
+
+ Remove the payload and all of the headers.
+
+
+ .. method:: clear_content()
+
+ Remove the payload and all of the :exc:`Content-` headers, leaving
+ all other headers intact and in their original order.
+
+
+.. class:: MIMEPart(policy=default)
+
+ This class represents a subpart of a MIME message. It is identical to
+ :class:`EmailMessage`, except that no :mailheader:`MIME-Version` headers are
+ added when :meth:`~EmailMessage.set_content` is called, since sub-parts do
+ not need their own :mailheader:`MIME-Version` headers.
+
+
+.. currentmodule:: email.contentmanager
+
+.. class:: ContentManager()
+
+ Base class for content managers. Provides the standard registry mechanisms
+ to register converters between MIME content and other representations, as
+ well as the ``get_content`` and ``set_content`` dispatch methods.
+
+
+ .. method:: get_content(msg, *args, **kw)
+
+ Look up a handler function based on the ``mimetype`` of *msg* (see next
+ paragraph), call it, passing through all arguments, and return the result
+ of the call. The expectation is that the handler will extract the
+ payload from *msg* and return an object that encodes information about
+ the extracted data.
+
+ To find the handler, look for the following keys in the registry,
+ stopping with the first one found:
+
+ * the string representing the full MIME type (``maintype/subtype``)
+ * the string representing the ``maintype``
+ * the empty string
+
+ If none of these keys produce a handler, raise a :exc:`KeyError` for the
+ full MIME type.
+
+
+ .. method:: set_content(msg, obj, *args, **kw)
+
+ If the ``maintype`` is ``multipart``, raise a :exc:`TypeError`; otherwise
+ look up a handler function based on the type of *obj* (see next
+ paragraph), call :meth:`~email.message.EmailMessage.clear_content` on the
+ *msg*, and call the handler function, passing through all arguments. The
+ expectation is that the handler will transform and store *obj* into
+ *msg*, possibly making other changes to *msg* as well, such as adding
+ various MIME headers to encode information needed to interpret the stored
+ data.
+
+ To find the handler, obtain the type of *obj* (``typ = type(obj)``), and
+ look for the following keys in the registry, stopping with the first one
+ found:
+
+ * the type itself (``typ``)
+ * the type's fully qualified name (``typ.__module__ + '.' +
+ typ.__qualname__``).
+ * the type's qualname (``typ.__qualname__``)
+ * the type's name (``typ.__name__``).
+
+ If none of the above match, repeat all of the checks above for each of
+ the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key
+ yields a handler, check for a handler for the key ``None``. If there is
+ no handler for ``None``, raise a :exc:`KeyError` for the fully
+ qualified name of the type.
+
+ Also add a :mailheader:`MIME-Version` header if one is not present (see
+ also :class:`.MIMEPart`).
+
+
+ .. method:: add_get_handler(key, handler)
+
+ Record the function *handler* as the handler for *key*. For the possible
+ values of *key*, see :meth:`get_content`.
+
+
+ .. method:: add_set_handler(typekey, handler)
+
+ Record *handler* as the function to call when an object of a type
+ matching *typekey* is passed to :meth:`set_content`. For the possible
+ values of *typekey*, see :meth:`set_content`.
+
+
+Content Manager Instances
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently the email package provides only one concrete content manager,
+:data:`raw_data_manager`, although more may be added in the future.
+:data:`raw_data_manager` is the
+:attr:`~email.policy.EmailPolicy.content_manager` provided by
+:attr:`~email.policy.EmailPolicy` and its derivatives.
+
+
+.. data:: raw_data_manager
+
+ This content manager provides only a minimum interface beyond that provided
+ by :class:`~email.message.Message` itself: it deals only with text, raw
+ byte strings, and :class:`~email.message.Message` objects. Nevertheless, it
+ provides significant advantages compared to the base API: ``get_content`` on
+ a text part will return a unicode string without the application needing to
+ manually decode it, ``set_content`` provides a rich set of options for
+ controlling the headers added to a part and controlling the content transfer
+ encoding, and it enables the use of the various ``add_`` methods, thereby
+ simplifying the creation of multipart messages.
+
+ .. method:: get_content(msg, errors='replace')
+
+ Return the payload of the part as either a string (for ``text`` parts), a
+ :class:`~email.message.EmailMessage` object (for ``message/rfc822``
+ parts), or a ``bytes`` object (for all other non-multipart types). Raise
+ a :exc:`KeyError` if called on a ``multipart``. If the part is a
+ ``text`` part and *errors* is specified, use it as the error handler when
+ decoding the payload to unicode. The default error handler is
+ ``replace``.
+
+ .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \
+ cte=None, \
+ disposition=None, filename=None, cid=None, \
+ params=None, headers=None)
+ set_content(msg, <'bytes'>, maintype, subtype, cte="base64", \
+ disposition=None, filename=None, cid=None, \
+ params=None, headers=None)
+ set_content(msg, <'Message'>, cte=None, \
+ disposition=None, filename=None, cid=None, \
+ params=None, headers=None)
+ set_content(msg, <'list'>, subtype='mixed', \
+ disposition=None, filename=None, cid=None, \
+ params=None, headers=None)
+
+ Add headers and payload to *msg*:
+
+ Add a :mailheader:`Content-Type` header with a ``maintype/subtype``
+ value.
+
+ * For ``str``, set the MIME ``maintype`` to ``text``, and set the
+ subtype to *subtype* if it is specified, or ``plain`` if it is not.
+ * For ``bytes``, use the specified *maintype* and *subtype*, or
+ raise a :exc:`TypeError` if they are not specified.
+ * For :class:`~email.message.Message` objects, set the maintype to
+ ``message``, and set the subtype to *subtype* if it is specified
+ or ``rfc822`` if it is not. If *subtype* is ``partial``, raise an
+ error (``bytes`` objects must be used to construct
+ ``message/partial`` parts).
+ * For *<'list'>*, which should be a list of
+ :class:`~email.message.Message` objects, set the ``maintype`` to
+ ``multipart``, and the ``subtype`` to *subtype* if it is
+ specified, and ``mixed`` if it is not. If the message parts in
+ the *<'list'>* have :mailheader:`MIME-Version` headers, remove
+ them.
+
+ If *charset* is provided (which is valid only for ``str``), encode the
+ string to bytes using the specified character set. The default is
+ ``utf-8``. If the specified *charset* is a known alias for a standard
+ MIME charset name, use the standard charset instead.
+
+ If *cte* is set, encode the payload using the specified content transfer
+ encoding, and set the :mailheader:`Content-Transfer-Endcoding` header to
+ that value. For ``str`` objects, if it is not set use heuristics to
+ determine the most compact encoding. Possible values for *cte* are
+ ``quoted-printable``, ``base64``, ``7bit``, ``8bit``, and ``binary``.
+ If the input cannot be encoded in the specified encoding (eg: ``7bit``),
+ raise a :exc:`ValueError`. For :class:`~email.message.Message`, per
+ :rfc:`2046`, raise an error if a *cte* of ``quoted-printable`` or
+ ``base64`` is requested for *subtype* ``rfc822``, and for any *cte*
+ other than ``7bit`` for *subtype* ``external-body``. For
+ ``message/rfc822``, use ``8bit`` if *cte* is not specified. For all
+ other values of *subtype*, use ``7bit``.
+
+ .. note:: A *cte* of ``binary`` does not actually work correctly yet.
+ The ``Message`` object as modified by ``set_content`` is correct, but
+ :class:`~email.generator.BytesGenerator` does not serialize it
+ correctly.
+
+ If *disposition* is set, use it as the value of the
+ :mailheader:`Content-Disposition` header. If not specified, and
+ *filename* is specified, add the header with the value ``attachment``.
+ If it is not specified and *filename* is also not specified, do not add
+ the header. The only valid values for *disposition* are ``attachment``
+ and ``inline``.
+
+ If *filename* is specified, use it as the value of the ``filename``
+ parameter of the :mailheader:`Content-Disposition` header. There is no
+ default.
+
+ If *cid* is specified, add a :mailheader:`Content-ID` header with
+ *cid* as its value.
+
+ If *params* is specified, iterate its ``items`` method and use the
+ resulting ``(key, value)`` pairs to set additional paramters on the
+ :mailheader:`Content-Type` header.
+
+ If *headers* is specified and is a list of strings of the form
+ ``headername: headervalue`` or a list of ``header`` objects
+ (distinguised from strings by having a ``name`` attribute), add the
+ headers to *msg*.
diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst
index 3075e86..58f708c 100644
--- a/Doc/library/email.message.rst
+++ b/Doc/library/email.message.rst
@@ -33,23 +33,31 @@ Here are the methods of the :class:`Message` class:
.. class:: Message(policy=compat32)
- The *policy* argument determiens the :mod:`~email.policy` that will be used
- to update the message model. The default value, :class:`compat32
- <email.policy.Compat32>` maintains backward compatibility with the
- Python 3.2 version of the email package. For more information see the
+ If *policy* is specified (it must be an instance of a :mod:`~email.policy`
+ class) use the rules it specifies to udpate and serialize the representation
+ of the message. If *policy* is not set, use the :class:`compat32
+ <email.policy.Compat32>` policy, which maintains backward compatibility with
+ the Python 3.2 version of the email package. For more information see the
:mod:`~email.policy` documentation.
.. versionchanged:: 3.3 The *policy* keyword argument was added.
- .. method:: as_string(unixfrom=False, maxheaderlen=0)
+ .. method:: as_string(unixfrom=False, maxheaderlen=0, policy=None)
Return the entire message flattened as a string. When optional *unixfrom*
- is ``True``, the envelope header is included in the returned string.
- *unixfrom* defaults to ``False``. Flattening the message may trigger
- changes to the :class:`Message` if defaults need to be filled in to
- complete the transformation to a string (for example, MIME boundaries may
- be generated or modified).
+ is true, the envelope header is included in the returned string.
+ *unixfrom* defaults to ``False``. For backward compabitility reasons,
+ *maxheaderlen* defaults to ``0``, so if you want a different value you
+ must override it explicitly (the value specified for *max_line_length* in
+ the policy will be ignored by this method). The *policy* argument may be
+ used to override the default policy obtained from the message instance.
+ This can be used to control some of the formatting produced by the
+ method, since the specified *policy* will be passed to the ``Generator``.
+
+ Flattening the message may trigger changes to the :class:`Message` if
+ defaults need to be filled in to complete the transformation to a string
+ (for example, MIME boundaries may be generated or modified).
Note that this method is provided as a convenience and may not always
format the message the way you want. For example, by default it does
@@ -65,10 +73,58 @@ Here are the methods of the :class:`Message` class:
g.flatten(msg)
text = fp.getvalue()
+ If the message object contains binary data that is not encoded according
+ to RFC standards, the non-compliant data will be replaced by unicode
+ "unknown character" code points. (See also :meth:`.as_bytes` and
+ :class:`~email.generator.BytesGenerator`.)
+
+ .. versionchanged:: 3.4 the *policy* keyword argument was added.
+
.. method:: __str__()
- Equivalent to ``as_string(unixfrom=True)``.
+ Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a
+ string containing the formatted message.
+
+
+ .. method:: as_bytes(unixfrom=False, policy=None)
+
+ Return the entire message flattened as a bytes object. When optional
+ *unixfrom* is true, the envelope header is included in the returned
+ string. *unixfrom* defaults to ``False``. The *policy* argument may be
+ used to override the default policy obtained from the message instance.
+ This can be used to control some of the formatting produced by the
+ method, since the specified *policy* will be passed to the
+ ``BytesGenerator``.
+
+ Flattening the message may trigger changes to the :class:`Message` if
+ defaults need to be filled in to complete the transformation to a string
+ (for example, MIME boundaries may be generated or modified).
+
+ Note that this method is provided as a convenience and may not always
+ format the message the way you want. For example, by default it does
+ not do the mangling of lines that begin with ``From`` that is
+ required by the unix mbox format. For more flexibility, instantiate a
+ :class:`~email.generator.BytesGenerator` instance and use its
+ :meth:`~email.generator.BytesGenerator.flatten` method directly.
+ For example::
+
+ from io import BytesIO
+ from email.generator import BytesGenerator
+ fp = BytesIO()
+ g = BytesGenerator(fp, mangle_from_=True, maxheaderlen=60)
+ g.flatten(msg)
+ text = fp.getvalue()
+
+ .. versionadded:: 3.4
+
+
+ .. method:: __bytes__()
+
+ Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a
+ bytes object containing the formatted message.
+
+ .. versionadded:: 3.4
.. method:: is_multipart()
@@ -410,7 +466,8 @@ Here are the methods of the :class:`Message` class:
to ``False``.
- .. method:: set_param(param, value, header='Content-Type', requote=True, charset=None, language='')
+ .. method:: set_param(param, value, header='Content-Type', requote=True, \
+ charset=None, language='', replace=False)
Set a parameter in the :mailheader:`Content-Type` header. If the
parameter already exists in the header, its value will be replaced with
@@ -427,6 +484,12 @@ Here are the methods of the :class:`Message` class:
language, defaulting to the empty string. Both *charset* and *language*
should be strings.
+ If *replace* is ``False`` (the default) the header is moved to the
+ end of the list of headers. If *replace* is ``True``, the header
+ will be updated in place.
+
+ .. versionchanged:: 3.4 ``replace`` keyword was added.
+
.. method:: del_param(param, header='content-type', requote=True)
diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst
index cb2023c..97358f0 100644
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -97,6 +97,17 @@ correct line separator characters when creating the binary string to feed into
``sendmail's`` ``stdin``, where the default policy would use ``\n`` line
separators.
+Some email package methods accept a *policy* keyword argument, allowing the
+policy to be overridden for that method. For example, the following code uses
+the :meth:`~email.message.Message.as_string` method of the *msg* object from
+the previous example and writes the message to a file using the native line
+separators for the platform on which it is running::
+
+ >>> import os
+ >>> with open('converted.txt', 'wb') as f:
+ ... f.write(msg.as_bytes(policy=msg.policy.clone(linesep=os.linesep)))
+ 17
+
Policy objects can also be combined using the addition operator, producing a
policy object whose settings are a combination of the non-default values of the
summed objects::
@@ -360,7 +371,7 @@ added matters. To illustrate::
to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs.
This policy adds new header parsing and folding algorithms. Instead of
- simple strings, headers are custom objects with custom attributes depending
+ simple strings, headers are ``str`` subclasses with attributes that depend
on the type of the field. The parsing and folding algorithm fully implement
:rfc:`2047` and :rfc:`5322`.
@@ -397,6 +408,20 @@ added matters. To illustrate::
fields are treated as unstructured. This list will be completed before
the extension is marked stable.)
+ .. attribute:: content_manager
+
+ An object with at least two methods: get_content and set_content. When
+ the :meth:`~email.message.Message.get_content` or
+ :meth:`~email.message.Message.set_content` method of a
+ :class:`~email.message.Message` object is called, it calls the
+ corresponding method of this object, passing it the message object as its
+ first argument, and any arguments or keywords that were passed to it as
+ additional arguments. By default ``content_manager`` is set to
+ :data:`~email.contentmanager.raw_data_manager`.
+
+ .. versionadded:: 3.4
+
+
The class provides the following concrete implementations of the abstract
methods of :class:`Policy`:
@@ -416,7 +441,7 @@ added matters. To illustrate::
The name is returned unchanged. If the input value has a ``name``
attribute and it matches *name* ignoring case, the value is returned
unchanged. Otherwise the *name* and *value* are passed to
- ``header_factory``, and the resulting custom header object is returned as
+ ``header_factory``, and the resulting header object is returned as
the value. In this case a ``ValueError`` is raised if the input value
contains CR or LF characters.
@@ -424,7 +449,7 @@ added matters. To illustrate::
If the value has a ``name`` attribute, it is returned to unmodified.
Otherwise the *name*, and the *value* with any CR or LF characters
- removed, are passed to the ``header_factory``, and the resulting custom
+ removed, are passed to the ``header_factory``, and the resulting
header object is returned. Any surrogateescaped bytes get turned into
the unicode unknown-character glyph.
@@ -434,9 +459,9 @@ added matters. To illustrate::
A value is considered to be a 'source value' if and only if it does not
have a ``name`` attribute (having a ``name`` attribute means it is a
header object of some sort). If a source value needs to be refolded
- according to the policy, it is converted into a custom header object by
+ according to the policy, it is converted into a header object by
passing the *name* and the *value* with any CR and LF characters removed
- to the ``header_factory``. Folding of a custom header object is done by
+ to the ``header_factory``. Folding of a header object is done by
calling its ``fold`` method with the current policy.
Source values are split into lines using :meth:`~str.splitlines`. If
@@ -491,23 +516,23 @@ With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of
the email package is changed from the Python 3.2 API in the following ways:
* Setting a header on a :class:`~email.message.Message` results in that
- header being parsed and a custom header object created.
+ header being parsed and a header object created.
* Fetching a header value from a :class:`~email.message.Message` results
- in that header being parsed and a custom header object created and
+ in that header being parsed and a header object created and
returned.
- * Any custom header object, or any header that is refolded due to the
+ * Any header object, or any header that is refolded due to the
policy settings, is folded using an algorithm that fully implements the
RFC folding algorithms, including knowing where encoded words are required
and allowed.
From the application view, this means that any header obtained through the
-:class:`~email.message.Message` is a custom header object with custom
+:class:`~email.message.Message` is a header object with extra
attributes, whose string value is the fully decoded unicode value of the
header. Likewise, a header may be assigned a new value, or a new header
created, using a unicode string, and the policy will take care of converting
the unicode string into the correct RFC encoded form.
-The custom header objects and their attributes are described in
+The header objects and their attributes are described in
:mod:`~email.headerregistry`.
diff --git a/Doc/library/email.rst b/Doc/library/email.rst
index a6cbbce..331d2ef 100644
--- a/Doc/library/email.rst
+++ b/Doc/library/email.rst
@@ -53,6 +53,7 @@ Contents of the :mod:`email` package documentation:
email.generator.rst
email.policy.rst
email.headerregistry.rst
+ email.contentmanager.rst
email.mime.rst
email.header.rst
email.charset.rst
diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst
new file mode 100644
index 0000000..8012f51
--- /dev/null
+++ b/Doc/library/ensurepip.rst
@@ -0,0 +1,131 @@
+:mod:`ensurepip` --- Bootstrapping the ``pip`` installer
+========================================================
+
+.. module:: ensurepip
+ :synopsis: Bootstrapping the ``pip`` installer into an existing Python
+ installation or virtual environment.
+
+.. versionadded:: 3.4
+
+The :mod:`ensurepip` package provides support for bootstrapping the ``pip``
+installer into an existing Python installation or virtual environment. This
+bootstrapping approach reflects the fact that ``pip`` is an independent
+project with its own release cycle, and the latest available stable version
+is bundled with maintenance and feature releases of the CPython reference
+interpreter.
+
+In most cases, end users of Python shouldn't need to invoke this module
+directly (as ``pip`` should be bootstrapped by default), but it may be
+needed if installing ``pip`` was skipped when installing Python (or
+when creating a virtual environment) or after explicitly uninstalling
+``pip``.
+
+.. note::
+
+ This module *does not* access the internet. All of the components
+ needed to bootstrap ``pip`` are included as internal parts of the
+ package.
+
+.. seealso::
+
+ :ref:`installing-index`
+ The end user guide for installing Python packages
+
+ :pep:`453`: Explicit bootstrapping of pip in Python installations
+ The original rationale and specification for this module.
+
+
+Command line interface
+----------------------
+
+The command line interface is invoked using the interpreter's ``-m`` switch.
+
+The simplest possible invocation is::
+
+ python -m ensurepip
+
+This invocation will install ``pip`` if it is not already installed,
+but otherwise does nothing. To ensure the installed version of ``pip``
+is at least as recent as the one bundled with ``ensurepip``, pass the
+``--upgrade`` option::
+
+ python -m ensurepip --upgrade
+
+By default, ``pip`` is installed into the current virtual environment
+(if one is active) or into the system site packages (if there is no
+active virtual environment). The installation location can be controlled
+through two additional command line options:
+
+* ``--root <dir>``: Installs ``pip`` relative to the given root directory
+ rather than the root of the currently active virtual environment (if any)
+ or the default root for the current Python installation.
+* ``--user``: Installs ``pip`` into the user site packages directory rather
+ than globally for the current Python installation (this option is not
+ permitted inside an active virtual environment).
+
+By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where
+X.Y stands for the version of Python used to invoke ``ensurepip``). The
+scripts installed can be controlled through two additional command line
+options:
+
+* ``--altinstall``: if an alternate installation is requested, the ``pipX``
+ script will *not* be installed.
+
+* ``--default-pip``: if a "default pip" installation is requested, the
+ ``pip`` script will be installed in addition to the two regular scripts.
+
+Providing both of the script selection options will trigger an exception.
+
+
+Module API
+----------
+
+:mod:`ensurepip` exposes two functions for programmatic use:
+
+.. function:: version()
+
+ Returns a string specifying the bundled version of pip that will be
+ installed when bootstrapping an environment.
+
+.. function:: bootstrap(root=None, upgrade=False, user=False, \
+ altinstall=False, default_pip=False, \
+ verbosity=0)
+
+ Bootstraps ``pip`` into the current or designated environment.
+
+ *root* specifies an alternative root directory to install relative to.
+ If *root* is None, then installation uses the default install location
+ for the current environment.
+
+ *upgrade* indicates whether or not to upgrade an existing installation
+ of an earlier version of ``pip`` to the bundled version.
+
+ *user* indicates whether to use the user scheme rather than installing
+ globally.
+
+ By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where
+ X.Y stands for the current version of Python).
+
+ If *altinstall* is set, then ``pipX`` will *not* be installed.
+
+ If *default_pip* is set, then ``pip`` will be installed in addition to
+ the two regular scripts.
+
+ Setting both *altinstall* and *default_pip* will trigger
+ :exc:`ValueError`.
+
+ *verbosity* controls the level of output to :data:`sys.stdout` from the
+ bootstrapping operation.
+
+ .. note::
+
+ The bootstrapping process has side effects on both ``sys.path`` and
+ ``os.environ``. Invoking the command line interface in a subprocess
+ instead allows these side effects to be avoided.
+
+ .. note::
+
+ The bootstrapping process may install additional modules required by
+ ``pip``, but other software should not assume those dependencies will
+ always be present by default (as the dependencies may be removed in a
+ future version of ``pip``).
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
new file mode 100644
index 0000000..acdcf7f
--- /dev/null
+++ b/Doc/library/enum.rst
@@ -0,0 +1,749 @@
+:mod:`enum` --- Support for enumerations
+========================================
+
+.. module:: enum
+ :synopsis: Implementation of an enumeration class.
+
+.. :moduleauthor:: Ethan Furman <ethan@stoneleaf.us>
+.. :sectionauthor:: Barry Warsaw <barry@python.org>,
+.. :sectionauthor:: Eli Bendersky <eliben@gmail.com>,
+.. :sectionauthor:: Ethan Furman <ethan@stoneleaf.us>
+
+.. versionadded:: 3.4
+
+**Source code:** :source:`Lib/enum.py`
+
+----------------
+
+An enumeration is a set of symbolic names (members) bound to unique,
+constant values. Within an enumeration, the members can be compared
+by identity, and the enumeration itself can be iterated over.
+
+
+Module Contents
+---------------
+
+This module defines two enumeration classes that can be used to define unique
+sets of names and values: :class:`Enum` and :class:`IntEnum`. It also defines
+one decorator, :func:`unique`.
+
+.. class:: Enum
+
+ Base class for creating enumerated constants. See section
+ `Functional API`_ for an alternate construction syntax.
+
+.. class:: IntEnum
+
+ Base class for creating enumerated constants that are also
+ subclasses of :class:`int`.
+
+.. function:: unique
+
+ Enum class decorator that ensures only one name is bound to any one value.
+
+
+Creating an Enum
+----------------
+
+Enumerations are created using the :keyword:`class` syntax, which makes them
+easy to read and write. An alternative creation method is described in
+`Functional API`_. To define an enumeration, subclass :class:`Enum` as
+follows::
+
+ >>> from enum import Enum
+ >>> class Color(Enum):
+ ... red = 1
+ ... green = 2
+ ... blue = 3
+ ...
+
+.. note:: Nomenclature
+
+ - The class :class:`Color` is an *enumeration* (or *enum*)
+ - The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are
+ *enumeration members* (or *enum members*).
+ - The enum members have *names* and *values* (the name of
+ :attr:`Color.red` is ``red``, the value of :attr:`Color.blue` is
+ ``3``, etc.)
+
+.. note::
+
+ Even though we use the :keyword:`class` syntax to create Enums, Enums
+ are not normal Python classes. See `How are Enums different?`_ for
+ more details.
+
+Enumeration members have human readable string representations::
+
+ >>> print(Color.red)
+ Color.red
+
+...while their ``repr`` has more information::
+
+ >>> print(repr(Color.red))
+ <Color.red: 1>
+
+The *type* of an enumeration member is the enumeration it belongs to::
+
+ >>> type(Color.red)
+ <enum 'Color'>
+ >>> isinstance(Color.green, Color)
+ True
+ >>>
+
+Enum members also have a property that contains just their item name::
+
+ >>> print(Color.red.name)
+ red
+
+Enumerations support iteration, in definition order::
+
+ >>> class Shake(Enum):
+ ... vanilla = 7
+ ... chocolate = 4
+ ... cookies = 9
+ ... mint = 3
+ ...
+ >>> for shake in Shake:
+ ... print(shake)
+ ...
+ Shake.vanilla
+ Shake.chocolate
+ Shake.cookies
+ Shake.mint
+
+Enumeration members are hashable, so they can be used in dictionaries and sets::
+
+ >>> apples = {}
+ >>> apples[Color.red] = 'red delicious'
+ >>> apples[Color.green] = 'granny smith'
+ >>> apples == {Color.red: 'red delicious', Color.green: 'granny smith'}
+ True
+
+
+Programmatic access to enumeration members and their attributes
+---------------------------------------------------------------
+
+Sometimes it's useful to access members in enumerations programmatically (i.e.
+situations where ``Color.red`` won't do because the exact color is not known
+at program-writing time). ``Enum`` allows such access::
+
+ >>> Color(1)
+ <Color.red: 1>
+ >>> Color(3)
+ <Color.blue: 3>
+
+If you want to access enum members by *name*, use item access::
+
+ >>> Color['red']
+ <Color.red: 1>
+ >>> Color['green']
+ <Color.green: 2>
+
+If you have an enum member and need its :attr:`name` or :attr:`value`::
+
+ >>> member = Color.red
+ >>> member.name
+ 'red'
+ >>> member.value
+ 1
+
+
+Duplicating enum members and values
+-----------------------------------
+
+Having two enum members with the same name is invalid::
+
+ >>> class Shape(Enum):
+ ... square = 2
+ ... square = 3
+ ...
+ Traceback (most recent call last):
+ ...
+ TypeError: Attempted to reuse key: 'square'
+
+However, two enum members are allowed to have the same value. Given two members
+A and B with the same value (and A defined first), B is an alias to A. By-value
+lookup of the value of A and B will return A. By-name lookup of B will also
+return A::
+
+ >>> class Shape(Enum):
+ ... square = 2
+ ... diamond = 1
+ ... circle = 3
+ ... alias_for_square = 2
+ ...
+ >>> Shape.square
+ <Shape.square: 2>
+ >>> Shape.alias_for_square
+ <Shape.square: 2>
+ >>> Shape(2)
+ <Shape.square: 2>
+
+.. note::
+
+ Attempting to create a member with the same name as an already
+ defined attribute (another member, a method, etc.) or attempting to create
+ an attribute with the same name as a member is not allowed.
+
+
+Ensuring unique enumeration values
+----------------------------------
+
+By default, enumerations allow multiple names as aliases for the same value.
+When this behavior isn't desired, the following decorator can be used to
+ensure each value is used only once in the enumeration:
+
+.. decorator:: unique
+
+A :keyword:`class` decorator specifically for enumerations. It searches an
+enumeration's :attr:`__members__` gathering any aliases it finds; if any are
+found :exc:`ValueError` is raised with the details::
+
+ >>> from enum import Enum, unique
+ >>> @unique
+ ... class Mistake(Enum):
+ ... one = 1
+ ... two = 2
+ ... three = 3
+ ... four = 3
+ ...
+ Traceback (most recent call last):
+ ...
+ ValueError: duplicate values found in <enum 'Mistake'>: four -> three
+
+
+Iteration
+---------
+
+Iterating over the members of an enum does not provide the aliases::
+
+ >>> list(Shape)
+ [<Shape.square: 2>, <Shape.diamond: 1>, <Shape.circle: 3>]
+
+The special attribute ``__members__`` is an ordered dictionary mapping names
+to members. It includes all names defined in the enumeration, including the
+aliases::
+
+ >>> for name, member in Shape.__members__.items():
+ ... name, member
+ ...
+ ('square', <Shape.square: 2>)
+ ('diamond', <Shape.diamond: 1>)
+ ('circle', <Shape.circle: 3>)
+ ('alias_for_square', <Shape.square: 2>)
+
+The ``__members__`` attribute can be used for detailed programmatic access to
+the enumeration members. For example, finding all the aliases::
+
+ >>> [name for name, member in Shape.__members__.items() if member.name != name]
+ ['alias_for_square']
+
+
+Comparisons
+-----------
+
+Enumeration members are compared by identity::
+
+ >>> Color.red is Color.red
+ True
+ >>> Color.red is Color.blue
+ False
+ >>> Color.red is not Color.blue
+ True
+
+Ordered comparisons between enumeration values are *not* supported. Enum
+members are not integers (but see `IntEnum`_ below)::
+
+ >>> Color.red < Color.blue
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ TypeError: unorderable types: Color() < Color()
+
+Equality comparisons are defined though::
+
+ >>> Color.blue == Color.red
+ False
+ >>> Color.blue != Color.red
+ True
+ >>> Color.blue == Color.blue
+ True
+
+Comparisons against non-enumeration values will always compare not equal
+(again, :class:`IntEnum` was explicitly designed to behave differently, see
+below)::
+
+ >>> Color.blue == 2
+ False
+
+
+Allowed members and attributes of enumerations
+----------------------------------------------
+
+The examples above use integers for enumeration values. Using integers is
+short and handy (and provided by default by the `Functional API`_), but not
+strictly enforced. In the vast majority of use-cases, one doesn't care what
+the actual value of an enumeration is. But if the value *is* important,
+enumerations can have arbitrary values.
+
+Enumerations are Python classes, and can have methods and special methods as
+usual. If we have this enumeration::
+
+ >>> class Mood(Enum):
+ ... funky = 1
+ ... happy = 3
+ ...
+ ... def describe(self):
+ ... # self is the member here
+ ... return self.name, self.value
+ ...
+ ... def __str__(self):
+ ... return 'my custom str! {0}'.format(self.value)
+ ...
+ ... @classmethod
+ ... def favorite_mood(cls):
+ ... # cls here is the enumeration
+ ... return cls.happy
+ ...
+
+Then::
+
+ >>> Mood.favorite_mood()
+ <Mood.happy: 3>
+ >>> Mood.happy.describe()
+ ('happy', 3)
+ >>> str(Mood.funky)
+ 'my custom str! 1'
+
+The rules for what is allowed are as follows: _sunder_ names (starting and
+ending with a single underscore) are reserved by enum and cannot be used;
+all other attributes defined within an enumeration will become members of this
+enumeration, with the exception of *__dunder__* names and descriptors (methods
+are also descriptors).
+
+Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then
+whatever value(s) were given to the enum member will be passed into those
+methods. See `Planet`_ for an example.
+
+
+Restricted subclassing of enumerations
+--------------------------------------
+
+Subclassing an enumeration is allowed only if the enumeration does not define
+any members. So this is forbidden::
+
+ >>> class MoreColor(Color):
+ ... pink = 17
+ ...
+ Traceback (most recent call last):
+ ...
+ TypeError: Cannot extend enumerations
+
+But this is allowed::
+
+ >>> class Foo(Enum):
+ ... def some_behavior(self):
+ ... pass
+ ...
+ >>> class Bar(Foo):
+ ... happy = 1
+ ... sad = 2
+ ...
+
+Allowing subclassing of enums that define members would lead to a violation of
+some important invariants of types and instances. On the other hand, it makes
+sense to allow sharing some common behavior between a group of enumerations.
+(See `OrderedEnum`_ for an example.)
+
+
+Pickling
+--------
+
+Enumerations can be pickled and unpickled::
+
+ >>> from test.test_enum import Fruit
+ >>> from pickle import dumps, loads
+ >>> Fruit.tomato is loads(dumps(Fruit.tomato))
+ True
+
+The usual restrictions for pickling apply: picklable enums must be defined in
+the top level of a module, since unpickling requires them to be importable
+from that module.
+
+.. note::
+
+ With pickle protocol version 4 it is possible to easily pickle enums
+ nested in other classes.
+
+It is possible to modify how Enum members are pickled/unpickled by defining
+:meth:`__reduce_ex__` in the enumeration class.
+
+
+Functional API
+--------------
+
+The :class:`Enum` class is callable, providing the following functional API::
+
+ >>> Animal = Enum('Animal', 'ant bee cat dog')
+ >>> Animal
+ <enum 'Animal'>
+ >>> Animal.ant
+ <Animal.ant: 1>
+ >>> Animal.ant.value
+ 1
+ >>> list(Animal)
+ [<Animal.ant: 1>, <Animal.bee: 2>, <Animal.cat: 3>, <Animal.dog: 4>]
+
+The semantics of this API resemble :class:`~collections.namedtuple`. The first
+argument of the call to :class:`Enum` is the name of the enumeration.
+
+The second argument is the *source* of enumeration member names. It can be a
+whitespace-separated string of names, a sequence of names, a sequence of
+2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to
+values. The last two options enable assigning arbitrary values to
+enumerations; the others auto-assign increasing integers starting with 1. A
+new class derived from :class:`Enum` is returned. In other words, the above
+assignment to :class:`Animal` is equivalent to::
+
+ >>> class Animals(Enum):
+ ... ant = 1
+ ... bee = 2
+ ... cat = 3
+ ... dog = 4
+ ...
+
+The reason for defaulting to ``1`` as the starting number and not ``0`` is
+that ``0`` is ``False`` in a boolean sense, but enum members all evaluate
+to ``True``.
+
+Pickling enums created with the functional API can be tricky as frame stack
+implementation details are used to try and figure out which module the
+enumeration is being created in (e.g. it will fail if you use a utility
+function in separate module, and also may not work on IronPython or Jython).
+The solution is to specify the module name explicitly as follows::
+
+ >>> Animals = Enum('Animals', 'ant bee cat dog', module=__name__)
+
+.. warning::
+
+ If ``module`` is not supplied, and Enum cannot determine what it is,
+ the new Enum members will not be unpicklable; to keep errors closer to
+ the source, pickling will be disabled.
+
+The new pickle protocol 4 also, in some circumstances, relies on
+:attr:`__qualname__` being set to the location where pickle will be able
+to find the class. For example, if the class was made available in class
+SomeData in the global scope::
+
+ >>> Animals = Enum('Animals', 'ant bee cat dog', qualname='SomeData.Animals')
+
+The complete signature is::
+
+ Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=<mixed-in class>)
+
+:value: What the new Enum class will record as its name.
+
+:names: The Enum members. This can be a whitespace or comma separated string
+ (values will start at 1)::
+
+ 'red green blue' | 'red,green,blue' | 'red, green, blue'
+
+ or an iterator of (name, value) pairs::
+
+ [('cyan', 4), ('magenta', 5), ('yellow', 6)]
+
+ or a mapping::
+
+ {'chartreuse': 7, 'sea_green': 11, 'rosemary': 42}
+
+:module: name of module where new Enum class can be found.
+
+:qualname: where in module new Enum class can be found.
+
+:type: type to mix in to new Enum class.
+
+
+Derived Enumerations
+--------------------
+
+IntEnum
+^^^^^^^
+
+A variation of :class:`Enum` is provided which is also a subclass of
+:class:`int`. Members of an :class:`IntEnum` can be compared to integers;
+by extension, integer enumerations of different types can also be compared
+to each other::
+
+ >>> from enum import IntEnum
+ >>> class Shape(IntEnum):
+ ... circle = 1
+ ... square = 2
+ ...
+ >>> class Request(IntEnum):
+ ... post = 1
+ ... get = 2
+ ...
+ >>> Shape == 1
+ False
+ >>> Shape.circle == 1
+ True
+ >>> Shape.circle == Request.post
+ True
+
+However, they still can't be compared to standard :class:`Enum` enumerations::
+
+ >>> class Shape(IntEnum):
+ ... circle = 1
+ ... square = 2
+ ...
+ >>> class Color(Enum):
+ ... red = 1
+ ... green = 2
+ ...
+ >>> Shape.circle == Color.red
+ False
+
+:class:`IntEnum` values behave like integers in other ways you'd expect::
+
+ >>> int(Shape.circle)
+ 1
+ >>> ['a', 'b', 'c'][Shape.circle]
+ 'b'
+ >>> [i for i in range(Shape.square)]
+ [0, 1]
+
+For the vast majority of code, :class:`Enum` is strongly recommended,
+since :class:`IntEnum` breaks some semantic promises of an enumeration (by
+being comparable to integers, and thus by transitivity to other
+unrelated enumerations). It should be used only in special cases where
+there's no other choice; for example, when integer constants are
+replaced with enumerations and backwards compatibility is required with code
+that still expects integers.
+
+
+Others
+^^^^^^
+
+While :class:`IntEnum` is part of the :mod:`enum` module, it would be very
+simple to implement independently::
+
+ class IntEnum(int, Enum):
+ pass
+
+This demonstrates how similar derived enumerations can be defined; for example
+a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`.
+
+Some rules:
+
+1. When subclassing :class:`Enum`, mix-in types must appear before
+ :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum`
+ example above.
+2. While :class:`Enum` can have members of any type, once you mix in an
+ additional type, all the members must have values of that type, e.g.
+ :class:`int` above. This restriction does not apply to mix-ins which only
+ add methods and don't specify another data type such as :class:`int` or
+ :class:`str`.
+3. When another data type is mixed in, the :attr:`value` attribute is *not the
+ same* as the enum member itself, although it is equivalent and will compare
+ equal.
+4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and
+ :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for
+ IntEnum) treat the enum member as its mixed-in type.
+5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in
+ type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or
+ :func:`repr` is desired use the `!s` or `!r` :class:`str` format codes.
+
+
+Interesting examples
+--------------------
+
+While :class:`Enum` and :class:`IntEnum` are expected to cover the majority of
+use-cases, they cannot cover them all. Here are recipes for some different
+types of enumerations that can be used directly, or as examples for creating
+one's own.
+
+
+AutoNumber
+^^^^^^^^^^
+
+Avoids having to specify the value for each enumeration member::
+
+ >>> class AutoNumber(Enum):
+ ... def __new__(cls):
+ ... value = len(cls.__members__) + 1
+ ... obj = object.__new__(cls)
+ ... obj._value_ = value
+ ... return obj
+ ...
+ >>> class Color(AutoNumber):
+ ... red = ()
+ ... green = ()
+ ... blue = ()
+ ...
+ >>> Color.green.value == 2
+ True
+
+.. note::
+
+ The :meth:`__new__` method, if defined, is used during creation of the Enum
+ members; it is then replaced by Enum's :meth:`__new__` which is used after
+ class creation for lookup of existing members. Due to the way Enums are
+ supposed to behave, there is no way to customize Enum's :meth:`__new__`.
+
+
+OrderedEnum
+^^^^^^^^^^^
+
+An ordered enumeration that is not based on :class:`IntEnum` and so maintains
+the normal :class:`Enum` invariants (such as not being comparable to other
+enumerations)::
+
+ >>> class OrderedEnum(Enum):
+ ... def __ge__(self, other):
+ ... if self.__class__ is other.__class__:
+ ... return self.value >= other.value
+ ... return NotImplemented
+ ... def __gt__(self, other):
+ ... if self.__class__ is other.__class__:
+ ... return self.value > other.value
+ ... return NotImplemented
+ ... def __le__(self, other):
+ ... if self.__class__ is other.__class__:
+ ... return self.value <= other.value
+ ... return NotImplemented
+ ... def __lt__(self, other):
+ ... if self.__class__ is other.__class__:
+ ... return self.value < other.value
+ ... return NotImplemented
+ ...
+ >>> class Grade(OrderedEnum):
+ ... A = 5
+ ... B = 4
+ ... C = 3
+ ... D = 2
+ ... F = 1
+ ...
+ >>> Grade.C < Grade.A
+ True
+
+
+DuplicateFreeEnum
+^^^^^^^^^^^^^^^^^
+
+Raises an error if a duplicate member name is found instead of creating an
+alias::
+
+ >>> class DuplicateFreeEnum(Enum):
+ ... def __init__(self, *args):
+ ... cls = self.__class__
+ ... if any(self.value == e.value for e in cls):
+ ... a = self.name
+ ... e = cls(self.value).name
+ ... raise ValueError(
+ ... "aliases not allowed in DuplicateFreeEnum: %r --> %r"
+ ... % (a, e))
+ ...
+ >>> class Color(DuplicateFreeEnum):
+ ... red = 1
+ ... green = 2
+ ... blue = 3
+ ... grene = 2
+ ...
+ Traceback (most recent call last):
+ ...
+ ValueError: aliases not allowed in DuplicateFreeEnum: 'grene' --> 'green'
+
+.. note::
+
+ This is a useful example for subclassing Enum to add or change other
+ behaviors as well as disallowing aliases. If the only desired change is
+ disallowing aliases, the :func:`unique` decorator can be used instead.
+
+
+Planet
+^^^^^^
+
+If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member
+will be passed to those methods::
+
+ >>> class Planet(Enum):
+ ... MERCURY = (3.303e+23, 2.4397e6)
+ ... VENUS = (4.869e+24, 6.0518e6)
+ ... EARTH = (5.976e+24, 6.37814e6)
+ ... MARS = (6.421e+23, 3.3972e6)
+ ... JUPITER = (1.9e+27, 7.1492e7)
+ ... SATURN = (5.688e+26, 6.0268e7)
+ ... URANUS = (8.686e+25, 2.5559e7)
+ ... NEPTUNE = (1.024e+26, 2.4746e7)
+ ... def __init__(self, mass, radius):
+ ... self.mass = mass # in kilograms
+ ... self.radius = radius # in meters
+ ... @property
+ ... def surface_gravity(self):
+ ... # universal gravitational constant (m3 kg-1 s-2)
+ ... G = 6.67300E-11
+ ... return G * self.mass / (self.radius * self.radius)
+ ...
+ >>> Planet.EARTH.value
+ (5.976e+24, 6378140.0)
+ >>> Planet.EARTH.surface_gravity
+ 9.802652743337129
+
+
+How are Enums different?
+------------------------
+
+Enums have a custom metaclass that affects many aspects of both derived Enum
+classes and their instances (members).
+
+
+Enum Classes
+^^^^^^^^^^^^
+
+The :class:`EnumMeta` metaclass is responsible for providing the
+:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that
+allow one to do things with an :class:`Enum` class that fail on a typical
+class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is
+responsible for ensuring that various other methods on the final :class:`Enum`
+class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`,
+:meth:`__str__` and :meth:`__repr__`)
+
+
+Enum Members (aka instances)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The most interesting thing about Enum members is that they are singletons.
+:class:`EnumMeta` creates them all while it is creating the :class:`Enum`
+class itself, and then puts a custom :meth:`__new__` in place to ensure
+that no new ones are ever instantiated by returning only the existing
+member instances.
+
+
+Finer Points
+^^^^^^^^^^^^
+
+Enum members are instances of an Enum class, and even though they are
+accessible as `EnumClass.member`, they are not accessible directly from
+the member::
+
+ >>> Color.red
+ <Color.red: 1>
+ >>> Color.red.blue
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'Color' object has no attribute 'blue'
+
+Likewise, the :attr:`__members__` is only available on the class.
+
+If you give your :class:`Enum` subclass extra methods, like the `Planet`_
+class above, those methods will show up in a :func:`dir` of the member,
+but not of the class::
+
+ >>> dir(Planet)
+ ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__']
+ >>> dir(Planet.EARTH)
+ ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value']
+
+A :meth:`__new__` method will only be used for the creation of the
+:class:`Enum` members -- after that it is replaced. This means if you wish to
+change how :class:`Enum` members are looked up you either have to write a
+helper function or a :func:`classmethod`.
diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst
index 6e4559c..0aac1bf 100644
--- a/Doc/library/exceptions.rst
+++ b/Doc/library/exceptions.rst
@@ -253,12 +253,22 @@ The following exceptions are the exceptions that are usually raised.
For exceptions that involve a file system path (such as :func:`open` or
:func:`os.unlink`), the exception instance will contain an additional
attribute, :attr:`filename`, which is the file name passed to the function.
+ For functions that involve two file system paths (such as
+ :func:`os.rename`), the exception instance will contain a second
+ :attr:`filename2` attribute corresponding to the second file name passed
+ to the function.
+
.. versionchanged:: 3.3
:exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`,
:exc:`VMSError`, :exc:`socket.error`, :exc:`select.error` and
:exc:`mmap.error` have been merged into :exc:`OSError`.
+ .. versionchanged:: 3.4
+ The :attr:`filename` attribute is now the original file name passed to
+ the function, instead of the name encoded to or decoded from the
+ filesystem encoding. Also, the :attr:`filename2` attribute was added.
+
.. exception:: OverflowError
diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst
index 3c33621..eb2016a 100644
--- a/Doc/library/faulthandler.rst
+++ b/Doc/library/faulthandler.rst
@@ -4,12 +4,14 @@
.. module:: faulthandler
:synopsis: Dump the Python traceback.
+.. versionadded:: 3.3
+
This module contains functions to dump Python tracebacks explicitly, on a fault,
after a timeout, or on a user signal. Call :func:`faulthandler.enable` to
install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`,
:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also
enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment
-variable or by using :option:`-X` ``faulthandler`` command line option.
+variable or by using the :option:`-X` ``faulthandler`` command line option.
The fault handler is compatible with system fault handlers like Apport or the
Windows fault handler. The module uses an alternative stack for signal handlers
@@ -27,6 +29,7 @@ tracebacks:
* Only the filename, the function name and the line number are
displayed. (no source code)
* It is limited to 100 frames and 100 threads.
+* The order is reversed: the most recent call is shown first.
By default, the Python traceback is written to :data:`sys.stderr`. To see
tracebacks, applications must be run in the terminal. A log file can
@@ -35,11 +38,9 @@ alternatively be passed to :func:`faulthandler.enable`.
The module is implemented in C, so tracebacks can be dumped on a crash or when
Python is deadlocked.
-.. versionadded:: 3.3
-
-Dump the traceback
-------------------
+Dumping the traceback
+---------------------
.. function:: dump_traceback(file=sys.stderr, all_threads=True)
@@ -68,8 +69,8 @@ Fault handler state
Check if the fault handler is enabled.
-Dump the tracebacks after a timeout
------------------------------------
+Dumping the tracebacks after a timeout
+--------------------------------------
.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False)
@@ -89,8 +90,8 @@ Dump the tracebacks after a timeout
Cancel the last call to :func:`dump_traceback_later`.
-Dump the traceback on a user signal
------------------------------------
+Dumping the traceback on a user signal
+--------------------------------------
.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False)
@@ -109,8 +110,8 @@ Dump the traceback on a user signal
Not available on Windows.
-File descriptor issue
----------------------
+Issue with file descriptors
+---------------------------
:func:`enable`, :func:`dump_traceback_later` and :func:`register` keep the
file descriptor of their *file* argument. If the file is closed and its file
@@ -122,14 +123,20 @@ these functions again each time that the file is replaced.
Example
-------
-Example of a segmentation fault on Linux: ::
+.. highlight:: sh
+
+Example of a segmentation fault on Linux with and without enabling the fault
+handler::
+
+ $ python3 -c "import ctypes; ctypes.string_at(0)"
+ Segmentation fault
- $ python -q -X faulthandler
+ $ python3 -q -X faulthandler
>>> import ctypes
>>> ctypes.string_at(0)
Fatal Python error: Segmentation fault
- Current thread 0x00007fb899f39700:
+ Current thread 0x00007fb899f39700 (most recent call first):
File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at
File "<stdin>", line 1 in <module>
Segmentation fault
diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst
index 8a88f8c..06d3f21 100644
--- a/Doc/library/filecmp.rst
+++ b/Doc/library/filecmp.rst
@@ -27,6 +27,10 @@ The :mod:`filecmp` module defines the following functions:
Note that no external programs are called from this function, giving it
portability and efficiency.
+ This function uses a cache for past comparisons and the results,
+ with cache entries invalidated if the :func:`os.stat` information for the
+ file changes. The entire cache may be cleared using :func:`clear_cache`.
+
.. function:: cmpfiles(dir1, dir2, common, shallow=True)
@@ -48,6 +52,15 @@ The :mod:`filecmp` module defines the following functions:
one of the three returned lists.
+.. function:: clear_cache()
+
+ Clear the filecmp cache. This may be useful if a file is compared so quickly
+ after it is modified that it is within the mtime resolution of
+ the underlying filesystem.
+
+ .. versionadded:: 3.4
+
+
.. _dircmp-objects:
The :class:`dircmp` class
@@ -55,28 +68,25 @@ The :class:`dircmp` class
.. class:: dircmp(a, b, ignore=None, hide=None)
- Construct a new directory comparison object, to compare the directories *a* and
- *b*. *ignore* is a list of names to ignore, and defaults to ``['RCS', 'CVS',
- 'tags']``. *hide* is a list of names to hide, and defaults to ``[os.curdir,
- os.pardir]``.
+ Construct a new directory comparison object, to compare the directories *a*
+ and *b*. *ignore* is a list of names to ignore, and defaults to
+ :attr:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and
+ defaults to ``[os.curdir, os.pardir]``.
The :class:`dircmp` class compares files by doing *shallow* comparisons
as described for :func:`filecmp.cmp`.
The :class:`dircmp` class provides the following methods:
-
.. method:: report()
Print (to :data:`sys.stdout`) a comparison between *a* and *b*.
-
.. method:: report_partial_closure()
Print a comparison between *a* and *b* and common immediate
subdirectories.
-
.. method:: report_full_closure()
Print a comparison between *a* and *b* and common subdirectories
@@ -133,7 +143,7 @@ The :class:`dircmp` class
.. attribute:: common_files
- Files in both *a* and *b*
+ Files in both *a* and *b*.
.. attribute:: common_funny
@@ -164,6 +174,12 @@ The :class:`dircmp` class
A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp`
objects.
+.. attribute:: DEFAULT_IGNORES
+
+ .. versionadded:: 3.4
+
+ List of directories ignored by :class:`dircmp` by default.
+
Here is a simplified example of using the ``subdirs`` attribute to search
recursively through two directories to show common different files::
diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst
index d5a4875..ee06830 100644
--- a/Doc/library/fileinput.rst
+++ b/Doc/library/fileinput.rst
@@ -160,6 +160,9 @@ available for subclassing as well:
.. versionchanged:: 3.2
Can be used as a context manager.
+ .. deprecated:: 3.4
+ The ``'rU'`` and ``'U'`` modes.
+
**Optional in-place filtering:** if the keyword argument ``inplace=True`` is
passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the
diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst
index 58998a8..03e085b 100644
--- a/Doc/library/filesys.rst
+++ b/Doc/library/filesys.rst
@@ -12,6 +12,7 @@ in this chapter is:
.. toctree::
+ pathlib.rst
os.path.rst
fileinput.rst
stat.rst
diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst
index e0434b0..ef93f05 100644
--- a/Doc/library/fnmatch.rst
+++ b/Doc/library/fnmatch.rst
@@ -86,7 +86,7 @@ patterns.
'.*\\.txt$'
>>> reobj = re.compile(regex)
>>> reobj.match('foobar.txt')
- <_sre.SRE_Match object at 0x...>
+ <_sre.SRE_Match object; span=(0, 10), match='foobar.txt'>
.. seealso::
diff --git a/Doc/library/formatter.rst b/Doc/library/formatter.rst
index 88be11c..1847a80 100644
--- a/Doc/library/formatter.rst
+++ b/Doc/library/formatter.rst
@@ -3,6 +3,11 @@
.. module:: formatter
:synopsis: Generic output formatter and device interface.
+ :deprecated:
+
+.. deprecated:: 3.4
+ Due to lack of usage, the formatter module has been deprecated and is slated
+ for removal in Python 3.6.
This module supports two interface definitions, each with multiple
diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst
index dcb2ac4..bfec9b0 100644
--- a/Doc/library/ftplib.rst
+++ b/Doc/library/ftplib.rst
@@ -80,40 +80,34 @@ The module defines the following items:
:rfc:`4217`.
Connect as usual to port 21 implicitly securing the FTP control connection
before authenticating. Securing the data connection requires the user to
- explicitly ask for it by calling the :meth:`prot_p` method.
- *keyfile* and *certfile* are optional -- they can contain a PEM formatted
- private key and certificate chain file name for the SSL connection.
- *context* parameter is a :class:`ssl.SSLContext` object which allows
- bundling SSL configuration options, certificates and private keys into a
- single (potentially long-lived) structure. *source_address* is a 2-tuple
- ``(host, port)`` for the socket to bind to as its source address before
- connecting.
+ explicitly ask for it by calling the :meth:`prot_p` method. *context*
+ is a :class:`ssl.SSLContext` object which allows bundling SSL configuration
+ options, certificates and private keys into a single (potentially
+ long-lived) structure. Please read :ref:`ssl-security` for best practices.
+
+ *keyfile* and *certfile* are a legacy alternative to *context* -- they
+ can point to PEM-formatted private key and certificate chain files
+ (respectively) for the SSL connection.
.. versionadded:: 3.2
.. versionchanged:: 3.3
*source_address* parameter was added.
- Here's a sample session using the :class:`FTP_TLS` class:
-
- >>> from ftplib import FTP_TLS
- >>> ftps = FTP_TLS('ftp.python.org')
- >>> ftps.login() # login anonymously before securing control channel
- >>> ftps.prot_p() # switch to secure data connection
- >>> ftps.retrlines('LIST') # list directory content securely
- total 9
- drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .
- drwxr-xr-x 8 root wheel 1024 Jan 3 1994 ..
- drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin
- drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc
- d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming
- drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib
- drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub
- drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr
- -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg
- '226 Transfer complete.'
- >>> ftps.quit()
- >>>
+ .. versionchanged:: 3.4
+ The class now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
+
+ Here's a sample session using the :class:`FTP_TLS` class::
+
+ >>> ftps = FTP_TLS('ftp.pureftpd.org')
+ >>> ftps.login()
+ '230 Anonymous user logged in'
+ >>> ftps.prot_p()
+ '200 Data protection level set to "private"'
+ >>> ftps.nlst()
+ ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp']
.. exception:: error_reply
@@ -427,6 +421,11 @@ FTP_TLS Objects
Set up secure control connection by using TLS or SSL, depending on what
specified in :meth:`ssl_version` attribute.
+ .. versionchanged:: 3.4
+ The method now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
+
.. method:: FTP_TLS.ccc()
Revert control channel back to plaintext. This can be useful to take
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 7ed25c1..9a8c9d7 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -540,8 +540,13 @@ are always available. They are listed here in alphabetical order.
A call to ``format(value, format_spec)`` is translated to
``type(value).__format__(format_spec)`` which bypasses the instance
dictionary when searching for the value's :meth:`__format__` method. A
- :exc:`TypeError` exception is raised if the method is not found or if either
- the *format_spec* or the return value are not strings.
+ :exc:`TypeError` exception is raised if the method search reaches
+ :mod:`object` and the *format_spec* is non-empty, or if either the
+ *format_spec* or the return value are not strings.
+
+ .. versionchanged:: 3.4
+ ``object().__format__(format_spec)`` raises :exc:`TypeError`
+ if *format_spec* is not an empty string.
.. _func-frozenset:
@@ -605,6 +610,10 @@ are always available. They are listed here in alphabetical order.
This function is added to the built-in namespace by the :mod:`site` module.
+ .. versionchanged:: 3.4
+ Changes to :mod:`pydoc` and :mod:`inspect` mean that the reported
+ signatures for callables are now more comprehensive and consistent.
+
.. function:: hex(x)
@@ -676,6 +685,12 @@ are always available. They are listed here in alphabetical order.
The integer type is described in :ref:`typesnumeric`.
+ .. versionchanged:: 3.4
+ If *base* is not an instance of :class:`int` and the *base* object has a
+ :meth:`base.__index__ <object.__index__>` method, that method is called
+ to obtain an integer for the base. Previous versions used
+ :meth:`base.__int__ <object.__int__>` instead of :meth:`base.__index__
+ <object.__index__>`.
.. function:: isinstance(object, classinfo)
@@ -758,25 +773,31 @@ are always available. They are listed here in alphabetical order.
already arranged into argument tuples, see :func:`itertools.starmap`\.
-.. function:: max(iterable, *[, key])
+.. function:: max(iterable, *[, default, key])
max(arg1, arg2, *args[, key])
Return the largest item in an iterable or the largest of two or more
arguments.
- If one positional argument is provided, *iterable* must be a non-empty
- iterable (such as a non-empty string, tuple or list). The largest item
- in the iterable is returned. If two or more positional arguments are
- provided, the largest of the positional arguments is returned.
+ If one positional argument is provided, it should be an :term:`iterable`.
+ The largest item in the iterable is returned. If two or more positional
+ arguments are provided, the smallest of the positional arguments is
+ returned.
- The optional keyword-only *key* argument specifies a one-argument ordering
- function like that used for :meth:`list.sort`.
+ There are two optional keyword-only arguments. The *key* argument specifies
+ a one-argument ordering function like that used for :meth:`list.sort`. The
+ *default* argument specifies an object to return if the provided iterable is
+ empty. If the iterable is empty and *default* is not provided, a
+ :exc:`ValueError` is raised.
If multiple items are maximal, the function returns the first one
encountered. This is consistent with other sort-stability preserving tools
such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and
``heapq.nlargest(1, iterable, key=keyfunc)``.
+ .. versionadded:: 3.4
+ The *default* keyword-only argument.
+
.. _func-memoryview:
.. function:: memoryview(obj)
@@ -786,25 +807,32 @@ are always available. They are listed here in alphabetical order.
:ref:`typememoryview` for more information.
-.. function:: min(iterable, *[, key])
+.. function:: min(iterable, *[, default, key])
min(arg1, arg2, *args[, key])
Return the smallest item in an iterable or the smallest of two or more
arguments.
- If one positional argument is provided, *iterable* must be a non-empty
- iterable (such as a non-empty string, tuple or list). The smallest item
- in the iterable is returned. If two or more positional arguments are
- provided, the smallest of the positional arguments is returned.
+ If one positional argument is provided, it should be an :term:`iterable`.
+ The smallest item in the iterable is returned. If two or more positional
+ arguments are provided, the smallest of the positional arguments is
+ returned.
- The optional keyword-only *key* argument specifies a one-argument ordering
- function like that used for :meth:`list.sort`.
+ There are two optional keyword-only arguments. The *key* argument specifies
+ a one-argument ordering function like that used for :meth:`list.sort`. The
+ *default* argument specifies an object to return if the provided iterable is
+ empty. If the iterable is empty and *default* is not provided, a
+ :exc:`ValueError` is raised.
If multiple items are minimal, the function returns the first one
encountered. This is consistent with other sort-stability preserving tools
such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1,
iterable, key=keyfunc)``.
+ .. versionadded:: 3.4
+ The *default* keyword-only argument.
+
+
.. function:: next(iterator[, default])
Retrieve the next item from the *iterator* by calling its
@@ -866,8 +894,7 @@ are always available. They are listed here in alphabetical order.
``'b'`` binary mode
``'t'`` text mode (default)
``'+'`` open a disk file for updating (reading and writing)
- ``'U'`` universal newlines mode (for backwards compatibility; should
- not be used in new code)
+ ``'U'`` :term:`universal newlines` mode (deprecated)
========= ===============================================================
The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``).
@@ -973,6 +1000,8 @@ are always available. They are listed here in alphabetical order.
:mod:`os.open` as *opener* results in functionality similar to passing
``None``).
+ The newly created file is :ref:`non-inheritable <fd_inheritance>`.
+
The following example uses the :ref:`dir_fd <dir_fd>` parameter of the
:func:`os.open` function to open a file relative to a given directory::
@@ -986,10 +1015,6 @@ are always available. They are listed here in alphabetical order.
...
>>> os.close(dir_fd) # don't leak a file descriptor
- .. versionchanged:: 3.3
- The *opener* parameter was added.
- The ``'x'`` mode was added.
-
The type of :term:`file object` returned by the :func:`open` function
depends on the mode. When :func:`open` is used to open a file in a text
mode (``'w'``, ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of
@@ -1016,10 +1041,18 @@ are always available. They are listed here in alphabetical order.
and :mod:`shutil`.
.. versionchanged:: 3.3
+ The *opener* parameter was added.
+ The ``'x'`` mode was added.
:exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`.
:exc:`FileExistsError` is now raised if the file opened in exclusive
creation mode (``'x'``) already exists.
+ .. versionchanged:: 3.4
+ The file is now non-inheritable.
+
+ .. deprecated-removed:: 3.4 4.0
+ The ``'U'`` mode.
+
.. XXX works for bytes too, but should it?
.. function:: ord(c)
diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst
index 3c946e3..c46b799 100644
--- a/Doc/library/functools.rst
+++ b/Doc/library/functools.rst
@@ -6,6 +6,7 @@
.. moduleauthor:: Peter Harris <scav@blueyonder.co.uk>
.. moduleauthor:: Raymond Hettinger <python@rcn.com>
.. moduleauthor:: Nick Coghlan <ncoghlan@gmail.com>
+.. moduleauthor:: Łukasz Langa <lukasz@langa.pl>
.. sectionauthor:: Peter Harris <scav@blueyonder.co.uk>
**Source code:** :source:`Lib/functools.py`
@@ -133,15 +134,34 @@ The :mod:`functools` module defines the following functions:
@total_ordering
class Student:
+ def _is_valid_operand(self, other):
+ return (hasattr(other, "lastname") and
+ hasattr(other, "firstname"))
def __eq__(self, other):
+ if not self._is_valid_operand(other):
+ return NotImplemented
return ((self.lastname.lower(), self.firstname.lower()) ==
(other.lastname.lower(), other.firstname.lower()))
def __lt__(self, other):
+ if not self._is_valid_operand(other):
+ return NotImplemented
return ((self.lastname.lower(), self.firstname.lower()) <
(other.lastname.lower(), other.firstname.lower()))
+ .. note::
+
+ While this decorator makes it easy to create well behaved totally
+ ordered types, it *does* come at the cost of slower execution and
+ more complex stack traces for the derived comparison methods. If
+ performance benchmarking indicates this is a bottleneck for a given
+ application, implementing all six rich comparison methods instead is
+ likely to provide an easy speed boost.
+
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ Returning NotImplemented from the underlying comparison function for
+ unrecognised types is now supported.
.. function:: partial(func, *args, **keywords)
@@ -174,6 +194,50 @@ The :mod:`functools` module defines the following functions:
18
+.. class:: partialmethod(func, *args, **keywords)
+
+ Return a new :class:`partialmethod` descriptor which behaves
+ like :class:`partial` except that it is designed to be used as a method
+ definition rather than being directly callable.
+
+ *func* must be a :term:`descriptor` or a callable (objects which are both,
+ like normal functions, are handled as descriptors).
+
+ When *func* is a descriptor (such as a normal Python function,
+ :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or
+ another instance of :class:`partialmethod`), calls to ``__get__`` are
+ delegated to the underlying descriptor, and an appropriate
+ :class:`partial` object returned as the result.
+
+ When *func* is a non-descriptor callable, an appropriate bound method is
+ created dynamically. This behaves like a normal Python function when
+ used as a method: the *self* argument will be inserted as the first
+ positional argument, even before the *args* and *keywords* supplied to
+ the :class:`partialmethod` constructor.
+
+ Example::
+
+ >>> class Cell(object):
+ ... def __init__(self):
+ ... self._alive = False
+ ... @property
+ ... def alive(self):
+ ... return self._alive
+ ... def set_state(self, state):
+ ... self._alive = bool(state)
+ ... set_alive = partialmethod(set_state, True)
+ ... set_dead = partialmethod(set_state, False)
+ ...
+ >>> c = Cell()
+ >>> c.alive
+ False
+ >>> c.set_alive()
+ >>> c.alive
+ True
+
+ .. versionadded:: 3.4
+
+
.. function:: reduce(function, iterable[, initializer])
Apply *function* of two arguments cumulatively to the items of *sequence*, from
@@ -198,6 +262,111 @@ The :mod:`functools` module defines the following functions:
return value
+.. decorator:: singledispatch(default)
+
+ Transforms a function into a :term:`single-dispatch <single
+ dispatch>` :term:`generic function`.
+
+ To define a generic function, decorate it with the ``@singledispatch``
+ decorator. Note that the dispatch happens on the type of the first argument,
+ create your function accordingly::
+
+ >>> from functools import singledispatch
+ >>> @singledispatch
+ ... def fun(arg, verbose=False):
+ ... if verbose:
+ ... print("Let me just say,", end=" ")
+ ... print(arg)
+
+ To add overloaded implementations to the function, use the :func:`register`
+ attribute of the generic function. It is a decorator, taking a type
+ parameter and decorating a function implementing the operation for that
+ type::
+
+ >>> @fun.register(int)
+ ... def _(arg, verbose=False):
+ ... if verbose:
+ ... print("Strength in numbers, eh?", end=" ")
+ ... print(arg)
+ ...
+ >>> @fun.register(list)
+ ... def _(arg, verbose=False):
+ ... if verbose:
+ ... print("Enumerate this:")
+ ... for i, elem in enumerate(arg):
+ ... print(i, elem)
+
+ To enable registering lambdas and pre-existing functions, the
+ :func:`register` attribute can be used in a functional form::
+
+ >>> def nothing(arg, verbose=False):
+ ... print("Nothing.")
+ ...
+ >>> fun.register(type(None), nothing)
+
+ The :func:`register` attribute returns the undecorated function which
+ enables decorator stacking, pickling, as well as creating unit tests for
+ each variant independently::
+
+ >>> @fun.register(float)
+ ... @fun.register(Decimal)
+ ... def fun_num(arg, verbose=False):
+ ... if verbose:
+ ... print("Half of your number:", end=" ")
+ ... print(arg / 2)
+ ...
+ >>> fun_num is fun
+ False
+
+ When called, the generic function dispatches on the type of the first
+ argument::
+
+ >>> fun("Hello, world.")
+ Hello, world.
+ >>> fun("test.", verbose=True)
+ Let me just say, test.
+ >>> fun(42, verbose=True)
+ Strength in numbers, eh? 42
+ >>> fun(['spam', 'spam', 'eggs', 'spam'], verbose=True)
+ Enumerate this:
+ 0 spam
+ 1 spam
+ 2 eggs
+ 3 spam
+ >>> fun(None)
+ Nothing.
+ >>> fun(1.23)
+ 0.615
+
+ Where there is no registered implementation for a specific type, its
+ method resolution order is used to find a more generic implementation.
+ The original function decorated with ``@singledispatch`` is registered
+ for the base ``object`` type, which means it is used if no better
+ implementation is found.
+
+ To check which implementation will the generic function choose for
+ a given type, use the ``dispatch()`` attribute::
+
+ >>> fun.dispatch(float)
+ <function fun_num at 0x1035a2840>
+ >>> fun.dispatch(dict) # note: default implementation
+ <function fun at 0x103fe0000>
+
+ To access all registered implementations, use the read-only ``registry``
+ attribute::
+
+ >>> fun.registry.keys()
+ dict_keys([<class 'NoneType'>, <class 'int'>, <class 'object'>,
+ <class 'decimal.Decimal'>, <class 'list'>,
+ <class 'float'>])
+ >>> fun.registry[float]
+ <function fun_num at 0x1035a2840>
+ >>> fun.registry[object]
+ <function fun at 0x103fe0000>
+
+ .. versionadded:: 3.4
+
+
.. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES)
Update a *wrapper* function to look like the *wrapped* function. The optional
@@ -212,8 +381,8 @@ The :mod:`functools` module defines the following functions:
To allow access to the original function for introspection and other purposes
(e.g. bypassing a caching decorator such as :func:`lru_cache`), this function
- automatically adds a __wrapped__ attribute to the wrapper that refers to
- the original function.
+ automatically adds a ``__wrapped__`` attribute to the wrapper that refers to
+ the function being wrapped.
The main intended use for this function is in :term:`decorator` functions which
wrap the decorated function and return the wrapper. If the wrapper function is
@@ -236,6 +405,11 @@ The :mod:`functools` module defines the following functions:
.. versionchanged:: 3.2
Missing attributes no longer trigger an :exc:`AttributeError`.
+ .. versionchanged:: 3.4
+ The ``__wrapped__`` attribute now always refers to the wrapped
+ function, even if that function defined a ``__wrapped__`` attribute.
+ (see :issue:`17482`)
+
.. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES)
@@ -301,4 +475,3 @@ differences. For instance, the :attr:`__name__` and :attr:`__doc__` attributes
are not created automatically. Also, :class:`partial` objects defined in
classes behave like static methods and do not transform into bound methods
during instance attribute look-up.
-
diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst
index 71449a3..8135542 100644
--- a/Doc/library/gc.rst
+++ b/Doc/library/gc.rst
@@ -67,6 +67,25 @@ The :mod:`gc` module provides the following functions:
returned.
+.. function:: get_stats()
+
+ Return a list of three per-generation dictionaries containing collection
+ statistics since interpreter start. The number of keys may change
+ in the future, but currently each dictionary will contain the following
+ items:
+
+ * ``collections`` is the number of times this generation was collected;
+
+ * ``collected`` is the total number of objects collected inside this
+ generation;
+
+ * ``uncollectable`` is the total number of objects which were found
+ to be uncollectable (and were therefore moved to the :data:`garbage`
+ list) inside this generation.
+
+ .. versionadded:: 3.4
+
+
.. function:: set_threshold(threshold0[, threshold1[, threshold2]])
Set the garbage collection thresholds (the collection frequency). Setting
@@ -158,24 +177,13 @@ values but should not rebind them):
.. data:: garbage
- A list of objects which the collector found to be unreachable but could not be
- freed (uncollectable objects). By default, this list contains only objects with
- :meth:`__del__` methods. Objects that have :meth:`__del__` methods and are
- part of a reference cycle cause the entire reference cycle to be uncollectable,
- including objects not necessarily in the cycle but reachable only from it.
- Python doesn't collect such cycles automatically because, in general, it isn't
- possible for Python to guess a safe order in which to run the :meth:`__del__`
- methods. If you know a safe order, you can force the issue by examining the
- *garbage* list, and explicitly breaking cycles due to your objects within the
- list. Note that these objects are kept alive even so by virtue of being in the
- *garbage* list, so they should be removed from *garbage* too. For example,
- after breaking cycles, do ``del gc.garbage[:]`` to empty the list. It's
- generally better to avoid the issue by not creating cycles containing objects
- with :meth:`__del__` methods, and *garbage* can be examined in that case to
- verify that no such cycles are being created.
-
- If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be added
- to this list rather than freed.
+ A list of objects which the collector found to be unreachable but could
+ not be freed (uncollectable objects). Starting with Python 3.4, this
+ list should be empty most of the time, except when using instances of
+ C extension types with a non-NULL ``tp_del`` slot.
+
+ If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be
+ added to this list rather than freed.
.. versionchanged:: 3.2
If this list is non-empty at interpreter shutdown, a
@@ -183,6 +191,10 @@ values but should not rebind them):
:const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects
are printed.
+ .. versionchanged:: 3.4
+ Following :pep:`442`, objects with a :meth:`__del__` method don't end
+ up in :attr:`gc.garbage` anymore.
+
.. data:: callbacks
A list of callbacks that will be invoked by the garbage collector before and
diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst
index eff138b..abcbf38 100644
--- a/Doc/library/glob.rst
+++ b/Doc/library/glob.rst
@@ -25,6 +25,10 @@ For a literal match, wrap the meta-characters in brackets.
For example, ``'[?]'`` matches the character ``'?'``.
+.. seealso::
+ The :mod:`pathlib` module offers high-level path objects.
+
+
.. function:: glob(pathname)
Return a possibly-empty list of path names that match *pathname*, which must be
@@ -40,6 +44,17 @@ For example, ``'[?]'`` matches the character ``'?'``.
without actually storing them all simultaneously.
+.. function:: escape(pathname)
+
+ Escape all special characters (``'?'``, ``'*'`` and ``'['``).
+ This is useful if you want to match an arbitrary literal string that may
+ have special characters in it. Special characters in drive/UNC
+ sharepoints are not escaped, e.g. on Windows
+ ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``.
+
+ .. versionadded:: 3.4
+
+
For example, consider a directory containing only the following files:
:file:`1.gif`, :file:`2.txt`, and :file:`card.gif`. :func:`glob` will produce
the following results. Notice how any leading components of the path are
diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst
index 354deed..78536fa 100644
--- a/Doc/library/gzip.rst
+++ b/Doc/library/gzip.rst
@@ -35,8 +35,8 @@ The module defines the following items:
:class:`bytes` object), or an existing file object to read from or write to.
The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``,
- ``'w'``, or ``'wb'`` for binary mode, or ``'rt'``, ``'at'``, or ``'wt'`` for
- text mode. The default is ``'rb'``.
+ ``'w'``, ``'wb'``, ``'x'`` or ``'xb'`` for binary mode, or ``'rt'``,
+ ``'at'``, ``'wt'``, or ``'xt'`` for text mode. The default is ``'rb'``.
The *compresslevel* argument is an integer from 0 to 9, as for the
:class:`GzipFile` constructor.
@@ -53,6 +53,9 @@ The module defines the following items:
Added support for *filename* being a file object, support for text mode,
and the *encoding*, *errors* and *newline* arguments.
+ .. versionchanged:: 3.4
+ Added support for the ``'x'``, ``'xb'`` and ``'xt'`` modes.
+
.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None)
@@ -73,8 +76,9 @@ The module defines the following items:
original filename is not included in the header.
The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``,
- or ``'wb'``, depending on whether the file will be read or written. The default
- is the mode of *fileobj* if discernible; otherwise, the default is ``'rb'``.
+ ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or
+ written. The default is the mode of *fileobj* if discernible; otherwise, the
+ default is ``'rb'``.
Note that the file is always opened in binary mode. To open a compressed file
in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an
@@ -130,6 +134,9 @@ The module defines the following items:
.. versionchanged:: 3.3
The :meth:`io.BufferedIOBase.read1` method is now implemented.
+ .. versionchanged:: 3.4
+ Added support for the ``'x'`` and ``'xb'`` modes.
+
.. function:: compress(data, compresslevel=9)
diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst
index a693f7e..d27902e 100644
--- a/Doc/library/hashlib.rst
+++ b/Doc/library/hashlib.rst
@@ -32,6 +32,12 @@ digests. The modern term is secure hash.
Some algorithms have known hash collision weaknesses, refer to the "See
also" section at the end.
+
+.. _hash-algorithms:
+
+Hash algorithms
+---------------
+
There is one constructor method named for each type of :dfn:`hash`. All return
a hash object with the same simple interface. For example: use :func:`sha1` to
create a SHA1 hash object. You can now feed this object with :term:`bytes-like
@@ -53,9 +59,9 @@ concatenation of the data fed to it so far using the :meth:`digest` or
.. index:: single: OpenSSL; (use in module hashlib)
Constructors for hash algorithms that are always present in this module are
-:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, and
-:func:`sha512`. Additional algorithms may also be available depending upon the
-OpenSSL library that Python uses on your platform.
+:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`,
+and :func:`sha512`. Additional algorithms may also be available depending upon
+the OpenSSL library that Python uses on your platform.
For example, to obtain the digest of the byte string ``b'Nobody inspects the
spammish repetition'``::
@@ -122,6 +128,18 @@ returned by the constructors:
The internal block size of the hash algorithm in bytes.
+A hash object has the following attributes:
+
+.. attribute:: hash.name
+
+ The canonical name of this hash, always lowercase and always suitable as a
+ parameter to :func:`new` to create another hash of this type.
+
+ .. versionchanged:: 3.4
+ The name attribute has been present in CPython since its inception, but
+ until Python 3.4 was not formally specified, so may not exist on some
+ platforms.
+
A hash object has the following methods:
@@ -158,6 +176,45 @@ A hash object has the following methods:
compute the digests of data sharing a common initial substring.
+Key Derivation Function
+-----------------------
+
+Key derivation and key stretching algorithms are designed for secure password
+hashing. Naive algorithms such as ``sha1(password)`` are not resistant
+against brute-force attacks. A good password hashing function must be tunable,
+slow and include a salt.
+
+
+.. function:: pbkdf2_hmac(name, password, salt, rounds, dklen=None)
+
+ The function provides PKCS#5 password-based key derivation function 2. It
+ uses HMAC as pseudorandom function.
+
+ The string *name* is the desired name of the hash digest algorithm for
+ HMAC, e.g. 'sha1' or 'sha256'. *password* and *salt* are interpreted as
+ buffers of bytes. Applications and libraries should limit *password* to
+ a sensible value (e.g. 1024). *salt* should be about 16 or more bytes from
+ a proper source, e.g. :func:`os.urandom`.
+
+ The number of *rounds* should be chosen based on the hash algorithm and
+ computing power. As of 2013 a value of at least 100,000 rounds of SHA-256
+ have been suggested.
+
+ *dklen* is the length of the derived key. If *dklen* is ``None`` then the
+ digest size of the hash algorithm *name* is used, e.g. 64 for SHA-512.
+
+ >>> import hashlib, binascii
+ >>> dk = hashlib.pbkdf2_hmac('sha256', b'password', b'salt', 100000)
+ >>> binascii.hexlify(dk)
+ b'0394a2ede332c9a13eb82e9b24631604c31df978b4e2f0fbd2c549944f9d79a5'
+
+ .. versionadded:: 3.4
+
+ .. note:: A fast implementation of *pbkdf2_hmac* is available with OpenSSL.
+ The Python implementation uses an inline version of :mod:`hmac`. It is
+ about three times slower and doesn't release the GIL.
+
+
.. seealso::
Module :mod:`hmac`
@@ -173,3 +230,5 @@ A hash object has the following methods:
Wikipedia article with information on which algorithms have known issues and
what that means regarding their use.
+ http://www.ietf.org/rfc/rfc2898.txt
+ PKCS #5: Password-Based Cryptography Specification Version 2.0
diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst
index 9575693..4858235 100644
--- a/Doc/library/hmac.rst
+++ b/Doc/library/hmac.rst
@@ -16,20 +16,32 @@ This module implements the HMAC algorithm as described by :rfc:`2104`.
.. function:: new(key, msg=None, digestmod=None)
- Return a new hmac object. *key* is a bytes object giving the secret key. If
- *msg* is present, the method call ``update(msg)`` is made. *digestmod* is
- the digest constructor or module for the HMAC object to use. It defaults to
- the :data:`hashlib.md5` constructor.
+ Return a new hmac object. *key* is a bytes or bytearray object giving the
+ secret key. If *msg* is present, the method call ``update(msg)`` is made.
+ *digestmod* is the digest name, digest constructor or module for the HMAC
+ object to use. It supports any name suitable to :func:`hashlib.new` and
+ defaults to the :data:`hashlib.md5` constructor.
+
+ .. versionchanged:: 3.4
+ Parameter *key* can be a bytes or bytearray object.
+ Parameter *msg* can be of any type supported by :mod:`hashlib`.
+ Paramter *digestmod* can be the name of a hash algorithm.
+
+ .. deprecated:: 3.4
+ MD5 as implicit default digest for *digestmod* is deprecated.
An HMAC object has the following methods:
.. method:: HMAC.update(msg)
- Update the hmac object with the bytes object *msg*. Repeated calls are
- equivalent to a single call with the concatenation of all the arguments:
+ Update the hmac object with *msg*. Repeated calls are equivalent to a
+ single call with the concatenation of all the arguments:
``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``.
+ .. versionchanged:: 3.4
+ Parameter *msg* can be of any type supported by :mod:`hashlib`.
+
.. method:: HMAC.digest()
@@ -66,6 +78,25 @@ An HMAC object has the following methods:
compute the digests of strings that share a common initial substring.
+A hash object has the following attributes:
+
+.. attribute:: HMAC.digest_size
+
+ The size of the resulting HMAC digest in bytes.
+
+.. attribute:: HMAC.block_size
+
+ The internal block size of the hash algorithm in bytes.
+
+ .. versionadded:: 3.4
+
+.. attribute:: HMAC.name
+
+ The canonical name of this HMAC, always lowercase, e.g. ``hmac-md5``.
+
+ .. versionadded:: 3.4
+
+
This module also provides the following helper function:
.. function:: compare_digest(a, b)
diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst
index 0aabbad..09b0abc 100644
--- a/Doc/library/html.entities.rst
+++ b/Doc/library/html.entities.rst
@@ -20,6 +20,7 @@ This module defines four dictionaries, :data:`html5`,
Note that the trailing semicolon is included in the name (e.g. ``'gt;'``),
however some of the names are accepted by the standard even without the
semicolon: in this case the name is present with and without the ``';'``.
+ See also :func:`html.unescape`.
.. versionadded:: 3.3
diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst
index e4154ef..44b7d6e 100644
--- a/Doc/library/html.parser.rst
+++ b/Doc/library/html.parser.rst
@@ -16,14 +16,21 @@
This module defines a class :class:`HTMLParser` which serves as the basis for
parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
-.. class:: HTMLParser(strict=False)
+.. class:: HTMLParser(strict=False, *, convert_charrefs=False)
- Create a parser instance. If *strict* is ``False`` (the default), the parser
- will accept and parse invalid markup. If *strict* is ``True`` the parser
- will raise an :exc:`~html.parser.HTMLParseError` exception instead [#]_ when
- it's not able to parse the markup.
- The use of ``strict=True`` is discouraged and the *strict* argument is
- deprecated.
+ Create a parser instance.
+
+ If *convert_charrefs* is ``True`` (default: ``False``), all character
+ references (except the ones in ``script``/``style`` elements) are
+ automatically converted to the corresponding Unicode characters.
+ The use of ``convert_charrefs=True`` is encouraged and will become
+ the default in Python 3.5.
+
+ If *strict* is ``False`` (the default), the parser will accept and parse
+ invalid markup. If *strict* is ``True`` the parser will raise an
+ :exc:`~html.parser.HTMLParseError` exception instead [#]_ when it's not
+ able to parse the markup. The use of ``strict=True`` is discouraged and
+ the *strict* argument is deprecated.
An :class:`.HTMLParser` instance is fed HTML data and calls handler methods
when start tags, end tags, text, comments, and other markup elements are
@@ -34,12 +41,15 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML.
handler for elements which are closed implicitly by closing an outer element.
.. versionchanged:: 3.2
- *strict* keyword added.
+ *strict* argument added.
.. deprecated-removed:: 3.3 3.5
The *strict* argument and the strict mode have been deprecated.
The parser is now able to accept and parse invalid markup too.
+ .. versionchanged:: 3.4
+ *convert_charrefs* keyword argument added.
+
An exception is defined as well:
@@ -74,7 +84,7 @@ as they are encountered::
def handle_data(self, data):
print("Encountered some data :", data)
- parser = MyHTMLParser(strict=False)
+ parser = MyHTMLParser()
parser.feed('<html><head><title>Test</title></head>'
'<body><h1>Parse me!</h1></body></html>')
@@ -181,7 +191,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`):
This method is called to process a named character reference of the form
``&name;`` (e.g. ``&gt;``), where *name* is a general entity reference
- (e.g. ``'gt'``).
+ (e.g. ``'gt'``). This method is never called if *convert_charrefs* is
+ ``True``.
.. method:: HTMLParser.handle_charref(name)
@@ -189,7 +200,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`):
This method is called to process decimal and hexadecimal numeric character
references of the form ``&#NNN;`` and ``&#xNNN;``. For example, the decimal
equivalent for ``&gt;`` is ``&#62;``, whereas the hexadecimal is ``&#x3E;``;
- in this case the method will receive ``'62'`` or ``'x3E'``.
+ in this case the method will receive ``'62'`` or ``'x3E'``. This method
+ is never called if *convert_charrefs* is ``True``.
.. method:: HTMLParser.handle_comment(data)
@@ -272,7 +284,7 @@ examples::
def handle_decl(self, data):
print("Decl :", data)
- parser = MyHTMLParser(strict=False)
+ parser = MyHTMLParser()
Parsing a doctype::
@@ -324,7 +336,8 @@ correct char (note: these 3 references are all equivalent to ``'>'``)::
Num ent : >
Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but
-:meth:`~HTMLParser.handle_data` might be called more than once::
+:meth:`~HTMLParser.handle_data` might be called more than once
+(unless *convert_charrefs* is set to ``True``)::
>>> for chunk in ['<sp', 'an>buff', 'ered ', 'text</s', 'pan>']:
... parser.feed(chunk)
diff --git a/Doc/library/html.rst b/Doc/library/html.rst
index 1107ca9..d0706bc 100644
--- a/Doc/library/html.rst
+++ b/Doc/library/html.rst
@@ -20,6 +20,17 @@ This module defines utilities to manipulate HTML.
.. versionadded:: 3.2
+
+.. function:: unescape(s)
+
+ Convert all named and numeric character references (e.g. ``&gt;``,
+ ``&#62;``, ``&x3e;``) in the string *s* to the corresponding unicode
+ characters. This function uses the rules defined by the HTML 5 standard
+ for both valid and invalid character references, and the :data:`list of
+ HTML 5 named character references <html.entities.html5>`.
+
+ .. versionadded:: 3.4
+
--------------
Submodules in the ``html`` package are:
diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst
index bb30f24..10704ce 100644
--- a/Doc/library/http.client.rst
+++ b/Doc/library/http.client.rst
@@ -27,7 +27,7 @@ HTTPS protocols. It is normally not used directly --- the module
The module provides the following classes:
-.. class:: HTTPConnection(host, port=None[, strict][, timeout], \
+.. class:: HTTPConnection(host, port=None[, timeout], \
source_address=None)
An :class:`HTTPConnection` instance represents one transaction with an HTTP
@@ -43,44 +43,48 @@ The module provides the following classes:
For example, the following calls all create instances that connect to the server
at the same host and port::
- >>> h1 = http.client.HTTPConnection('www.cwi.nl')
- >>> h2 = http.client.HTTPConnection('www.cwi.nl:80')
- >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80)
- >>> h3 = http.client.HTTPConnection('www.cwi.nl', 80, timeout=10)
+ >>> h1 = http.client.HTTPConnection('www.python.org')
+ >>> h2 = http.client.HTTPConnection('www.python.org:80')
+ >>> h3 = http.client.HTTPConnection('www.python.org', 80)
+ >>> h4 = http.client.HTTPConnection('www.python.org', 80, timeout=10)
.. versionchanged:: 3.2
*source_address* was added.
- .. deprecated-removed:: 3.2 3.4
- The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses"
- are not supported anymore.
+ .. versionchanged:: 3.4
+ The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are
+ not longer supported.
.. class:: HTTPSConnection(host, port=None, key_file=None, \
- cert_file=None[, strict][, timeout], \
+ cert_file=None[, timeout], \
source_address=None, *, context=None, \
check_hostname=None)
A subclass of :class:`HTTPConnection` that uses SSL for communication with
secure servers. Default port is ``443``. If *context* is specified, it
must be a :class:`ssl.SSLContext` instance describing the various SSL
- options. If *context* is specified and has a :attr:`~ssl.SSLContext.verify_mode`
- of either :data:`~ssl.CERT_OPTIONAL` or :data:`~ssl.CERT_REQUIRED`, then
- by default *host* is matched against the host name(s) allowed by the
- server's certificate. If you want to change that behaviour, you can
- explicitly set *check_hostname* to False.
+ options.
*key_file* and *cert_file* are deprecated, please use
- :meth:`ssl.SSLContext.load_cert_chain` instead.
+ :meth:`ssl.SSLContext.load_cert_chain` instead, or let
+ :func:`ssl.create_default_context` select the system's trusted CA
+ certificates for you.
- If you access arbitrary hosts on the Internet, it is recommended to
- require certificate checking and feed the *context* with a set of
- trusted CA certificates::
+ The recommended way to connect to HTTPS hosts on the Internet is as
+ follows::
- context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
- context.verify_mode = ssl.CERT_REQUIRED
- context.load_verify_locations('/etc/pki/tls/certs/ca-bundle.crt')
- h = client.HTTPSConnection('svn.python.org', 443, context=context)
+ context = ssl.create_default_context()
+ h = client.HTTPSConnection('www.python.org', 443, context=context)
+
+ Please read :ref:`ssl-security` for more information on best practices.
+
+ .. note::
+ If *context* is specified and has a :attr:`~ssl.SSLContext.verify_mode`
+ of either :data:`~ssl.CERT_OPTIONAL` or :data:`~ssl.CERT_REQUIRED`, then
+ by default *host* is matched against the host name(s) allowed by the
+ server's certificate. If you want to change that behaviour, you can
+ explicitly set *check_hostname* to False.
.. versionchanged:: 3.2
*source_address*, *context* and *check_hostname* were added.
@@ -89,19 +93,19 @@ The module provides the following classes:
This class now supports HTTPS virtual hosts if possible (that is,
if :data:`ssl.HAS_SNI` is true).
- .. deprecated-removed:: 3.2 3.4
- The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses"
- are not supported anymore.
+ .. versionchanged:: 3.4
+ The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are
+ no longer supported.
-.. class:: HTTPResponse(sock, debuglevel=0[, strict], method=None, url=None)
+.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None)
Class whose instances are returned upon successful connection. Not
instantiated directly by user.
- .. deprecated-removed:: 3.2 3.4
- The *strict* parameter is deprecated. HTTP 0.9-style "Simple Responses"
- are not supported anymore.
+ .. versionchanged:: 3.4
+ The *strict* parameter was removed. HTTP 0.9 style "Simple Responses" are
+ no longer supported.
The following exceptions are raised as appropriate:
diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst
index 5aeb719..dd19fb4 100644
--- a/Doc/library/http.server.rst
+++ b/Doc/library/http.server.rst
@@ -116,7 +116,7 @@ of which this module provides three different variants:
HTTP error code value. *message* should be a string containing a
(detailed) error message of what occurred, and *explain* should be an
explanation of the error code number. Default *message* and *explain*
- values can found in the *responses* class variable.
+ values can found in the :attr:`responses` class variable.
.. attribute:: error_content_type
@@ -170,12 +170,22 @@ of which this module provides three different variants:
.. versionadded:: 3.2
- .. method:: send_error(code, message=None)
+ .. method:: send_error(code, message=None, explain=None)
Sends and logs a complete error reply to the client. The numeric *code*
- specifies the HTTP error code, with *message* as optional, more specific text. A
- complete set of headers is sent, followed by text composed using the
- :attr:`error_message_format` class variable.
+ specifies the HTTP error code, with *message* as an optional, short, human
+ readable description of the error. The *explain* argument can be used to
+ provide more detailed information about the error; it will be formatted
+ using the :attr:`error_message_format` class variable and emitted, after
+ a complete set of headers, as the response body. The :attr:`responses`
+ class variable holds the default values for *message* and *explain* that
+ will be used if no value is provided; for unknown codes the default value
+ for both is the string ``???``.
+
+ .. versionchanged:: 3.4
+ The error response includes a Content-Length header.
+ Added the *explain* argument.
+
.. method:: send_response(code, message=None)
@@ -341,7 +351,7 @@ of which this module provides three different variants:
The :class:`SimpleHTTPRequestHandler` class can be used in the following
manner in order to create a very basic webserver serving files relative to
-the current directory. ::
+the current directory::
import http.server
import socketserver
@@ -355,12 +365,23 @@ the current directory. ::
print("serving at port", PORT)
httpd.serve_forever()
+.. _http-server-cli:
+
:mod:`http.server` can also be invoked directly using the :option:`-m`
switch of the interpreter with a ``port number`` argument. Similar to
-the previous example, this serves files relative to the current directory. ::
+the previous example, this serves files relative to the current directory::
python -m http.server 8000
+By default, server binds itself to all interfaces. The option ``-b/--bind``
+specifies a specific address to which it should bind. For example, the
+following command causes the server to bind to localhost only::
+
+ python -m http.server 8000 --bind 127.0.0.1
+
+.. versionadded:: 3.4
+ ``--bind`` argument was introduced.
+
.. class:: CGIHTTPRequestHandler(request, client_address, server)
@@ -403,7 +424,7 @@ the previous example, this serves files relative to the current directory. ::
reasons. Problems with the CGI script will be translated to error 403.
:class:`CGIHTTPRequestHandler` can be enabled in the command line by passing
-the ``--cgi`` option.::
+the ``--cgi`` option::
python -m http.server --cgi 8000
diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
index 36d78b0..2718cef 100644
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -16,70 +16,82 @@ IDLE has the following features:
* coded in 100% pure Python, using the :mod:`tkinter` GUI toolkit
-* cross-platform: works on Windows and Unix
+* cross-platform: works on Windows, Unix, and Mac OS X
-* multi-window text editor with multiple undo, Python colorizing and many other
- features, e.g. smart indent and call tips
+* multi-window text editor with multiple undo, Python colorizing,
+ smart indent, call tips, and many other features
* Python shell window (a.k.a. interactive interpreter)
-* debugger (not complete, but you can set breakpoints, view and step)
+* debugger (not complete, but you can set breakpoints, view and step)
Menus
-----
+IDLE has two window types, the Shell window and the Editor window. It is
+possible to have multiple editor windows simultaneously. IDLE's
+menus dynamically change based on which window is currently selected. Each menu
+documented below indicates which window type it is associated with. Click on
+the dotted line at the top of a menu to "tear it off": a separate window
+containing the menu is created (for Unix and Windows only).
-File menu
-^^^^^^^^^
+File menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
New file
- create a new file editing window
+ Create a new file editing window
Open...
- open an existing file
+ Open an existing file
Open module...
- open an existing module (searches sys.path)
+ Open an existing module (searches sys.path)
+
+Recent Files
+ Open a list of recent files
Class browser
- show classes and methods in current file
+ Show classes and methods in current file
Path browser
- show sys.path directories, modules, classes and methods
+ Show sys.path directories, modules, classes and methods
.. index::
single: Class browser
single: Path browser
Save
- save current window to the associated file (unsaved windows have a \* before and
- after the window title)
+ Save current window to the associated file (unsaved windows have a
+ \* before and after the window title)
Save As...
- save current window to new file, which becomes the associated file
+ Save current window to new file, which becomes the associated file
Save Copy As...
- save current window to different file without changing the associated file
+ Save current window to different file without changing the associated file
+
+Print Window
+ Print the current window
Close
- close current window (asks to save if unsaved)
+ Close current window (asks to save if unsaved)
Exit
- close all windows and quit IDLE (asks to save if unsaved)
+ Close all windows and quit IDLE (asks to save if unsaved)
-Edit menu
-^^^^^^^^^
+Edit menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Undo
- Undo last change to current window (max 1000 changes)
+ Undo last change to current window (a maximum of 1000 changes may be undone)
Redo
Redo last undone change to current window
Cut
- Copy selection into system-wide clipboard; then delete selection
+ Copy selection into system-wide clipboard; then delete the selection
Copy
Copy selection into system-wide clipboard
@@ -108,11 +120,30 @@ Replace...
Go to line
Ask for a line number and show that line
+Expand word
+ Expand the word you have typed to match another word in the same buffer;
+ repeat to get a different expansion
+
+Show call tip
+ After an unclosed parenthesis for a function, open a small window with
+ function parameter hints
+
+Show surrounding parens
+ Highlight the surrounding parenthesis
+
+Show Completions
+ Open a scroll window allowing selection keywords and attributes. See
+ Completions below.
+
+
+Format menu (Editor window only)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
Indent region
- Shift selected lines right 4 spaces
+ Shift selected lines right by the indent width (default 4 spaces)
Dedent region
- Shift selected lines left 4 spaces
+ Shift selected lines left by the indent width (default 4 spaces)
Comment out region
Insert ## in front of selected lines
@@ -121,67 +152,121 @@ Uncomment region
Remove leading # or ## from selected lines
Tabify region
- Turns *leading* stretches of spaces into tabs
+ Turns *leading* stretches of spaces into tabs. (Note: We recommend using
+ 4 space blocks to indent Python code.)
Untabify region
- Turn *all* tabs into the right number of spaces
+ Turn *all* tabs into the correct number of spaces
-Expand word
- Expand the word you have typed to match another word in the same buffer; repeat
- to get a different expansion
+Toggle tabs
+ Open a dialog to switch between indenting with spaces and tabs.
-Format Paragraph
- Reformat the current blank-line-separated paragraph
+New Indent Width
+ Open a dialog to change indent width. The accepted default by the Python
+ community is 4 spaces.
-Import module
- Import or reload the current module
+Format Paragraph
+ Reformat the current blank-line-separated paragraph. All lines in the
+ paragraph will be formatted to less than 80 columns.
-Run script
- Execute the current file in the __main__ namespace
+Strip trailing whitespace
+ Removes any space characters after the end of the last non-space character
.. index::
single: Import module
single: Run script
-Windows menu
-^^^^^^^^^^^^
+Run menu (Editor window only)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Zoom Height
- toggles the window between normal size (24x80) and maximum height.
+Python Shell
+ Open or wake up the Python Shell window
-The rest of this menu lists the names of all open windows; select one to bring
-it to the foreground (deiconifying it if necessary).
+Check module
+ Check the syntax of the module currently open in the Editor window. If the
+ module has not been saved IDLE will prompt the user to save the code.
+
+Run module
+ Restart the shell to clean the environment, then execute the currently
+ open module. If the module has not been saved IDLE will prompt the user
+ to save the code.
+Shell menu (Shell window only)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Debug menu
-^^^^^^^^^^
+View Last Restart
+ Scroll the shell window to the last Shell restart
-* in the Python Shell window only
+Restart Shell
+ Restart the shell to clean the environment
+
+Debug menu (Shell window only)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Go to file/line
Look around the insert point for a filename and line number, open the file,
and show the line. Useful to view the source lines referenced in an
- exception traceback.
+ exception traceback. Available in the context menu of the Shell window.
-Debugger
- Run commands in the shell under the debugger.
+Debugger (toggle)
+ This feature is not complete and considered experimental. Run commands in
+ the shell under the debugger
Stack viewer
- Show the stack traceback of the last exception.
+ Show the stack traceback of the last exception
Auto-open Stack Viewer
- Open stack viewer on traceback.
+ Toggle automatically opening the stack viewer on unhandled exception
.. index::
single: stack viewer
single: debugger
+Options menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Configure IDLE
+ Open a configuration dialog. Fonts, indentation, keybindings, and color
+ themes may be altered. Startup Preferences may be set, and additional
+ help sources can be specified.
+
+Code Context (toggle)(Editor Window only)
+ Open a pane at the top of the edit window which shows the block context
+ of the section of code which is scrolling off the top of the window.
+
+Windows menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Zoom Height
+ Toggles the window between normal size (40x80 initial setting) and maximum
+ height. The initial size is in the Configure IDLE dialog under the general
+ tab.
+
+The rest of this menu lists the names of all open windows; select one to bring
+it to the foreground (deiconifying it if necessary).
+
+Help menu (Shell and Editor)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+About IDLE
+ Version, copyright, license, credits
+
+IDLE Help
+ Display a help file for IDLE detailing the menu options, basic editing and
+ navigation, and other tips.
+
+Python Docs
+ Access local Python documentation, if installed. Or will start a web browser
+ and open docs.python.org showing the latest Python documentation.
-Edit context menu
-^^^^^^^^^^^^^^^^^
+Additional help sources may be added here with the Configure IDLE dialog under
+the General tab.
-* Right-click in Edit window (Control-click on OS X)
+Editor Window context menu
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* Right-click in Editor window (Control-click on OS X)
Cut
Copy selection into system-wide clipboard; then delete selection
@@ -207,8 +292,8 @@ Clear Breakpoint
single: breakpoints
-Shell context menu
-^^^^^^^^^^^^^^^^^^
+Shell Window context menu
+^^^^^^^^^^^^^^^^^^^^^^^^^
* Right-click in Python Shell window (Control-click on OS X)
@@ -225,19 +310,44 @@ Go to file/line
Same as in Debug menu.
-Basic editing and navigation
-----------------------------
+Editing and navigation
+----------------------
* :kbd:`Backspace` deletes to the left; :kbd:`Del` deletes to the right
+* :kbd:`C-Backspace` delete word left; :kbd:`C-Del` delete word to the right
+
* Arrow keys and :kbd:`Page Up`/:kbd:`Page Down` to move around
+* :kbd:`C-LeftArrow` and :kbd:`C-RightArrow` moves by words
+
* :kbd:`Home`/:kbd:`End` go to begin/end of line
* :kbd:`C-Home`/:kbd:`C-End` go to begin/end of file
-* Some :program:`Emacs` bindings may also work, including :kbd:`C-B`,
- :kbd:`C-P`, :kbd:`C-A`, :kbd:`C-E`, :kbd:`C-D`, :kbd:`C-L`
+* Some useful Emacs bindings are inherited from Tcl/Tk:
+
+ * :kbd:`C-a` beginning of line
+
+ * :kbd:`C-e` end of line
+
+ * :kbd:`C-k` kill line (but doesn't put it in clipboard)
+
+ * :kbd:`C-l` center window around the insertion point
+
+ * :kbd:`C-b` go backwards one character without deleting (usually you can
+ also use the cursor key for this)
+
+ * :kbd:`C-f` go forward one character without deleting (usually you can
+ also use the cursor key for this)
+
+ * :kbd:`C-p` go up one line (usually you can also use the cursor key for
+ this)
+
+ * :kbd:`C-d` delete next character
+
+Standard keybindings (like :kbd:`C-c` to copy and :kbd:`C-v` to paste)
+may work. Keybindings are selected in the Configure IDLE dialog.
Automatic indentation
@@ -246,27 +356,76 @@ Automatic indentation
After a block-opening statement, the next line is indented by 4 spaces (in the
Python Shell window by one tab). After certain keywords (break, return etc.)
the next line is dedented. In leading indentation, :kbd:`Backspace` deletes up
-to 4 spaces if they are there. :kbd:`Tab` inserts 1-4 spaces (in the Python
-Shell window one tab). See also the indent/dedent region commands in the edit
-menu.
-
+to 4 spaces if they are there. :kbd:`Tab` inserts spaces (in the Python
+Shell window one tab), number depends on Indent width. Currently tabs
+are restricted to four spaces due to Tcl/Tk limitations.
+
+See also the indent/dedent region commands in the edit menu.
+
+Completions
+^^^^^^^^^^^
+
+Completions are supplied for functions, classes, and attributes of classes,
+both built-in and user-defined. Completions are also provided for
+filenames.
+
+The AutoCompleteWindow (ACW) will open after a predefined delay (default is
+two seconds) after a '.' or (in a string) an os.sep is typed. If after one
+of those characters (plus zero or more other characters) a tab is typed
+the ACW will open immediately if a possible continuation is found.
+
+If there is only one possible completion for the characters entered, a
+:kbd:`Tab` will supply that completion without opening the ACW.
+
+'Show Completions' will force open a completions window, by default the
+:kbd:`C-space` will open a completions window. In an empty
+string, this will contain the files in the current directory. On a
+blank line, it will contain the built-in and user-defined functions and
+classes in the current name spaces, plus any modules imported. If some
+characters have been entered, the ACW will attempt to be more specific.
+
+If a string of characters is typed, the ACW selection will jump to the
+entry most closely matching those characters. Entering a :kbd:`tab` will
+cause the longest non-ambiguous match to be entered in the Editor window or
+Shell. Two :kbd:`tab` in a row will supply the current ACW selection, as
+will return or a double click. Cursor keys, Page Up/Down, mouse selection,
+and the scroll wheel all operate on the ACW.
+
+"Hidden" attributes can be accessed by typing the beginning of hidden
+name after a '.', e.g. '_'. This allows access to modules with
+``__all__`` set, or to class-private attributes.
+
+Completions and the 'Expand Word' facility can save a lot of typing!
+
+Completions are currently limited to those in the namespaces. Names in
+an Editor window which are not via ``__main__`` and :data:`sys.modules` will
+not be found. Run the module once with your imports to correct this situation.
+Note that IDLE itself places quite a few modules in sys.modules, so
+much can be found by default, e.g. the re module.
+
+If you don't like the ACW popping up unbidden, simply make the delay
+longer or disable the extension. Or another option is the delay could
+be set to zero. Another alternative to preventing ACW popups is to
+disable the call tips extension.
Python Shell window
^^^^^^^^^^^^^^^^^^^
-* :kbd:`C-C` interrupts executing command
+* :kbd:`C-c` interrupts executing command
-* :kbd:`C-D` sends end-of-file; closes window if typed at a ``>>>`` prompt
+* :kbd:`C-d` sends end-of-file; closes window if typed at a ``>>>`` prompt
+ (this is :kbd:`C-z` on Windows).
-* :kbd:`Alt-p` retrieves previous command matching what you have typed
+* :kbd:`Alt-/` (Expand word) is also useful to reduce typing
-* :kbd:`Alt-n` retrieves next
+ Command history
-* :kbd:`Return` while on any previous command retrieves that command
+ * :kbd:`Alt-p` retrieves previous command matching what you have typed. On
+ OS X use :kbd:`C-p`.
-* :kbd:`Alt-/` (Expand word) is also useful here
+ * :kbd:`Alt-n` retrieves next. On OS X use :kbd:`C-n`.
-.. index:: single: indentation
+ * :kbd:`Return` while on any previous command retrieves that command
Syntax colors
@@ -308,17 +467,17 @@ Startup
Upon startup with the ``-s`` option, IDLE will execute the file referenced by
the environment variables :envvar:`IDLESTARTUP` or :envvar:`PYTHONSTARTUP`.
-Idle first checks for ``IDLESTARTUP``; if ``IDLESTARTUP`` is present the file
-referenced is run. If ``IDLESTARTUP`` is not present, Idle checks for
+IDLE first checks for ``IDLESTARTUP``; if ``IDLESTARTUP`` is present the file
+referenced is run. If ``IDLESTARTUP`` is not present, IDLE checks for
``PYTHONSTARTUP``. Files referenced by these environment variables are
-convenient places to store functions that are used frequently from the Idle
+convenient places to store functions that are used frequently from the IDLE
shell, or for executing import statements to import common modules.
In addition, ``Tk`` also loads a startup file if it is present. Note that the
Tk file is loaded unconditionally. This additional file is ``.Idle.py`` and is
looked for in the user's home directory. Statements in this file will be
executed in the Tk namespace, so this file is not useful for importing functions
-to be used from Idle's Python shell.
+to be used from IDLE's Python shell.
Command line usage
@@ -349,3 +508,45 @@ If there are arguments:
the arguments are still available in ``sys.argv``.
+Additional help sources
+-----------------------
+
+IDLE includes a help menu entry called "Python Docs" that will open the
+extensive sources of help, including tutorials, available at docs.python.org.
+Selected URLs can be added or removed from the help menu at any time using the
+Configure IDLE dialog. See the IDLE help option in the help menu of IDLE for
+more information.
+
+
+Other preferences
+-----------------
+
+The font preferences, highlighting, keys, and general preferences can be
+changed via the Configure IDLE menu option. Be sure to note that
+keys can be user defined, IDLE ships with four built in key sets. In
+addition a user can create a custom key set in the Configure IDLE dialog
+under the keys tab.
+
+Extensions
+----------
+
+IDLE contains an extension facility. See the beginning of
+config-extensions.def in the idlelib directory for further information. The
+default extensions are currently:
+
+* FormatParagraph
+
+* AutoExpand
+
+* ZoomHeight
+
+* ScriptBinding
+
+* CallTips
+
+* ParenMatch
+
+* AutoComplete
+
+* CodeContext
+
diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst
index 01236fb..fa736fe 100644
--- a/Doc/library/imaplib.rst
+++ b/Doc/library/imaplib.rst
@@ -69,17 +69,25 @@ There's also a subclass for secure connections:
This is a subclass derived from :class:`IMAP4` that connects over an SSL
encrypted socket (to use this class you need a socket module that was compiled
with SSL support). If *host* is not specified, ``''`` (the local host) is used.
- If *port* is omitted, the standard IMAP4-over-SSL port (993) is used. *keyfile*
- and *certfile* are also optional - they can contain a PEM formatted private key
- and certificate chain file for the SSL connection. *ssl_context* parameter is a
- :class:`ssl.SSLContext` object which allows bundling SSL configuration
- options, certificates and private keys into a single (potentially long-lived)
- structure. Note that the *keyfile*/*certfile* parameters are mutually exclusive with *ssl_context*,
- a :class:`ValueError` is raised if *keyfile*/*certfile* is provided along with *ssl_context*.
+ If *port* is omitted, the standard IMAP4-over-SSL port (993) is used.
+ *ssl_context* is a :class:`ssl.SSLContext` object which allows bundling
+ SSL configuration options, certificates and private keys into a single
+ (potentially long-lived) structure. Please read :ref:`ssl-security` for
+ best practices.
+
+ *keyfile* and *certfile* are a legacy alternative to *ssl_context* - they
+ can point to PEM-formatted private key and certificate chain files for
+ the SSL connection. Note that the *keyfile*/*certfile* parameters are
+ mutually exclusive with *ssl_context*, a :class:`ValueError` is raised
+ if *keyfile*/*certfile* is provided along with *ssl_context*.
.. versionchanged:: 3.3
*ssl_context* parameter added.
+ .. versionchanged:: 3.4
+ The class now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
The second subclass allows for connections created by a child process:
@@ -433,10 +441,16 @@ An :class:`IMAP4` instance has the following methods:
Send a ``STARTTLS`` command. The *ssl_context* argument is optional
and should be a :class:`ssl.SSLContext` object. This will enable
- encryption on the IMAP connection.
+ encryption on the IMAP connection. Please read :ref:`ssl-security` for
+ best practices.
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ The method now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
+
.. method:: IMAP4.status(mailbox, names)
diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index 364d81e..ed4820e 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -3,7 +3,10 @@
.. module:: imp
:synopsis: Access the implementation of the import statement.
+ :deprecated:
+.. deprecated:: 3.4
+ The :mod:`imp` package is pending deprecation in favor of :mod:`importlib`.
.. index:: statement: import
@@ -11,10 +14,6 @@ This module provides an interface to the mechanisms used to implement the
:keyword:`import` statement. It defines the following constants and functions:
-.. note::
- New programs should use :mod:`importlib` rather than this module.
-
-
.. function:: get_magic()
.. index:: pair: file; byte-code
@@ -22,6 +21,9 @@ This module provides an interface to the mechanisms used to implement the
Return the magic string value used to recognize byte-compiled code files
(:file:`.pyc` files). (This value may be different for each Python version.)
+ .. deprecated:: 3.4
+ Use :attr:`importlib.util.MAGIC_NUMBER` instead.
+
.. function:: get_suffixes()
@@ -101,8 +103,10 @@ This module provides an interface to the mechanisms used to implement the
using a :keyword:`try` ... :keyword:`finally` statement.
.. deprecated:: 3.3
- Unneeded as loaders should be used to load modules and
- :func:`find_module` is deprecated.
+ If previously used in conjunction with :func:`imp.find_module` then
+ call ``load_module()`` on the returned loader. If you wish to load a
+ module from a specific file, then use one of the file-based loaders found
+ in :mod:`importlib.machinery`.
.. function:: new_module(name)
@@ -110,6 +114,9 @@ This module provides an interface to the mechanisms used to implement the
Return a new empty module object called *name*. This object is *not* inserted
in ``sys.modules``.
+ .. deprecated:: 3.4
+ Use :class:`types.ModuleType` instead.
+
.. function:: reload(module)
@@ -176,6 +183,9 @@ This module provides an interface to the mechanisms used to implement the
Relies on both ``__name__`` and ``__loader__`` being defined on the module
being reloaded instead of just ``__name__``.
+ .. deprecated:: 3.4
+ Use :func:`importlib.reload` instead.
+
The following functions are conveniences for handling :pep:`3147` byte-compiled
file paths.
@@ -201,6 +211,9 @@ file paths.
If :attr:`sys.implementation.cache_tag` is ``None``, then
:exc:`NotImplementedError` is raised.
+ .. deprecated:: 3.4
+ Use :func:`importlib.util.cache_from_source` instead.
+
.. function:: source_from_cache(path)
@@ -216,14 +229,17 @@ file paths.
Raise :exc:`NotImplementedError` when
:attr:`sys.implementation.cache_tag` is not defined.
+ .. deprecated:: 3.4
+ Use :func:`importlib.util.source_from_cache` instead.
+
.. function:: get_tag()
Return the :pep:`3147` magic tag string matching this version of Python's
magic number, as returned by :func:`get_magic`.
- .. note::
- You may use :attr:`sys.implementation.cache_tag` directly starting
+ .. deprecated:: 3.4
+ Use :attr:`sys.implementation.cache_tag` directly starting
in Python 3.3.
@@ -246,10 +262,12 @@ that circular imports work without any deadlocks.
exception is made for circular imports, which by construction have to
expose an incomplete module object at some point.
-.. versionchanged:: 3.3
- The locking scheme has changed to per-module locks for
- the most part. A global import lock is kept for some critical tasks,
- such as initializing the per-module locks.
+ .. versionchanged:: 3.3
+ The locking scheme has changed to per-module locks for
+ the most part. A global import lock is kept for some critical tasks,
+ such as initializing the per-module locks.
+
+ .. deprecated:: 3.4
.. function:: acquire_lock()
@@ -264,10 +282,12 @@ that circular imports work without any deadlocks.
On platforms without threads, this function does nothing.
-.. versionchanged:: 3.3
- The locking scheme has changed to per-module locks for
- the most part. A global import lock is kept for some critical tasks,
- such as initializing the per-module locks.
+ .. versionchanged:: 3.3
+ The locking scheme has changed to per-module locks for
+ the most part. A global import lock is kept for some critical tasks,
+ such as initializing the per-module locks.
+
+ .. deprecated:: 3.4
.. function:: release_lock()
@@ -275,10 +295,12 @@ that circular imports work without any deadlocks.
Release the interpreter's global import lock. On platforms without
threads, this function does nothing.
-.. versionchanged:: 3.3
- The locking scheme has changed to per-module locks for
- the most part. A global import lock is kept for some critical tasks,
- such as initializing the per-module locks.
+ .. versionchanged:: 3.3
+ The locking scheme has changed to per-module locks for
+ the most part. A global import lock is kept for some critical tasks,
+ such as initializing the per-module locks.
+
+ .. deprecated:: 3.4
The following constants with integer values, defined in this module, are used
@@ -345,6 +367,9 @@ to indicate the search result of :func:`find_module`.
``None`` is inserted into ``sys.path_importer_cache`` instead of an
instance of :class:`NullImporter`.
+ .. deprecated:: 3.4
+ Insert ``None`` into ``sys.path_importer_cache`` instead.
+
.. _examples-imp:
diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 5f740a2..f3b2063 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -95,7 +95,7 @@ Functions
Find the loader for a module, optionally within the specified *path*. If the
module is in :attr:`sys.modules`, then ``sys.modules[name].__loader__`` is
- returned (unless the loader would be ``None``, in which case
+ returned (unless the loader would be ``None`` or is not set, in which case
:exc:`ValueError` is raised). Otherwise a search using :attr:`sys.meta_path`
is done. ``None`` is returned if no loader is found.
@@ -104,6 +104,15 @@ Functions
will need to import all parent packages of the submodule and use the correct
argument to *path*.
+ .. versionadded:: 3.3
+
+ .. versionchanged:: 3.4
+ If ``__loader__`` is not set, raise :exc:`ValueError`, just like when the
+ attribute is set to ``None``.
+
+ .. deprecated:: 3.4
+ Use :func:`importlib.util.find_spec` instead.
+
.. function:: invalidate_caches()
Invalidate the internal caches of finders stored at
@@ -114,6 +123,74 @@ Functions
.. versionadded:: 3.3
+.. function:: reload(module)
+
+ Reload a previously imported *module*. The argument must be a module object,
+ so it must have been successfully imported before. This is useful if you
+ have edited the module source file using an external editor and want to try
+ out the new version without leaving the Python interpreter. The return value
+ is the module object (which can be different if re-importing causes a
+ different object to be placed in :data:`sys.modules`).
+
+ When :func:`reload` is executed:
+
+ * Python module's code is recompiled and the module-level code re-executed,
+ defining a new set of objects which are bound to names in the module's
+ dictionary by reusing the :term:`loader` which originally loaded the
+ module. The ``init`` function of extension modules is not called a second
+ time.
+
+ * As with all other objects in Python the old objects are only reclaimed
+ after their reference counts drop to zero.
+
+ * The names in the module namespace are updated to point to any new or
+ changed objects.
+
+ * Other references to the old objects (such as names external to the module) are
+ not rebound to refer to the new objects and must be updated in each namespace
+ where they occur if that is desired.
+
+ There are a number of other caveats:
+
+ If a module is syntactically correct but its initialization fails, the first
+ :keyword:`import` statement for it does not bind its name locally, but does
+ store a (partially initialized) module object in ``sys.modules``. To reload
+ the module you must first :keyword:`import` it again (this will bind the name
+ to the partially initialized module object) before you can :func:`reload` it.
+
+ When a module is reloaded, its dictionary (containing the module's global
+ variables) is retained. Redefinitions of names will override the old
+ definitions, so this is generally not a problem. If the new version of a
+ module does not define a name that was defined by the old version, the old
+ definition remains. This feature can be used to the module's advantage if it
+ maintains a global table or cache of objects --- with a :keyword:`try`
+ statement it can test for the table's presence and skip its initialization if
+ desired::
+
+ try:
+ cache
+ except NameError:
+ cache = {}
+
+ It is legal though generally not very useful to reload built-in or
+ dynamically loaded modules (this is not true for e.g. :mod:`sys`,
+ :mod:`__main__`, :mod:`builtins` and other key modules where reloading is
+ frowned upon). In many cases, however, extension modules are not designed to
+ be initialized more than once, and may fail in arbitrary ways when reloaded.
+
+ If a module imports objects from another module using :keyword:`from` ...
+ :keyword:`import` ..., calling :func:`reload` for the other module does not
+ redefine the objects imported from it --- one way around this is to
+ re-execute the :keyword:`from` statement, another is to use :keyword:`import`
+ and qualified names (*module.name*) instead.
+
+ If a module instantiates instances of a class, reloading the module that
+ defines the class does not affect the method definitions of the instances ---
+ they continue to use the old class definition. The same is true for derived
+ classes.
+
+ .. versionadded:: 3.4
+
:mod:`importlib.abc` -- Abstract base classes related to import
---------------------------------------------------------------
@@ -137,8 +214,6 @@ ABC hierarchy::
+-- ExecutionLoader --+
+-- FileLoader
+-- SourceLoader
- +-- PyLoader (deprecated)
- +-- PyPycLoader (deprecated)
.. class:: Finder
@@ -154,6 +229,10 @@ ABC hierarchy::
module. Originally specified in :pep:`302`, this method was meant
for use in :data:`sys.meta_path` and in the path-based import subsystem.
+ .. versionchanged:: 3.4
+ Returns ``None`` when called instead of raising
+ :exc:`NotImplementedError`.
+
.. class:: MetaPathFinder
@@ -162,20 +241,46 @@ ABC hierarchy::
.. versionadded:: 3.3
+ .. method:: find_spec(fullname, path, target=None)
+
+ An abstract method for finding a :term:`spec <module spec>` for
+ the specified module. If this is a top-level import, *path* will
+ be ``None``. Otherwise, this is a search for a subpackage or
+ module and *path* will be the value of :attr:`__path__` from the
+ parent package. If a spec cannot be found, ``None`` is returned.
+ When passed in, ``target`` is a module object that the finder may
+ use to make a more educated about what spec to return.
+
+ .. versionadded:: 3.4
+
.. method:: find_module(fullname, path)
- An abstract method for finding a :term:`loader` for the specified
+ A legacy method for finding a :term:`loader` for the specified
module. If this is a top-level import, *path* will be ``None``.
Otherwise, this is a search for a subpackage or module and *path*
will be the value of :attr:`__path__` from the parent
package. If a loader cannot be found, ``None`` is returned.
+ If :meth:`find_spec` is defined, backwards-compatible functionality is
+ provided.
+
+ .. versionchanged:: 3.4
+ Returns ``None`` when called instead of raising
+ :exc:`NotImplementedError`. Can use :meth:`find_spec` to provide
+ functionality.
+
+ .. deprecated:: 3.4
+ Use :meth:`find_spec` instead.
+
.. method:: invalidate_caches()
An optional method which, when called, should invalidate any internal
cache used by the finder. Used by :func:`importlib.invalidate_caches`
when invalidating the caches of all finders on :data:`sys.meta_path`.
+ .. versionchanged:: 3.4
+ Returns ``None`` when called instead of ``NotImplemented``.
+
.. class:: PathEntryFinder
@@ -183,27 +288,51 @@ ABC hierarchy::
it bears some similarities to :class:`MetaPathFinder`, ``PathEntryFinder``
is meant for use only within the path-based import subsystem provided
by :class:`PathFinder`. This ABC is a subclass of :class:`Finder` for
- compatibility.
+ compatibility reasons only.
.. versionadded:: 3.3
+ .. method:: find_spec(fullname, target=None)
+
+ An abstract method for finding a :term:`spec <module spec>` for
+ the specified module. The finder will search for the module only
+ within the :term:`path entry` to which it is assigned. If a spec
+ cannot be found, ``None`` is returned. When passed in, ``target``
+ is a module object that the finder may use to make a more educated
+ about what spec to return.
+
+ .. versionadded:: 3.4
+
.. method:: find_loader(fullname)
- An abstract method for finding a :term:`loader` for the specified
+ A legacy method for finding a :term:`loader` for the specified
module. Returns a 2-tuple of ``(loader, portion)`` where ``portion``
is a sequence of file system locations contributing to part of a namespace
package. The loader may be ``None`` while specifying ``portion`` to
signify the contribution of the file system locations to a namespace
package. An empty list can be used for ``portion`` to signify the loader
- is not part of a package. If ``loader`` is ``None`` and ``portion`` is
- the empty list then no loader or location for a namespace package were
- found (i.e. failure to find anything for the module).
+ is not part of a namespace package. If ``loader`` is ``None`` and
+ ``portion`` is the empty list then no loader or location for a namespace
+ package were found (i.e. failure to find anything for the module).
+
+ If :meth:`find_spec` is defined then backwards-compatible functionality is
+ provided.
+
+ .. versionchanged:: 3.4
+ Returns ``(None, [])`` instead of raising :exc:`NotImplementedError`.
+ Uses :meth:`find_spec` when available to provide functionality.
+
+ .. deprecated:: 3.4
+ Use :meth:`find_spec` instead.
.. method:: find_module(fullname)
A concrete implementation of :meth:`Finder.find_module` which is
equivalent to ``self.find_loader(fullname)[0]``.
+ .. deprecated:: 3.4
+ Use :meth:`find_spec` instead.
+
.. method:: invalidate_caches()
An optional method which, when called, should invalidate any internal
@@ -216,9 +345,26 @@ ABC hierarchy::
An abstract base class for a :term:`loader`.
See :pep:`302` for the exact definition for a loader.
+ .. method:: create_module(spec)
+
+ An optional method that returns the module object to use when
+ importing a module. create_module() may also return ``None``,
+ indicating that the default module creation should take place
+ instead.
+
+ .. versionadded:: 3.4
+
+ .. method:: exec_module(module)
+
+ An abstract method that executes the module in its own namespace
+ when a module is imported or reloaded. The module should already
+ be initialized when exec_module() is called.
+
+ .. versionadded:: 3.4
+
.. method:: load_module(fullname)
- An abstract method for loading a module. If the module cannot be
+ A legacy method for loading a module. If the module cannot be
loaded, :exc:`ImportError` is raised, otherwise the loaded module is
returned.
@@ -229,12 +375,11 @@ ABC hierarchy::
from the import. If the loader inserted a module and the load fails, it
must be removed by the loader from :data:`sys.modules`; modules already
in :data:`sys.modules` before the loader began execution should be left
- alone. The :func:`importlib.util.module_for_loader` decorator handles
- all of these details.
+ alone (see :func:`importlib.util.module_for_loader`).
The loader should set several attributes on the module.
(Note that some of these attributes can change when a module is
- reloaded.)
+ reloaded):
- :attr:`__name__`
The name of the module.
@@ -254,20 +399,42 @@ ABC hierarchy::
- :attr:`__package__`
The parent package for the module/package. If the module is
top-level then it has a value of the empty string. The
- :func:`importlib.util.set_package` decorator can handle the details
- for :attr:`__package__`.
+ :func:`importlib.util.module_for_loader` decorator can handle the
+ details for :attr:`__package__`.
- :attr:`__loader__`
- The loader used to load the module.
- (This is not set by the built-in import machinery,
- but it should be set whenever a :term:`loader` is used.)
+ The loader used to load the module. The
+ :func:`importlib.util.module_for_loader` decorator can handle the
+ details for :attr:`__package__`.
+
+ When :meth:`exec_module` is available then backwards-compatible
+ functionality is provided.
+
+ .. versionchanged:: 3.4
+ Raise :exc:`ImportError` when called instead of
+ :exc:`NotImplementedError`. Functionality provided when
+ :meth:`exec_module` is available.
+
+ .. deprecated:: 3.4
+ The recommended API for loading a module is :meth:`exec_module`
+ (and optionally :meth:`create_module`). Loaders should implement
+ it instead of load_module(). The import machinery takes care of
+ all the other responsibilities of load_module() when exec_module()
+ is implemented.
.. method:: module_repr(module)
- An abstract method which when implemented calculates and returns the
- given module's repr, as a string.
+ A legacy method which when implemented calculates and returns the
+ given module's repr, as a string. The module type's default repr() will
+ use the result of this method as appropriate.
+
+ .. versionadded:: 3.3
+
+ .. versionchanged:: 3.4
+ Made optional instead of an abstractmethod.
- .. versionadded: 3.3
+ .. deprecated:: 3.4
+ The import machinery now takes care of this automatically.
.. class:: ResourceLoader
@@ -286,6 +453,9 @@ ABC hierarchy::
be found. The *path* is expected to be constructed using a module's
:attr:`__file__` attribute or an item from a package's :attr:`__path__`.
+ .. versionchanged:: 3.4
+ Raises :exc:`IOError` instead of :exc:`NotImplementedError`.
+
.. class:: InspectLoader
@@ -294,14 +464,21 @@ ABC hierarchy::
.. method:: get_code(fullname)
- An abstract method to return the :class:`code` object for a module.
- ``None`` is returned if the module does not have a code object
- (e.g. built-in module). :exc:`ImportError` is raised if loader cannot
- find the requested module.
+ Return the code object for a module, or ``None`` if the module does not
+ have a code object (as would be the case, for example, for a built-in
+ module). Raise an :exc:`ImportError` if loader cannot find the
+ requested module.
+
+ .. note::
+ While the method has a default implementation, it is suggested that
+ it be overridden if possible for performance.
.. index::
single: universal newlines; importlib.abc.InspectLoader.get_source method
+ .. versionchanged:: 3.4
+ No longer abstract and a concrete implementation is provided.
+
.. method:: get_source(fullname)
An abstract method to return the source of a module. It is returned as
@@ -310,12 +487,42 @@ ABC hierarchy::
if no source is available (e.g. a built-in module). Raises
:exc:`ImportError` if the loader cannot find the module specified.
+ .. versionchanged:: 3.4
+ Raises :exc:`ImportError` instead of :exc:`NotImplementedError`.
+
.. method:: is_package(fullname)
An abstract method to return a true value if the module is a package, a
false value otherwise. :exc:`ImportError` is raised if the
:term:`loader` cannot find the module.
+ .. versionchanged:: 3.4
+ Raises :exc:`ImportError` instead of :exc:`NotImplementedError`.
+
+ .. method:: source_to_code(data, path='<string>')
+
+ Create a code object from Python source.
+
+ The *data* argument can be whatever the :func:`compile` function
+ supports (i.e. string or bytes). The *path* argument should be
+ the "path" to where the source code originated from, which can be an
+ abstract concept (e.g. location in a zip file).
+
+ .. versionadded:: 3.4
+
+ .. method:: exec_module(module)
+
+ Implementation of :meth:`Loader.exec_module`.
+
+ .. versionadded:: 3.4
+
+ .. method:: load_module(fullname)
+
+ Implementation of :meth:`Loader.load_module`.
+
+ .. deprecated:: 3.4
+ use :meth:`exec_module` instead.
+
.. class:: ExecutionLoader
@@ -333,6 +540,9 @@ ABC hierarchy::
the source file, regardless of whether a bytecode was used to load the
module.
+ .. versionchanged:: 3.4
+ Raises :exc:`ImportError` instead of :exc:`NotImplementedError`.
+
.. class:: FileLoader(fullname, path)
@@ -357,13 +567,16 @@ ABC hierarchy::
Calls super's ``load_module()``.
+ .. deprecated:: 3.4
+ Use :meth:`Loader.exec_module` instead.
+
.. method:: get_filename(fullname)
Returns :attr:`path`.
.. method:: get_data(path)
- Returns the open, binary file for *path*.
+ Reads *path* as a binary file and returns the bytes from it.
.. class:: SourceLoader
@@ -378,7 +591,8 @@ ABC hierarchy::
loading is not supported.
The abstract methods defined by this class are to add optional bytecode
- file support. Not implementing these optional methods causes the loader to
+ file support. Not implementing these optional methods (or causing them to
+ raise :exc:`NotImplementedError`) causes the loader to
only work with source code. Implementing the methods allows the loader to
work with source *and* bytecode files; it does not allow for *sourceless*
loading where only bytecode is provided. Bytecode files are an
@@ -395,10 +609,13 @@ ABC hierarchy::
- ``'size'`` (optional): the size in bytes of the source code.
Any other keys in the dictionary are ignored, to allow for future
- extensions.
+ extensions. If the path cannot be handled, :exc:`IOError` is raised.
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ Raise :exc:`IOError` instead of :exc:`NotImplementedError`.
+
.. method:: path_mtime(path)
Optional abstract method which returns the modification time for the
@@ -407,7 +624,10 @@ ABC hierarchy::
.. deprecated:: 3.3
This method is deprecated in favour of :meth:`path_stats`. You don't
have to implement it, but it is still available for compatibility
- purposes.
+ purposes. Raise :exc:`IOError` if the path cannot be handled.
+
+ .. versionchanged:: 3.4
+ Raise :exc:`IOError` instead of :exc:`NotImplementedError`.
.. method:: set_data(path, data)
@@ -419,13 +639,25 @@ ABC hierarchy::
(:attr:`errno.EACCES`/:exc:`PermissionError`), do not propagate the
exception.
+ .. versionchanged:: 3.4
+ No longer raises :exc:`NotImplementedError` when called.
+
.. method:: get_code(fullname)
Concrete implementation of :meth:`InspectLoader.get_code`.
+ .. method:: exec_module(module)
+
+ Concrete implementation of :meth:`Loader.exec_module`.
+
+ .. versionadded:: 3.4
+
.. method:: load_module(fullname)
- Concrete implementation of :meth:`Loader.load_module`.
+ Concrete implementation of :meth:`Loader.load_module`.
+
+ .. deprecated:: 3.4
+ Use :meth:`exec_module` instead.
.. method:: get_source(fullname)
@@ -440,142 +672,6 @@ ABC hierarchy::
itself does not end in ``__init__``.
-.. class:: PyLoader
-
- An abstract base class inheriting from
- :class:`ExecutionLoader` and
- :class:`ResourceLoader` designed to ease the loading of
- Python source modules (bytecode is not handled; see
- :class:`SourceLoader` for a source/bytecode ABC). A subclass
- implementing this ABC will only need to worry about exposing how the source
- code is stored; all other details for loading Python source code will be
- handled by the concrete implementations of key methods.
-
- .. deprecated:: 3.2
- This class has been deprecated in favor of :class:`SourceLoader` and is
- slated for removal in Python 3.4. See below for how to create a
- subclass that is compatible with Python 3.1 onwards.
-
- If compatibility with Python 3.1 is required, then use the following idiom
- to implement a subclass that will work with Python 3.1 onwards (make sure
- to implement :meth:`ExecutionLoader.get_filename`)::
-
- try:
- from importlib.abc import SourceLoader
- except ImportError:
- from importlib.abc import PyLoader as SourceLoader
-
-
- class CustomLoader(SourceLoader):
- def get_filename(self, fullname):
- """Return the path to the source file."""
- # Implement ...
-
- def source_path(self, fullname):
- """Implement source_path in terms of get_filename."""
- try:
- return self.get_filename(fullname)
- except ImportError:
- return None
-
- def is_package(self, fullname):
- """Implement is_package by looking for an __init__ file
- name as returned by get_filename."""
- filename = os.path.basename(self.get_filename(fullname))
- return os.path.splitext(filename)[0] == '__init__'
-
-
- .. method:: source_path(fullname)
-
- An abstract method that returns the path to the source code for a
- module. Should return ``None`` if there is no source code.
- Raises :exc:`ImportError` if the loader knows it cannot handle the
- module.
-
- .. method:: get_filename(fullname)
-
- A concrete implementation of
- :meth:`importlib.abc.ExecutionLoader.get_filename` that
- relies on :meth:`source_path`. If :meth:`source_path` returns
- ``None``, then :exc:`ImportError` is raised.
-
- .. method:: load_module(fullname)
-
- A concrete implementation of :meth:`importlib.abc.Loader.load_module`
- that loads Python source code. All needed information comes from the
- abstract methods required by this ABC. The only pertinent assumption
- made by this method is that when loading a package
- :attr:`__path__` is set to ``[os.path.dirname(__file__)]``.
-
- .. method:: get_code(fullname)
-
- A concrete implementation of
- :meth:`importlib.abc.InspectLoader.get_code` that creates code objects
- from Python source code, by requesting the source code (using
- :meth:`source_path` and :meth:`get_data`) and compiling it with the
- built-in :func:`compile` function.
-
- .. method:: get_source(fullname)
-
- A concrete implementation of
- :meth:`importlib.abc.InspectLoader.get_source`. Uses
- :meth:`importlib.abc.ResourceLoader.get_data` and :meth:`source_path`
- to get the source code. It tries to guess the source encoding using
- :func:`tokenize.detect_encoding`.
-
-
-.. class:: PyPycLoader
-
- An abstract base class inheriting from :class:`PyLoader`.
- This ABC is meant to help in creating loaders that support both Python
- source and bytecode.
-
- .. deprecated:: 3.2
- This class has been deprecated in favor of :class:`SourceLoader` and to
- properly support :pep:`3147`. If compatibility is required with
- Python 3.1, implement both :class:`SourceLoader` and :class:`PyLoader`;
- instructions on how to do so are included in the documentation for
- :class:`PyLoader`. Do note that this solution will not support
- sourceless/bytecode-only loading; only source *and* bytecode loading.
-
- .. versionchanged:: 3.3
- Updated to parse (but not use) the new source size field in bytecode
- files when reading and to write out the field properly when writing.
-
- .. method:: source_mtime(fullname)
-
- An abstract method which returns the modification time for the source
- code of the specified module. The modification time should be an
- integer. If there is no source code, return ``None``. If the
- module cannot be found then :exc:`ImportError` is raised.
-
- .. method:: bytecode_path(fullname)
-
- An abstract method which returns the path to the bytecode for the
- specified module, if it exists. It returns ``None``
- if no bytecode exists (yet).
- Raises :exc:`ImportError` if the loader knows it cannot handle the
- module.
-
- .. method:: get_filename(fullname)
-
- A concrete implementation of
- :meth:`ExecutionLoader.get_filename` that relies on
- :meth:`PyLoader.source_path` and :meth:`bytecode_path`.
- If :meth:`source_path` returns a path, then that value is returned.
- Else if :meth:`bytecode_path` returns a path, that path will be
- returned. If a path is not available from both methods,
- :exc:`ImportError` is raised.
-
- .. method:: write_bytecode(fullname, bytecode)
-
- An abstract method which has the loader write *bytecode* for future
- use. If the bytecode is written, return ``True``. Return
- ``False`` if the bytecode could not be written. This method
- should not be called if :data:`sys.dont_write_bytecode` is true.
- The *bytecode* argument should be a bytes string or bytes array.
-
-
:mod:`importlib.machinery` -- Importers and path hooks
------------------------------------------------------
@@ -642,6 +738,10 @@ find and load modules.
Only class methods are defined by this class to alleviate the need for
instantiation.
+ .. note::
+ Due to limitations in the extension module C-API, for now
+ BuiltinImporter does not implement :meth:`Loader.exec_module`.
+
.. class:: FrozenImporter
@@ -672,24 +772,37 @@ find and load modules.
Only class methods are defined by this class to alleviate the need for
instantiation.
+ .. classmethod:: find_spec(fullname, path=None, target=None)
+
+ Class method that attempts to find a :term:`spec <module spec>`
+ for the module specified by *fullname* on :data:`sys.path` or, if
+ defined, on *path*. For each path entry that is searched,
+ :data:`sys.path_importer_cache` is checked. If a non-false object
+ is found then it is used as the :term:`path entry finder` to look
+ for the module being searched for. If no entry is found in
+ :data:`sys.path_importer_cache`, then :data:`sys.path_hooks` is
+ searched for a finder for the path entry and, if found, is stored
+ in :data:`sys.path_importer_cache` along with being queried about
+ the module. If no finder is ever found then ``None`` is both
+ stored in the cache and returned.
+
+ .. versionadded:: 3.4
+
.. classmethod:: find_module(fullname, path=None)
- Class method that attempts to find a :term:`loader` for the module
- specified by *fullname* on :data:`sys.path` or, if defined, on
- *path*. For each path entry that is searched,
- :data:`sys.path_importer_cache` is checked. If a non-false object is
- found then it is used as the :term:`path entry finder` to look for the
- module being searched for. If no entry is found in
- :data:`sys.path_importer_cache`, then :data:`sys.path_hooks` is
- searched for a finder for the path entry and, if found, is stored in
- :data:`sys.path_importer_cache` along with being queried about the
- module. If no finder is ever found then ``None`` is both stored in
- the cache and returned.
+ A legacy wrapper around :meth:`find_spec`.
+
+ .. deprecated:: 3.4
+ Use :meth:`find_spec` instead.
.. classmethod:: invalidate_caches()
- Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all
- finders stored in :attr:`sys.path_importer_cache`.
+ Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all
+ finders stored in :attr:`sys.path_importer_cache`.
+
+ .. versionchanged:: 3.4
+ Calls objects in :data:`sys.path_hooks` with the current working
+ directory for ``''`` (i.e. the empty string).
.. class:: FileFinder(path, \*loader_details)
@@ -721,6 +834,12 @@ find and load modules.
The path the finder will search in.
+ .. method:: find_spec(fullname, target=None)
+
+ Attempt to find the spec to handle *fullname* within :attr:`path`.
+
+ .. versionadded:: 3.4
+
.. method:: find_loader(fullname)
Attempt to find the loader to handle *fullname* within :attr:`path`.
@@ -805,7 +924,7 @@ find and load modules.
.. class:: ExtensionFileLoader(fullname, path)
- A concrete implementation of :class:`importlib.abc.InspectLoader` for
+ A concrete implementation of :class:`importlib.abc.ExecutionLoader` for
extension modules.
The *fullname* argument specifies the name of the module the loader is to
@@ -826,6 +945,10 @@ find and load modules.
Loads the extension module if and only if *fullname* is the same as
:attr:`name` or is ``None``.
+ .. note::
+ Due to limitations in the extension module C-API, for now
+ ExtensionFileLoader does not implement :meth:`Loader.exec_module`.
+
.. method:: is_package(fullname)
Returns ``True`` if the file path points to a package's ``__init__``
@@ -839,6 +962,70 @@ find and load modules.
Returns ``None`` as extension modules do not have source code.
+ .. method:: get_filename(fullname)
+
+ Returns :attr:`path`.
+
+ .. versionadded:: 3.4
+
+
+.. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None)
+
+ A specification for a module's import-system-related state.
+
+ .. versionadded:: 3.4
+
+ .. attribute:: name
+
+ (``__name__``)
+
+ A string for the fully-qualified name of the module.
+
+ .. attribute:: loader
+
+ (``__loader__``)
+
+ The loader to use for loading. For namespace packages this should be
+ set to None.
+
+ .. attribute:: origin
+
+ (``__file__``)
+
+ Name of the place from which the module is loaded, e.g. "builtin" for
+ built-in modules and the filename for modules loaded from source.
+ Normally "origin" should be set, but it may be None (the default)
+ which indicates it is unspecified.
+
+ .. attribute:: submodule_search_locations
+
+ (``__path__``)
+
+ List of strings for where to find submodules, if a package (None
+ otherwise).
+
+ .. attribute:: loader_state
+
+ Container of extra module-specific data for use during loading (or
+ None).
+
+ .. attribute:: cached
+
+ (``__cached__``)
+
+ String for where the compiled module should be stored (or None).
+
+ .. attribute:: parent
+
+ (``__package__``)
+
+ (Read-only) Fully-qualified name of the package to which the module
+ belongs as a submodule (or None).
+
+ .. attribute:: has_location
+
+ Boolean indicating whether or not the module's "origin"
+ attribute refers to a loadable location.
:mod:`importlib.util` -- Utility code for importers
---------------------------------------------------
@@ -849,6 +1036,51 @@ find and load modules.
This module contains the various objects that help in the construction of
an :term:`importer`.
+.. attribute:: MAGIC_NUMBER
+
+ The bytes which represent the bytecode version number. If you need help with
+ loading/writing bytecode then consider :class:`importlib.abc.SourceLoader`.
+
+ .. versionadded:: 3.4
+
+.. function:: cache_from_source(path, debug_override=None)
+
+ Return the :pep:`3147` path to the byte-compiled file associated with the
+ source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return
+ value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2.
+ The ``cpython-32`` string comes from the current magic tag (see
+ :func:`get_tag`; if :attr:`sys.implementation.cache_tag` is not defined then
+ :exc:`NotImplementedError` will be raised). The returned path will end in
+ ``.pyc`` when ``__debug__`` is ``True`` or ``.pyo`` for an optimized Python
+ (i.e. ``__debug__`` is ``False``). By passing in ``True`` or ``False`` for
+ *debug_override* you can override the system's value for ``__debug__`` for
+ extension selection.
+
+ *path* need not exist.
+
+ .. versionadded:: 3.4
+
+
+.. function:: source_from_cache(path)
+
+ Given the *path* to a :pep:`3147` file name, return the associated source code
+ file path. For example, if *path* is
+ ``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be
+ ``/foo/bar/baz.py``. *path* need not exist, however if it does not conform
+ to :pep:`3147` format, a ``ValueError`` is raised. If
+ :attr:`sys.implementation.cache_tag` is not defined,
+ :exc:`NotImplementedError` is raised.
+
+ .. versionadded:: 3.4
+
+.. function:: decode_source(source_bytes)
+
+ Decode the given bytes representing source code and return it as a string
+ with universal newlines (as required by
+ :meth:`importlib.abc.InspectLoader.get_source`).
+
+ .. versionadded:: 3.4
+
.. function:: resolve_name(name, package)
Resolve a relative module name to an absolute one.
@@ -865,9 +1097,25 @@ an :term:`importer`.
.. versionadded:: 3.3
+.. function:: find_spec(name, package=None)
+
+ Find the :term:`spec <module spec>` for a module, optionally relative to
+ the specified **package** name. If the module is in :attr:`sys.modules`,
+ then ``sys.modules[name].__spec__`` is returned (unless the spec would be
+ ``None`` or is not set, in which case :exc:`ValueError` is raised).
+ Otherwise a search using :attr:`sys.meta_path` is done. ``None`` is
+ returned if no spec is found.
+
+ If **name** is for a submodule (contains a dot), the parent module is
+ automatically imported.
+
+ **name** and **package** work the same as for :func:`import_module`.
+
+ .. versionadded:: 3.4
+
.. decorator:: module_for_loader
- A :term:`decorator` for a :term:`loader` method,
+ A :term:`decorator` for :meth:`importlib.abc.Loader.load_module`
to handle selecting the proper
module object to load with. The decorated method is expected to have a call
signature taking two positional arguments
@@ -878,55 +1126,68 @@ an :term:`importer`.
The decorated method will take in the **name** of the module to be loaded
as expected for a :term:`loader`. If the module is not found in
- :data:`sys.modules` then a new one is constructed with its
- :attr:`__name__` attribute set to **name**, :attr:`__loader__` set to
- **self**, and :attr:`__package__` set if
- :meth:`importlib.abc.InspectLoader.is_package` is defined for **self** and
- does not raise :exc:`ImportError` for **name**. If a new module is not
- needed then the module found in :data:`sys.modules` will be passed into the
- method.
+ :data:`sys.modules` then a new one is constructed. Regardless of where the
+ module came from, :attr:`__loader__` set to **self** and :attr:`__package__`
+ is set based on what :meth:`importlib.abc.InspectLoader.is_package` returns
+ (if available). These attributes are set unconditionally to support
+ reloading.
If an exception is raised by the decorated method and a module was added to
- :data:`sys.modules` it will be removed to prevent a partially initialized
- module from being in left in :data:`sys.modules`. If the module was already
- in :data:`sys.modules` then it is left alone.
-
- Use of this decorator handles all the details of which module object a
- loader should initialize as specified by :pep:`302` as best as possible.
+ :data:`sys.modules`, then the module will be removed to prevent a partially
+ initialized module from being in left in :data:`sys.modules`. If the module
+ was already in :data:`sys.modules` then it is left alone.
.. versionchanged:: 3.3
:attr:`__loader__` and :attr:`__package__` are automatically set
(when possible).
-.. decorator:: set_loader
+ .. versionchanged:: 3.4
+ Set :attr:`__name__`, :attr:`__loader__` :attr:`__package__`
+ unconditionally to support reloading.
- A :term:`decorator` for a :term:`loader` method,
- to set the :attr:`__loader__`
- attribute on loaded modules. If the attribute is already set the decorator
- does nothing. It is assumed that the first positional argument to the
- wrapped method (i.e. ``self``) is what :attr:`__loader__` should be set to.
+ .. deprecated:: 3.4
+ The import machinery now directly performs all the functionality
+ provided by this function.
- .. note::
+.. decorator:: set_loader
- It is recommended that :func:`module_for_loader` be used over this
- decorator as it subsumes this functionality.
+ A :term:`decorator` for :meth:`importlib.abc.Loader.load_module`
+ to set the :attr:`__loader__`
+ attribute on the returned module. If the attribute is already set the
+ decorator does nothing. It is assumed that the first positional argument to
+ the wrapped method (i.e. ``self``) is what :attr:`__loader__` should be set
+ to.
+ .. versionchanged:: 3.4
+ Set ``__loader__`` if set to ``None``, as if the attribute does not
+ exist.
+
+ .. deprecated:: 3.4
+ The import machinery takes care of this automatically.
.. decorator:: set_package
- A :term:`decorator` for a :term:`loader` to set the :attr:`__package__`
- attribute on the module returned by the loader. If :attr:`__package__` is
- set and has a value other than ``None`` it will not be changed.
- Note that the module returned by the loader is what has the attribute
- set on and not the module found in :data:`sys.modules`.
+ A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the :attr:`__package__` attribute on the returned module. If :attr:`__package__`
+ is set and has a value other than ``None`` it will not be changed.
+
+ .. deprecated:: 3.4
+ The import machinery takes care of this automatically.
+
+.. function:: spec_from_loader(name, loader, *, origin=None, is_package=None)
+
+ A factory function for creating a :class:`ModuleSpec` instance based
+ on a loader. The parameters have the same meaning as they do for
+ ModuleSpec. The function uses available :term:`loader` APIs, such as
+ :meth:`InspectLoader.is_package`, to fill in any missing
+ information on the spec.
+
+ .. versionadded:: 3.4
- Reliance on this decorator is discouraged when it is possible to set
- :attr:`__package__` before importing. By
- setting it beforehand the code for the module is executed with the
- attribute set and thus can be used by global level code during
- initialization.
+.. function:: spec_from_file_location(name, location, *, loader=None, submodule_search_locations=None)
- .. note::
+ A factory function for creating a :class:`ModuleSpec` instance based
+ on the path to a file. Missing information will be filled in on the
+ spec by making use of loader APIs and by the implication that the
+ module will be file-based.
- It is recommended that :func:`module_for_loader` be used over this
- decorator as it subsumes this functionality.
+ .. versionadded:: 3.4
diff --git a/Doc/library/index.rst b/Doc/library/index.rst
index 1b25c6e..81289a5 100644
--- a/Doc/library/index.rst
+++ b/Doc/library/index.rst
@@ -65,6 +65,7 @@ the `Python Package Index <http://pypi.python.org/pypi>`_.
tk.rst
development.rst
debug.rst
+ distribution.rst
python.rst
custominterp.rst
modules.rst
diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst
index d4cf905..0c08712 100644
--- a/Doc/library/inspect.rst
+++ b/Doc/library/inspect.rst
@@ -69,7 +69,12 @@ attributes:
| | | :term:`bytecode` |
+-----------+-----------------+---------------------------+
| | __defaults__ | tuple of any default |
-| | | values for arguments |
+| | | values for positional or |
+| | | keyword parameters |
++-----------+-----------------+---------------------------+
+| | __kwdefaults__ | mapping of any default |
+| | | values for keyword-only |
+| | | parameters |
+-----------+-----------------+---------------------------+
| | __globals__ | global namespace in which |
| | | this function was defined |
@@ -173,8 +178,9 @@ attributes:
.. note::
- :func:`getmembers` does not return metaclass attributes when the argument
- is a class (this behavior is inherited from the :func:`dir` function).
+ :func:`getmembers` will only return class attributes defined in the
+ metaclass when the argument is a class and those attributes have been
+ listed in the metaclass' custom :meth:`__dir__`.
.. function:: getmoduleinfo(path)
@@ -428,11 +434,14 @@ function.
Accepts a wide range of python callables, from plain functions and classes to
:func:`functools.partial` objects.
+ Raises :exc:`ValueError` if no signature can be provided, and
+ :exc:`TypeError` if that type of object is not supported.
+
.. note::
Some callables may not be introspectable in certain implementations of
- Python. For example, in CPython, built-in functions defined in C provide
- no metadata about their arguments.
+ Python. For example, in CPython, some built-in functions defined in
+ C provide no metadata about their arguments.
.. class:: Signature(parameters=None, \*, return_annotation=Signature.empty)
@@ -510,9 +519,8 @@ function.
.. attribute:: Parameter.name
- The name of the parameter as a string. Must be a valid python identifier
- name (with the exception of ``POSITIONAL_ONLY`` parameters, which can have
- it set to ``None``).
+ The name of the parameter as a string. The name must be a valid
+ Python identifier.
.. attribute:: Parameter.default
@@ -596,6 +604,10 @@ function.
>>> str(param.replace(default=Parameter.empty, annotation='spam'))
"foo:'spam'"
+ .. versionchanged:: 3.4
+ In Python 3.3 Parameter objects were allowed to have ``name`` set
+ to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``.
+ This is no longer permitted.
.. class:: BoundArguments
@@ -717,6 +729,11 @@ Classes and functions
Consider using the new :ref:`Signature Object <inspect-signature-object>`
interface, which provides a better way of introspecting functions.
+ .. versionchanged:: 3.4
+ This function is now based on :func:`signature`, but still ignores
+ ``__wrapped__`` attributes and includes the already bound first
+ parameter in the signature output for bound methods.
+
.. function:: getargvalues(frame)
@@ -806,6 +823,23 @@ Classes and functions
.. versionadded:: 3.3
+.. function:: unwrap(func, *, stop=None)
+
+ Get the object wrapped by *func*. It follows the chain of :attr:`__wrapped__`
+ attributes returning the last object in the chain.
+
+ *stop* is an optional callback accepting an object in the wrapper chain
+ as its sole argument that allows the unwrapping to be terminated early if
+ the callback returns a true value. If the callback never returns a true
+ value, the last object in the chain is returned as usual. For example,
+ :func:`signature` uses this to stop unwrapping if any object in the
+ chain has a ``__signature__`` attribute defined.
+
+ :exc:`ValueError` is raised if a cycle is encountered.
+
+ .. versionadded:: 3.4
+
+
.. _inspect-stack:
The interpreter stack
@@ -838,6 +872,10 @@ index of the current line within that list.
finally:
del frame
+ If you want to keep the frame around (for example to print a traceback
+ later), you can also break reference cycles by using the
+ :meth:`frame.clear` method.
+
The optional *context* argument supported by most of these functions specifies
the number of lines of context to return, which are centered around the current
line.
@@ -994,3 +1032,22 @@ updated as expected:
return an empty dictionary.
.. versionadded:: 3.3
+
+
+.. _inspect-module-cli:
+
+Command Line Interface
+----------------------
+
+The :mod:`inspect` module also provides a basic introspection capability
+from the command line.
+
+.. program:: inspect
+
+By default, accepts the name of a module and prints the source of that
+module. A class or function within the module can be printed instead by
+appended a colon and the qualified name of the target object.
+
+.. cmdoption:: --details
+
+ Print information about the specified object rather than the source code
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 716767f..79f65e0 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -283,10 +283,10 @@ I/O Base Classes
Return ``True`` if the stream can be read from. If ``False``, :meth:`read`
will raise :exc:`OSError`.
- .. method:: readline(limit=-1)
+ .. method:: readline(size=-1)
- Read and return one line from the stream. If *limit* is specified, at
- most *limit* bytes will be read.
+ Read and return one line from the stream. If *size* is specified, at
+ most *size* bytes will be read.
The line terminator is always ``b'\n'`` for binary files; for text files,
the *newlines* argument to :func:`open` can be used to select the line
@@ -366,14 +366,14 @@ I/O Base Classes
In addition to the attributes and methods from :class:`IOBase`,
:class:`RawIOBase` provides the following methods:
- .. method:: read(n=-1)
+ .. method:: read(size=-1)
- Read up to *n* bytes from the object and return them. As a convenience,
- if *n* is unspecified or -1, :meth:`readall` is called. Otherwise,
- only one system call is ever made. Fewer than *n* bytes may be
- returned if the operating system call returns fewer than *n* bytes.
+ Read up to *size* bytes from the object and return them. As a convenience,
+ if *size* is unspecified or -1, :meth:`readall` is called. Otherwise,
+ only one system call is ever made. Fewer than *size* bytes may be
+ returned if the operating system call returns fewer than *size* bytes.
- If 0 bytes are returned, and *n* was not 0, this indicates end of file.
+ If 0 bytes are returned, and *size* was not 0, this indicates end of file.
If the object is in non-blocking mode and no bytes are available,
``None`` is returned.
@@ -442,10 +442,10 @@ I/O Base Classes
.. versionadded:: 3.1
- .. method:: read(n=-1)
+ .. method:: read(size=-1)
- Read and return up to *n* bytes. If the argument is omitted, ``None``, or
- negative, data is read and returned until EOF is reached. An empty
+ Read and return up to *size* bytes. If the argument is omitted, ``None``,
+ or negative, data is read and returned until EOF is reached. An empty
:class:`bytes` object is returned if the stream is already at EOF.
If the argument is positive, and the underlying raw stream is not
@@ -457,9 +457,9 @@ I/O Base Classes
A :exc:`BlockingIOError` is raised if the underlying raw stream is in
non blocking-mode, and has no data available at the moment.
- .. method:: read1(n=-1)
+ .. method:: read1(size=-1)
- Read and return up to *n* bytes, with at most one call to the underlying
+ Read and return up to *size* bytes, with at most one call to the underlying
raw stream's :meth:`~RawIOBase.read` method. This can be useful if you
are implementing your own buffering on top of a :class:`BufferedIOBase`
object.
@@ -522,6 +522,8 @@ Raw File I/O
:mod:`os.open` as *opener* results in functionality similar to passing
``None``).
+ The newly created file is :ref:`non-inheritable <fd_inheritance>`.
+
See the :func:`open` built-in function for examples on using the *opener*
parameter.
@@ -529,6 +531,9 @@ Raw File I/O
The *opener* parameter was added.
The ``'x'`` mode was added.
+ .. versionchanged:: 3.4
+ The file is now non-inheritable.
+
In addition to the attributes and methods from :class:`IOBase` and
:class:`RawIOBase`, :class:`FileIO` provides the following data
attributes:
@@ -601,21 +606,21 @@ than raw I/O does.
:class:`BufferedReader` provides or overrides these methods in addition to
those from :class:`BufferedIOBase` and :class:`IOBase`:
- .. method:: peek([n])
+ .. method:: peek([size])
Return bytes from the stream without advancing the position. At most one
single read on the raw stream is done to satisfy the call. The number of
bytes returned may be less or more than requested.
- .. method:: read([n])
+ .. method:: read([size])
- Read and return *n* bytes, or if *n* is not given or negative, until EOF
- or if the read call would block in non-blocking mode.
+ Read and return *size* bytes, or if *size* is not given or negative, until
+ EOF or if the read call would block in non-blocking mode.
- .. method:: read1(n)
+ .. method:: read1(size)
- Read and return up to *n* bytes with only one call on the raw stream. If
- at least one byte is buffered, only buffered bytes are returned.
+ Read and return up to *size* bytes with only one call on the raw stream.
+ If at least one byte is buffered, only buffered bytes are returned.
Otherwise, one raw stream read call is made.
@@ -735,17 +740,17 @@ Text I/O
.. versionadded:: 3.1
- .. method:: read(n)
+ .. method:: read(size)
- Read and return at most *n* characters from the stream as a single
- :class:`str`. If *n* is negative or ``None``, reads until EOF.
+ Read and return at most *size* characters from the stream as a single
+ :class:`str`. If *size* is negative or ``None``, reads until EOF.
- .. method:: readline(limit=-1)
+ .. method:: readline(size=-1)
Read until newline or EOF and return a single ``str``. If the stream is
already at EOF, an empty string is returned.
- If *limit* is specified, at most *limit* characters will be read.
+ If *size* is specified, at most *size* characters will be read.
.. method:: seek(offset, whence=SEEK_SET)
diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst
index 8eac92f..9625e71 100644
--- a/Doc/library/ipaddress.rst
+++ b/Doc/library/ipaddress.rst
@@ -9,13 +9,6 @@
--------------
-.. note::
-
- The ``ipaddress`` module has been included in the standard library on a
- :term:`provisional basis <provisional package>`. Backwards incompatible
- changes (up to and including removal of the package) may occur if deemed
- necessary by the core developers.
-
:mod:`ipaddress` provides the capabilities to create, manipulate and
operate on IPv4 and IPv6 addresses and networks.
@@ -161,7 +154,16 @@ write code that handles both IP versions correctly.
.. attribute:: is_private
``True`` if the address is allocated for private networks. See
- :RFC:`1918` (for IPv4) or :RFC:`4193` (for IPv6).
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6).
+
+ .. attribute:: is_global
+
+ ``True`` if the address is allocated for public networks. See
+ iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_
+ (for IPv6).
+
+ .. versionadded:: 3.4
.. attribute:: is_unspecified
@@ -182,6 +184,9 @@ write code that handles both IP versions correctly.
``True`` if the address is reserved for link-local usage. See
:RFC:`3927`.
+.. _iana-ipv4-special-registry: http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+.. _iana-ipv6-special-registry: http://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
+
.. class:: IPv6Address(address)
@@ -216,18 +221,23 @@ write code that handles both IP versions correctly.
The long form of the address representation, with all leading zeroes and
groups consisting entirely of zeroes included.
+
+ For the following attributes, see the corresponding documention of the
+ :class:`IPv4Address` class:
+
.. attribute:: packed
.. attribute:: version
.. attribute:: max_prefixlen
.. attribute:: is_multicast
.. attribute:: is_private
+ .. attribute:: is_global
.. attribute:: is_unspecified
.. attribute:: is_reserved
.. attribute:: is_loopback
.. attribute:: is_link_local
- Refer to the corresponding attribute documentation in
- :class:`IPv4Address`
+ .. versionadded:: 3.4
+ is_global
.. attribute:: is_site_local
diff --git a/Doc/library/ipc.rst b/Doc/library/ipc.rst
index 91ec693..6b17563 100644
--- a/Doc/library/ipc.rst
+++ b/Doc/library/ipc.rst
@@ -9,7 +9,7 @@ to communicate.
Some modules only work for two processes that are on the same machine, e.g.
:mod:`signal` and :mod:`mmap`. Other modules support networking protocols
-that two or more processes can used to communicate across machines.
+that two or more processes can use to communicate across machines.
The list of modules described in this chapter is:
@@ -18,6 +18,9 @@ The list of modules described in this chapter is:
socket.rst
ssl.rst
+ select.rst
+ selectors.rst
+ asyncio.rst
asyncore.rst
asynchat.rst
signal.rst
diff --git a/Doc/library/json.rst b/Doc/library/json.rst
index f652039..5d97ee8 100644
--- a/Doc/library/json.rst
+++ b/Doc/library/json.rst
@@ -42,8 +42,7 @@ Compact encoding::
Pretty printing::
>>> import json
- >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True,
- ... indent=4, separators=(',', ': ')))
+ >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4))
{
"4": 5,
"6": 7
@@ -158,15 +157,13 @@ Basic Usage
.. versionchanged:: 3.2
Allow strings for *indent* in addition to integers.
- .. note::
-
- Since the default item separator is ``', '``, the output might include
- trailing whitespace when *indent* is specified. You can use
- ``separators=(',', ': ')`` to avoid this.
+ If specified, *separators* should be an ``(item_separator, key_separator)``
+ tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
+ ``(',', ': ')`` otherwise. To get the most compact JSON representation,
+ you should specify ``(',', ':')`` to eliminate whitespace.
- If *separators* is an ``(item_separator, dict_separator)`` tuple, then it
- will be used instead of the default ``(', ', ': ')`` separators. ``(',',
- ':')`` is the most compact JSON representation.
+ .. versionchanged:: 3.4
+ Use ``(',', ': ')`` as default if *indent* is not ``None``.
*default(obj)* is a function that should return a serializable version of
*obj* or raise :exc:`TypeError`. The default simply raises :exc:`TypeError`.
@@ -248,6 +245,8 @@ Basic Usage
kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments
will be passed to the constructor of the class.
+ If the data being deserialized is not a valid JSON document, a
+ :exc:`ValueError` will be raised.
.. function:: loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
@@ -257,6 +256,8 @@ Basic Usage
The other arguments have the same meaning as in :func:`load`, except
*encoding* which is ignored and deprecated.
+ If the data being deserialized is not a valid JSON document, a
+ :exc:`ValueError` will be raised.
Encoders and Decoders
---------------------
@@ -354,23 +355,26 @@ Encoders and Decoders
.. _py-to-json-table:
- +-------------------+---------------+
- | Python | JSON |
- +===================+===============+
- | dict | object |
- +-------------------+---------------+
- | list, tuple | array |
- +-------------------+---------------+
- | str | string |
- +-------------------+---------------+
- | int, float | number |
- +-------------------+---------------+
- | True | true |
- +-------------------+---------------+
- | False | false |
- +-------------------+---------------+
- | None | null |
- +-------------------+---------------+
+ +----------------------------------------+---------------+
+ | Python | JSON |
+ +========================================+===============+
+ | dict | object |
+ +----------------------------------------+---------------+
+ | list, tuple | array |
+ +----------------------------------------+---------------+
+ | str | string |
+ +----------------------------------------+---------------+
+ | int, float, int- & float-derived Enums | number |
+ +----------------------------------------+---------------+
+ | True | true |
+ +----------------------------------------+---------------+
+ | False | false |
+ +----------------------------------------+---------------+
+ | None | null |
+ +----------------------------------------+---------------+
+
+ .. versionchanged:: 3.4
+ Added support for int- and float-derived Enum classes.
To extend this to recognize other objects, subclass and implement a
:meth:`default` method with another method that returns a serializable object
@@ -410,15 +414,13 @@ Encoders and Decoders
.. versionchanged:: 3.2
Allow strings for *indent* in addition to integers.
- .. note::
-
- Since the default item separator is ``', '``, the output might include
- trailing whitespace when *indent* is specified. You can use
- ``separators=(',', ': ')`` to avoid this.
-
If specified, *separators* should be an ``(item_separator, key_separator)``
- tuple. The default is ``(', ', ': ')``. To get the most compact JSON
- representation, you should specify ``(',', ':')`` to eliminate whitespace.
+ tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
+ ``(',', ': ')`` otherwise. To get the most compact JSON representation,
+ you should specify ``(',', ':')`` to eliminate whitespace.
+
+ .. versionchanged:: 3.4
+ Use ``(',', ': ')`` as default if *indent* is not ``None``.
If specified, *default* is a function that gets called for objects that can't
otherwise be serialized. It should return a JSON encodable version of the
diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst
index 303b4d8..171b0a3 100644
--- a/Doc/library/logging.config.rst
+++ b/Doc/library/logging.config.rst
@@ -80,12 +80,25 @@ in :mod:`logging` itself) and defining handlers which are declared either in
.. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True)
- Reads the logging configuration from a :mod:`configparser`\-format file
- named *fname*. The format of the file should be as described in
- :ref:`logging-config-fileformat`. This function can be called several times
- from an application, allowing an end user to select from various pre-canned
- configurations (if the developer provides a mechanism to present the choices
- and load the chosen configuration).
+ Reads the logging configuration from a :mod:`configparser`\-format file. The
+ format of the file should be as described in
+ :ref:`logging-config-fileformat`.
+ This function can be called several times from an application, allowing an
+ end user to select from various pre-canned configurations (if the developer
+ provides a mechanism to present the choices and load the chosen
+ configuration).
+
+ :param fname: A filename, or a file-like object, or an instance derived
+ from :class:`~configparser.RawConfigParser`. If a
+ ``RawConfigParser``-derived instance is passed, it is used as
+ is. Otherwise, a :class:`~configparser.Configparser` is
+ instantiated, and the configuration read by it from the
+ object passed in ``fname``. If that has a :meth:`readline`
+ method, it is assumed to be a file-like object and read using
+ :meth:`~configparser.ConfigParser.read_file`; otherwise,
+ it is assumed to be a filename and passed to
+ :meth:`~configparser.ConfigParser.read`.
+
:param defaults: Defaults to be passed to the ConfigParser can be specified
in this argument.
@@ -99,8 +112,17 @@ in :mod:`logging` itself) and defining handlers which are declared either in
their ancestors are explicitly named in the
logging configuration.
+ .. versionchanged:: 3.4
+ An instance of a subclass of :class:`~configparser.RawConfigParser` is
+ now accepted as a value for ``fname``. This facilitates:
+
+ * Use of a configuration file where logging configuration is just part
+ of the overall application configuration.
+ * Use of a configuration read from a file, and then modified by the using
+ application (e.g. based on command-line parameters or other aspects
+ of the runtime environment) before being passed to ``fileConfig``.
-.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT)
+.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None)
Starts up a socket server on the specified port, and listens for new
configurations. If no port is specified, the module's default
@@ -111,6 +133,17 @@ in :mod:`logging` itself) and defining handlers which are declared either in
:meth:`~threading.Thread.join` when appropriate. To stop the server,
call :func:`stopListening`.
+ The ``verify`` argument, if specified, should be a callable which should
+ verify whether bytes received across the socket are valid and should be
+ processed. This could be done by encrypting and/or signing what is sent
+ across the socket, such that the ``verify`` callable can perform
+ signature verification and/or decryption. The ``verify`` callable is called
+ with a single argument - the bytes received across the socket - and should
+ return the bytes to be processed, or None to indicate that the bytes should
+ be discarded. The returned bytes could be the same as the passed in bytes
+ (e.g. when only verification is done), or they could be completely different
+ (perhaps if decryption were performed).
+
To send a configuration to the socket, read in the configuration file and
send it to the socket as a string of bytes preceded by a four-byte length
string packed in binary using ``struct.pack('>L', n)``.
@@ -129,7 +162,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in
:func:`listen` socket and sending a configuration which runs whatever
code the attacker wants to have executed in the victim's process. This is
especially easy to do if the default port is used, but not hard even if a
- different port is used).
+ different port is used). To avoid the risk of this happening, use the
+ ``verify`` argument to :func:`listen` to prevent unrecognised
+ configurations from being applied.
+
+ .. versionchanged:: 3.4.
+ The ``verify`` argument was added.
.. function:: stopListening()
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
index 415e397..315c168 100644
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -300,7 +300,7 @@ The :class:`TimedRotatingFileHandler` class, located in the
timed intervals.
-.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False)
+.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None)
Returns a new instance of the :class:`TimedRotatingFileHandler` class. The
specified file is opened and used as the stream for logging. On rotating it also
@@ -350,6 +350,12 @@ timed intervals.
If *delay* is true, then file opening is deferred until the first call to
:meth:`emit`.
+ If *atTime* is not ``None``, it must be a ``datetime.time`` instance which
+ specifies the time of day when rollover occurs, for the cases where rollover
+ is set to happen "at midnight" or "on a particular weekday".
+
+ .. versionchanged:: 3.4
+ *atTime* parameter was added.
.. method:: doRollover()
@@ -375,6 +381,9 @@ sends logging output to a network socket. The base class uses a TCP socket.
Returns a new instance of the :class:`SocketHandler` class intended to
communicate with a remote machine whose address is given by *host* and *port*.
+ .. versionchanged:: 3.4
+ If ``port`` is specified as ``None``, a Unix domain socket is created
+ using the value in ``host`` - otherwise, a TCP socket is created.
.. method:: close()
@@ -460,6 +469,9 @@ over UDP sockets.
Returns a new instance of the :class:`DatagramHandler` class intended to
communicate with a remote machine whose address is given by *host* and *port*.
+ .. versionchanged:: 3.4
+ If ``port`` is specified as ``None``, a Unix domain socket is created
+ using the value in ``host`` - otherwise, a TCP socket is created.
.. method:: emit()
diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst
index 07b69eb..b71051d 100644
--- a/Doc/library/lzma.rst
+++ b/Doc/library/lzma.rst
@@ -39,8 +39,8 @@ Reading and writing compressed files
opened, or it can be an existing file object to read from or write to.
The *mode* argument can be any of ``"r"``, ``"rb"``, ``"w"``, ``"wb"``,
- ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, ``"wt"``, or ``"at"`` for
- text mode. The default is ``"rb"``.
+ ``"x"``, ``"xb"``, ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``,
+ ``"wt"``, ``"xt"``, or ``"at"`` for text mode. The default is ``"rb"``.
When opening a file for reading, the *format* and *filters* arguments have
the same meanings as for :class:`LZMADecompressor`. In this case, the *check*
@@ -57,6 +57,9 @@ Reading and writing compressed files
:class:`io.TextIOWrapper` instance with the specified encoding, error
handling behavior, and line ending(s).
+ .. versionchanged:: 3.4
+ Added support for the ``"x"``, ``"xb"`` and ``"xt"`` modes.
+
.. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None)
@@ -69,8 +72,9 @@ Reading and writing compressed files
file will not be closed when the :class:`LZMAFile` is closed.
The *mode* argument can be either ``"r"`` for reading (default), ``"w"`` for
- overwriting, or ``"a"`` for appending. These can equivalently be given as
- ``"rb"``, ``"wb"``, and ``"ab"`` respectively.
+ overwriting, ``"x"`` for exclusive creation, or ``"a"`` for appending. These
+ can equivalently be given as ``"rb"``, ``"wb"``, ``"xb"`` and ``"ab"``
+ respectively.
If *filename* is a file object (rather than an actual file name), a mode of
``"w"`` does not truncate the file, and is instead equivalent to ``"a"``.
@@ -103,6 +107,9 @@ Reading and writing compressed files
file object (e.g. if the :class:`LZMAFile` was constructed by passing a
file object for *filename*).
+ .. versionchanged:: 3.4
+ Added support for the ``"x"`` and ``"xb"`` modes.
+
Compressing and decompressing data in memory
--------------------------------------------
diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst
index 3b9e3d2..124eb61 100644
--- a/Doc/library/marshal.rst
+++ b/Doc/library/marshal.rst
@@ -40,10 +40,11 @@ this module. The following types are supported: booleans, integers, floating
point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets,
frozensets, dictionaries, and code objects, where it should be understood that
tuples, lists, sets, frozensets and dictionaries are only supported as long as
-the values contained therein are themselves supported; and recursive lists, sets
-and dictionaries should not be written (they will cause infinite loops). The
+the values contained therein are themselves supported.
singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be
marshalled and unmarshalled.
+For format *version* lower than 3, recursive lists, sets and dictionaries cannot
+be written (see below).
There are functions that read/write files as well as functions operating on
strings.
@@ -103,7 +104,9 @@ In addition, the following constants are defined:
Indicates the format that the module uses. Version 0 is the historical
format, version 1 shares interned strings and version 2 uses a binary format
- for floating point numbers. The current version is 2.
+ for floating point numbers.
+ Version 3 adds support for object instancing and recursion.
+ The current version is 3.
.. rubric:: Footnotes
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index 5f50cf1..5fac730 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -93,11 +93,112 @@ To show the individual process IDs involved, here is an expanded example::
p.start()
p.join()
-For an explanation of why (on Windows) the ``if __name__ == '__main__'`` part is
+For an explanation of why the ``if __name__ == '__main__'`` part is
necessary, see :ref:`multiprocessing-programming`.
+Contexts and start methods
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _multiprocessing-start-methods:
+
+Depending on the platform, :mod:`multiprocessing` supports three ways
+to start a process. These *start methods* are
+
+ *spawn*
+ The parent process starts a fresh python interpreter process. The
+ child process will only inherit those resources necessary to run
+ the process objects :meth:`~Process.run` method. In particular,
+ unnecessary file descriptors and handles from the parent process
+ will not be inherited. Starting a process using this method is
+ rather slow compared to using *fork* or *forkserver*.
+
+ Available on Unix and Windows. The default on Windows.
+
+ *fork*
+ The parent process uses :func:`os.fork` to fork the Python
+ interpreter. The child process, when it begins, is effectively
+ identical to the parent process. All resources of the parent are
+ inherited by the child process. Note that safely forking a
+ multithreaded process is problematic.
+
+ Available on Unix only. The default on Unix.
+
+ *forkserver*
+ When the program starts and selects the *forkserver* start method,
+ a server process is started. From then on, whenever a new process
+ is needed, the parent process connects to the server and requests
+ that it fork a new process. The fork server process is single
+ threaded so it is safe for it to use :func:`os.fork`. No
+ unnecessary resources are inherited.
+
+ Available on Unix platforms which support passing file descriptors
+ over Unix pipes.
+
+.. versionchanged:: 3.4
+ *spawn* added on all unix platforms, and *forkserver* added for
+ some unix platforms.
+ Child processes no longer inherit all of the parents inheritable
+ handles on Windows.
+
+On Unix using the *spawn* or *forkserver* start methods will also
+start a *semaphore tracker* process which tracks the unlinked named
+semaphores created by processes of the program. When all processes
+have exited the semaphore tracker unlinks any remaining semaphores.
+Usually there should be none, but if a process was killed by a signal
+there may some "leaked" semaphores. (Unlinking the named semaphores
+is a serious matter since the system allows only a limited number, and
+they will not be automatically unlinked until the next reboot.)
+
+To select a start method you use the :func:`set_start_method` in
+the ``if __name__ == '__main__'`` clause of the main module. For
+example::
+
+ import multiprocessing as mp
+
+ def foo(q):
+ q.put('hello')
+
+ if __name__ == '__main__':
+ mp.set_start_method('spawn')
+ q = mp.Queue()
+ p = mp.Process(target=foo, args=(q,))
+ p.start()
+ print(q.get())
+ p.join()
+
+:func:`set_start_method` should not be used more than once in the
+program.
+
+Alternatively, you can use :func:`get_context` to obtain a context
+object. Context objects have the same API as the multiprocessing
+module, and allow one to use multiple start methods in the same
+program. ::
+
+ import multiprocessing as mp
+
+ def foo(q):
+ q.put('hello')
+
+ if __name__ == '__main__':
+ ctx = mp.get_context('spawn')
+ q = ctx.Queue()
+ p = ctx.Process(target=foo, args=(q,))
+ p.start()
+ print(q.get())
+ p.join()
+
+Note that objects related to one context may not be compatible with
+processes for a different context. In particular, locks created using
+the *fork* context cannot be passed to a processes started using the
+*spawn* or *forkserver* start methods.
+
+A library which wants to use a particular start method should probably
+use :func:`get_context` to avoid interfering with the choice of the
+library user.
+
+
Exchanging objects between processes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -274,15 +375,31 @@ processes in a few different ways.
For example::
from multiprocessing import Pool
+ from time import sleep
def f(x):
return x*x
if __name__ == '__main__':
- with Pool(processes=4) as pool: # start 4 worker processes
- result = pool.apply_async(f, [10]) # evaluate "f(10)" asynchronously
- print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow
- print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]"
+ # start 4 worker processes
+ with Pool(processes=4) as pool:
+
+ # print "[0, 1, 4,..., 81]"
+ print(pool.map(f, range(10)))
+
+ # print same numbers in arbitrary order
+ for i in pool.imap_unordered(f, range(10)):
+ print(i)
+
+ # evaluate "f(10)" asynchronously
+ res = pool.apply_async(f, [10])
+ print(res.get(timeout=1)) # prints "100"
+
+ # make worker sleep for 10 secs
+ res = pool.apply_async(sleep, 10)
+ print(res.get(timeout=1)) # raises multiprocessing.TimeoutError
+
+ # exiting the 'with'-block has stopped the pool
Note that the methods of a pool should only ever be used by the
process which created it.
@@ -731,6 +848,9 @@ Miscellaneous
Return the number of CPUs in the system. May raise
:exc:`NotImplementedError`.
+ .. seealso::
+ :func:`os.cpu_count`
+
.. function:: current_process()
Return the :class:`Process` object corresponding to the current process.
@@ -761,6 +881,43 @@ Miscellaneous
If the module is being run normally by the Python interpreter then
:func:`freeze_support` has no effect.
+.. function:: get_all_start_methods()
+
+ Returns a list of the supported start methods, the first of which
+ is the default. The possible start methods are ``'fork'``,
+ ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is
+ available. On Unix ``'fork'`` and ``'spawn'`` are always
+ supported, with ``'fork'`` being the default.
+
+ .. versionadded:: 3.4
+
+.. function:: get_context(method=None)
+
+ Return a context object which has the same attributes as the
+ :mod:`multiprocessing` module.
+
+ If *method* is *None* then the default context is returned.
+ Otherwise *method* should be ``'fork'``, ``'spawn'``,
+ ``'forkserver'``. :exc:`ValueError` is raised if the specified
+ start method is not available.
+
+ .. versionadded:: 3.4
+
+.. function:: get_start_method(allow_none=False)
+
+ Return the name of start method used for starting processes.
+
+ If the start method has not been fixed and *allow_none* is false,
+ then the start method is fixed to the default and the name is
+ returned. If the start method has not been fixed and *allow_none*
+ is true then *None* is returned.
+
+ The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'``
+ or *None*. ``'fork'`` is the default on Unix, while ``'spawn'`` is
+ the default on Windows.
+
+ .. versionadded:: 3.4
+
.. function:: set_executable()
Sets the path of the Python interpreter to use when starting a child process.
@@ -769,8 +926,21 @@ Miscellaneous
set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe'))
- before they can create child processes. (Windows only)
+ before they can create child processes.
+
+ .. versionchanged:: 3.4
+ Now supported on Unix when the ``'spawn'`` start method is used.
+.. function:: set_start_method(method)
+
+ Set the method which should be used to start child processes.
+ *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``.
+
+ Note that this should be called at most once, and it should be
+ protected inside the ``if __name__ == '__main__'`` clause of the
+ main module.
+
+ .. versionadded:: 3.4
.. note::
@@ -1678,25 +1848,37 @@ Process Pools
One can create a pool of processes which will carry out tasks submitted to it
with the :class:`Pool` class.
-.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild]]]])
+.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild [, context]]]]])
A process pool object which controls a pool of worker processes to which jobs
can be submitted. It supports asynchronous results with timeouts and
callbacks and has a parallel map implementation.
*processes* is the number of worker processes to use. If *processes* is
- ``None`` then the number returned by :func:`cpu_count` is used. If
- *initializer* is not ``None`` then each worker process will call
+ ``None`` then the number returned by :func:`os.cpu_count` is used.
+
+ If *initializer* is not ``None`` then each worker process will call
``initializer(*initargs)`` when it starts.
+ *maxtasksperchild* is the number of tasks a worker process can complete
+ before it will exit and be replaced with a fresh worker process, to enable
+ unused resources to be freed. The default *maxtasksperchild* is None, which
+ means worker processes will live as long as the pool.
+
+ *context* can be used to specify the context used for starting
+ the worker processes. Usually a pool is created using the
+ function :func:`multiprocessing.Pool` or the :meth:`Pool` method
+ of a context object. In both cases *context* is set
+ appropriately.
+
Note that the methods of the pool object should only be called by
the process which created the pool.
.. versionadded:: 3.2
- *maxtasksperchild* is the number of tasks a worker process can complete
- before it will exit and be replaced with a fresh worker process, to enable
- unused resources to be freed. The default *maxtasksperchild* is None, which
- means worker processes will live as long as the pool.
+ *maxtasksperchild*
+
+ .. versionadded:: 3.4
+ *context*
.. note::
@@ -2189,43 +2371,8 @@ Below is an example session with logging turned on::
[INFO/MainProcess] sending shutdown message to manager
[INFO/SyncManager-...] manager exiting with exitcode 0
-In addition to having these two logging functions, the multiprocessing also
-exposes two additional logging level attributes. These are :const:`SUBWARNING`
-and :const:`SUBDEBUG`. The table below illustrates where theses fit in the
-normal level hierarchy.
-
-+----------------+----------------+
-| Level | Numeric value |
-+================+================+
-| ``SUBWARNING`` | 25 |
-+----------------+----------------+
-| ``SUBDEBUG`` | 5 |
-+----------------+----------------+
-
For a full table of logging levels, see the :mod:`logging` module.
-These additional logging levels are used primarily for certain debug messages
-within the multiprocessing module. Below is the same example as above, except
-with :const:`SUBDEBUG` enabled::
-
- >>> import multiprocessing, logging
- >>> logger = multiprocessing.log_to_stderr()
- >>> logger.setLevel(multiprocessing.SUBDEBUG)
- >>> logger.warning('doomed')
- [WARNING/MainProcess] doomed
- >>> m = multiprocessing.Manager()
- [INFO/SyncManager-...] child process calling self.run()
- [INFO/SyncManager-...] created temp directory /.../pymp-...
- [INFO/SyncManager-...] manager serving at '/.../pymp-djGBXN/listener-...'
- >>> del m
- [SUBDEBUG/MainProcess] finalizer calling ...
- [INFO/MainProcess] sending shutdown message to manager
- [DEBUG/SyncManager-...] manager received shutdown message
- [SUBDEBUG/SyncManager-...] calling <Finalize object, callback=unlink, ...
- [SUBDEBUG/SyncManager-...] finalizer calling <built-in function unlink> ...
- [SUBDEBUG/SyncManager-...] calling <Finalize object, dead>
- [SUBDEBUG/SyncManager-...] finalizer calling <function rmtree at 0x5aa730> ...
- [INFO/SyncManager-...] manager exiting with exitcode 0
The :mod:`multiprocessing.dummy` module
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2246,8 +2393,10 @@ There are certain guidelines and idioms which should be adhered to when using
:mod:`multiprocessing`.
-All platforms
-~~~~~~~~~~~~~
+All start methods
+~~~~~~~~~~~~~~~~~
+
+The following applies to all start methods.
Avoid shared state
@@ -2281,11 +2430,13 @@ Joining zombie processes
Better to inherit than pickle/unpickle
- On Windows many types from :mod:`multiprocessing` need to be picklable so
- that child processes can use them. However, one should generally avoid
- sending shared objects to other processes using pipes or queues. Instead
- you should arrange the program so that a process which needs access to a
- shared resource created elsewhere can inherit it from an ancestor process.
+ When using the *spawn* or *forkserver* start methods many types
+ from :mod:`multiprocessing` need to be picklable so that child
+ processes can use them. However, one should generally avoid
+ sending shared objects to other processes using pipes or queues.
+ Instead you should arrange the program so that a process which
+ needs access to a shared resource created elsewhere can inherit it
+ from an ancestor process.
Avoid terminating processes
@@ -2332,15 +2483,17 @@ Joining processes that use queues
Explicitly pass resources to child processes
- On Unix a child process can make use of a shared resource created in a
- parent process using a global resource. However, it is better to pass the
- object as an argument to the constructor for the child process.
+ On Unix using the *fork* start method, a child process can make
+ use of a shared resource created in a parent process using a
+ global resource. However, it is better to pass the object as an
+ argument to the constructor for the child process.
- Apart from making the code (potentially) compatible with Windows this also
- ensures that as long as the child process is still alive the object will not
- be garbage collected in the parent process. This might be important if some
- resource is freed when the object is garbage collected in the parent
- process.
+ Apart from making the code (potentially) compatible with Windows
+ and the other start methods this also ensures that as long as the
+ child process is still alive the object will not be garbage
+ collected in the parent process. This might be important if some
+ resource is freed when the object is garbage collected in the
+ parent process.
So for instance ::
@@ -2399,17 +2552,19 @@ Beware of replacing :data:`sys.stdin` with a "file like object"
For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331`
-Windows
-~~~~~~~
+The *spawn* and *forkserver* start methods
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since Windows lacks :func:`os.fork` it has a few extra restrictions:
+There are a few extra restriction which don't apply to the *fork*
+start method.
More picklability
- Ensure that all arguments to :meth:`Process.__init__` are picklable. This
- means, in particular, that bound or unbound methods cannot be used directly
- as the ``target`` argument on Windows --- just define a function and use
- that instead.
+ Ensure that all arguments to :meth:`Process.__init__` are
+ picklable. This means, in particular, that bound or unbound
+ methods cannot be used directly as the ``target`` (unless you use
+ the *fork* start method) --- just define a function and use that
+ instead.
Also, if you subclass :class:`~multiprocessing.Process` then make sure that
instances will be picklable when the :meth:`Process.start
@@ -2431,7 +2586,8 @@ Safe importing of main module
interpreter without causing unintended side effects (such a starting a new
process).
- For example, under Windows running the following module would fail with a
+ For example, using the *spawn* or *forkserver* start method
+ running the following module would fail with a
:exc:`RuntimeError`::
from multiprocessing import Process
@@ -2445,13 +2601,14 @@ Safe importing of main module
Instead one should protect the "entry point" of the program by using ``if
__name__ == '__main__':`` as follows::
- from multiprocessing import Process, freeze_support
+ from multiprocessing import Process, freeze_support, set_start_method
def foo():
print('hello')
if __name__ == '__main__':
freeze_support()
+ set_start_method('spawn')
p = Process(target=foo)
p.start()
@@ -2482,26 +2639,7 @@ Using :class:`~multiprocessing.pool.Pool`:
:language: python3
-Synchronization types like locks, conditions and queues:
-
-.. literalinclude:: ../includes/mp_synchronize.py
- :language: python3
-
-
An example showing how to use queues to feed tasks to a collection of worker
processes and collect the results:
.. literalinclude:: ../includes/mp_workers.py
-
-
-An example of how a pool of worker processes can each run a
-:class:`~http.server.SimpleHTTPRequestHandler` instance while sharing a single
-listening socket.
-
-.. literalinclude:: ../includes/mp_webserver.py
-
-
-Some simple benchmarks comparing :mod:`multiprocessing` with :mod:`threading`:
-
-.. literalinclude:: ../includes/mp_benchmarks.py
-
diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst
index 564f101..23ffed6 100644
--- a/Doc/library/netrc.rst
+++ b/Doc/library/netrc.rst
@@ -29,7 +29,7 @@ the Unix :program:`ftp` program and other FTP clients.
This implements security behavior equivalent to that of ftp and other
programs that use :file:`.netrc`.
- .. versionchanged:: 3.3.3 Added the POSIX permission check.
+ .. versionchanged:: 3.4 Added the POSIX permission check.
.. exception:: NetrcParseError
diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst
index a15d7f7..3943f2c 100644
--- a/Doc/library/nntplib.rst
+++ b/Doc/library/nntplib.rst
@@ -1,4 +1,3 @@
-
:mod:`nntplib` --- NNTP protocol client
=======================================
@@ -71,7 +70,7 @@ The module itself defines the following classes:
reader-specific commands, such as ``group``. If you get unexpected
:exc:`NNTPPermanentError`\ s, you might need to set *readermode*.
:class:`NNTP` class supports the :keyword:`with` statement to
- unconditionally consume :exc:`socket.error` exceptions and to close the NNTP
+ unconditionally consume :exc:`OSError` exceptions and to close the NNTP
connection when done. Here is a sample on how using it:
>>> from nntplib import NNTP
@@ -95,6 +94,7 @@ The module itself defines the following classes:
port *port*. :class:`NNTP_SSL` objects have the same methods as
:class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used.
*ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object.
+ Please read :ref:`ssl-security` for best practices.
All other parameters behave the same as for :class:`NNTP`.
Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of
@@ -103,6 +103,10 @@ The module itself defines the following classes:
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ The class now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
.. exception:: NNTPError
@@ -231,9 +235,10 @@ tuples or objects that the method normally returns will be empty.
.. method:: NNTP.starttls(ssl_context=None)
- Send a ``STARTTLS`` command. The *ssl_context* argument is optional
- and should be a :class:`ssl.SSLContext` object. This will enable
- encryption on the NNTP connection.
+ Send a ``STARTTLS`` command. This will enable encryption on the NNTP
+ connection. The *ssl_context* argument is optional and should be a
+ :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best
+ practices.
Note that this may not be done after authentication information has
been transmitted, and authentication occurs by default if possible during a
@@ -242,6 +247,10 @@ tuples or objects that the method normally returns will be empty.
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ The method now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
.. method:: NNTP.newgroups(date, *, file=None)
diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst
index 2732a84..7c76a47 100644
--- a/Doc/library/numeric.rst
+++ b/Doc/library/numeric.rst
@@ -23,3 +23,4 @@ The following modules are documented in this chapter:
decimal.rst
fractions.rst
random.rst
+ statistics.rst
diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst
index 24becf9..3bcbaa4 100644
--- a/Doc/library/operator.rst
+++ b/Doc/library/operator.rst
@@ -11,6 +11,9 @@
import operator
from operator import itemgetter, iadd
+**Source code:** :source:`Lib/operator.py`
+
+--------------
The :mod:`operator` module exports a set of efficient functions corresponding to
the intrinsic operators of Python. For example, ``operator.add(x, y)`` is
@@ -235,6 +238,14 @@ their character equivalents.
.. XXX: find a better, readable, example
+.. function:: length_hint(obj, default=0)
+
+ Return an estimated length for the object *o*. First try to return its
+ actual length, then an estimate using :meth:`object.__length_hint__`, and
+ finally return the default value.
+
+ .. versionadded:: 3.4
+
The :mod:`operator` module also defines tools for generalized attribute and item
lookups. These are useful for making fast field extractors as arguments for
:func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that
diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst
index 13395b6..72145aa 100644
--- a/Doc/library/optparse.rst
+++ b/Doc/library/optparse.rst
@@ -8,8 +8,8 @@
.. sectionauthor:: Greg Ward <gward@python.net>
.. deprecated:: 3.2
- The :mod:`optparse` module is deprecated and will not be developed further;
- development will continue with the :mod:`argparse` module.
+ The :mod:`optparse` module is deprecated and will not be developed further;
+ development will continue with the :mod:`argparse` module.
**Source code:** :source:`Lib/optparse.py`
diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index 57a6bca..6b3a3b6 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -22,6 +22,11 @@ Functions such as :func:`expanduser` and :func:`expandvars` can be invoked
explicitly when an application desires shell-like path expansion. (See also
the :mod:`glob` module.)
+
+.. seealso::
+ The :mod:`pathlib` module offers high-level path objects.
+
+
.. note::
All of these functions accept either only bytes or only string objects as
@@ -42,7 +47,6 @@ the :mod:`glob` module.)
* :mod:`posixpath` for UNIX-style paths
* :mod:`ntpath` for Windows paths
* :mod:`macpath` for old-style MacOS paths
- * :mod:`os2emxpath` for OS/2 EMX paths
.. function:: abspath(path)
@@ -189,11 +193,17 @@ the :mod:`glob` module.)
.. function:: ismount(path)
- Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a file
- system where a different file system has been mounted. The function checks
- whether *path*'s parent, :file:`path/..`, is on a different device than *path*,
- or whether :file:`path/..` and *path* point to the same i-node on the same
- device --- this should detect mount points for all Unix and POSIX variants.
+ Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a
+ file system where a different file system has been mounted. On POSIX, the
+ function checks whether *path*'s parent, :file:`path/..`, is on a different
+ device than *path*, or whether :file:`path/..` and *path* point to the same
+ i-node on the same device --- this should detect mount points for all Unix
+ and POSIX variants. On Windows, a drive letter root and a share UNC are
+ always mount points, and for any other path ``GetVolumePathName`` is called
+ to see if it is different from the input path.
+
+ .. versionadded:: 3.4
+ Support for detecting non-root mount points on Windows.
.. function:: join(path1[, path2[, ...]])
@@ -232,7 +242,7 @@ the :mod:`glob` module.)
links encountered in the path (if they are supported by the operating system).
-.. function:: relpath(path, start=None)
+.. function:: relpath(path, start=os.curdir)
Return a relative filepath to *path* either from the current directory or
from an optional *start* directory. This is a path computation: the
@@ -247,18 +257,17 @@ the :mod:`glob` module.)
.. function:: samefile(path1, path2)
Return ``True`` if both pathname arguments refer to the same file or directory.
- On Unix, this is determined by the device number and i-node number and raises an
+ This is determined by the device number and i-node number and raises an
exception if a :func:`os.stat` call on either pathname fails.
- On Windows, two files are the same if they resolve to the same final path
- name using the Windows API call GetFinalPathNameByHandle. This function
- raises an exception if handles cannot be obtained to either file.
-
Availability: Unix, Windows.
.. versionchanged:: 3.2
Added Windows support.
+ .. versionchanged:: 3.4
+ Windows now uses the same implementation as all other platforms.
+
.. function:: sameopenfile(fp1, fp2)
@@ -277,7 +286,10 @@ the :mod:`glob` module.)
:func:`os.lstat`, or :func:`os.stat`. This function implements the
underlying comparison used by :func:`samefile` and :func:`sameopenfile`.
- Availability: Unix.
+ Availability: Unix, Windows.
+
+ .. versionchanged:: 3.4
+ Added Windows support.
.. function:: split(path)
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 72693f1..8e19c25 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -54,7 +54,7 @@ Notes on the availability of these functions:
The name of the operating system dependent module imported. The following
names have currently been registered: ``'posix'``, ``'nt'``, ``'mac'``,
- ``'os2'``, ``'ce'``, ``'java'``.
+ ``'ce'``, ``'java'``.
.. seealso::
:attr:`sys.platform` has a finer granularity. :func:`os.uname` gives
@@ -687,17 +687,30 @@ as internal buffering of data.
.. function:: dup(fd)
- Return a duplicate of file descriptor *fd*.
+ Return a duplicate of file descriptor *fd*. The new file descriptor is
+ :ref:`non-inheritable <fd_inheritance>`.
+
+ On Windows, when duplicating a standard stream (0: stdin, 1: stdout,
+ 2: stderr), the new file descriptor is :ref:`inheritable
+ <fd_inheritance>`.
Availability: Unix, Windows.
+ .. versionchanged:: 3.4
+ The new file descriptor is now non-inheritable.
+
-.. function:: dup2(fd, fd2)
+.. function:: dup2(fd, fd2, inheritable=True)
Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary.
+ The file descriptor *fd2* is :ref:`inheritable <fd_inheritance>` by default,
+ or non-inheritable if *inheritable* is ``False``.
Availability: Unix, Windows.
+ .. versionchanged:: 3.4
+ Add the optional *inheritable* parameter.
+
.. function:: fchmod(fd, mode)
@@ -848,17 +861,21 @@ as internal buffering of data.
Open the file *file* and set various flags according to *flags* and possibly
its mode according to *mode*. When computing *mode*, the current umask value
is first masked out. Return the file descriptor for the newly opened file.
+ The new file descriptor is :ref:`non-inheritable <fd_inheritance>`.
For a description of the flag and mode values, see the C run-time documentation;
flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in
- this module too (see :ref:`open-constants`). In particular, on Windows adding
+ the :mod:`os` module. In particular, on Windows adding
:const:`O_BINARY` is needed to open files in binary mode.
This function can support :ref:`paths relative to directory descriptors
- <dir_fd>`.
+ <dir_fd>` with the *dir_fd* parameter.
Availability: Unix, Windows.
+ .. versionchanged:: 3.4
+ The new file descriptor is now non-inheritable.
+
.. note::
This function is intended for low-level I/O. For normal usage, use the
@@ -869,25 +886,93 @@ as internal buffering of data.
.. versionadded:: 3.3
The *dir_fd* argument.
+The following constants are options for the *flags* parameter to the
+:func:`~os.open` function. They can be combined using the bitwise OR operator
+``|``. Some of them are not available on all platforms. For descriptions of
+their availability and use, consult the :manpage:`open(2)` manual page on Unix
+or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windows.
+
+
+.. data:: O_RDONLY
+ O_WRONLY
+ O_RDWR
+ O_APPEND
+ O_CREAT
+ O_EXCL
+ O_TRUNC
+
+ These constants are available on Unix and Windows.
+
+
+.. data:: O_DSYNC
+ O_RSYNC
+ O_SYNC
+ O_NDELAY
+ O_NONBLOCK
+ O_NOCTTY
+ O_SHLOCK
+ O_EXLOCK
+ O_CLOEXEC
+
+ These constants are only available on Unix.
+
+ .. versionchanged:: 3.3
+ Add :data:`O_CLOEXEC` constant.
+
+.. data:: O_BINARY
+ O_NOINHERIT
+ O_SHORT_LIVED
+ O_TEMPORARY
+ O_RANDOM
+ O_SEQUENTIAL
+ O_TEXT
+
+ These constants are only available on Windows.
+
+
+.. data:: O_ASYNC
+ O_DIRECT
+ O_DIRECTORY
+ O_NOFOLLOW
+ O_NOATIME
+ O_PATH
+ O_TMPFILE
+
+ These constants are GNU extensions and not present if they are not defined by
+ the C library.
+
+ .. versionchanged:: 3.4
+ Add :data:`O_PATH` on systems that support it.
+ Add :data:`O_TMPFILE`, only available on Linux Kernel 3.11
+ or newer.
+
.. function:: openpty()
.. index:: module: pty
- Open a new pseudo-terminal pair. Return a pair of file descriptors ``(master,
- slave)`` for the pty and the tty, respectively. For a (slightly) more portable
- approach, use the :mod:`pty` module.
+ Open a new pseudo-terminal pair. Return a pair of file descriptors
+ ``(master, slave)`` for the pty and the tty, respectively. The new file
+ descriptors are :ref:`non-inheritable <fd_inheritance>`. For a (slightly) more
+ portable approach, use the :mod:`pty` module.
Availability: some flavors of Unix.
+ .. versionchanged:: 3.4
+ The new file descriptors are now non-inheritable.
+
.. function:: pipe()
- Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for reading
- and writing, respectively.
+ Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for
+ reading and writing, respectively. The new file descriptor is
+ :ref:`non-inheritable <fd_inheritance>`.
Availability: Unix, Windows.
+ .. versionchanged:: 3.4
+ The new file descriptors are now non-inheritable.
+
.. function:: pipe2(flags)
@@ -1027,7 +1112,7 @@ as internal buffering of data.
sequence to hold the rest of the data. :func:`~os.readv` returns the total
number of bytes read (which may be less than the total capacity of all the
objects).
-
+
Availability: Unix.
.. versionadded:: 3.3
@@ -1080,80 +1165,8 @@ as internal buffering of data.
sequence of :term:`bytes-like objects <bytes-like object>`.
:func:`~os.writev` writes the contents of each object to the file descriptor
and returns the total number of bytes written.
-
- Availability: Unix.
-
- .. versionadded:: 3.3
-
-
-.. _open-constants:
-
-``open()`` flag constants
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The following constants are options for the *flags* parameter to the
-:func:`~os.open` function. They can be combined using the bitwise OR operator
-``|``. Some of them are not available on all platforms. For descriptions of
-their availability and use, consult the :manpage:`open(2)` manual page on Unix
-or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Windows.
-
-
-.. data:: O_RDONLY
- O_WRONLY
- O_RDWR
- O_APPEND
- O_CREAT
- O_EXCL
- O_TRUNC
-
- These constants are available on Unix and Windows.
-
-
-.. data:: O_DSYNC
- O_RSYNC
- O_SYNC
- O_NDELAY
- O_NONBLOCK
- O_NOCTTY
- O_SHLOCK
- O_EXLOCK
- O_CLOEXEC
-
- These constants are only available on Unix.
-
- .. versionchanged:: 3.3
- Add :data:`O_CLOEXEC` constant.
-.. data:: O_BINARY
- O_NOINHERIT
- O_SHORT_LIVED
- O_TEMPORARY
- O_RANDOM
- O_SEQUENTIAL
- O_TEXT
-
- These constants are only available on Windows.
-
-
-.. data:: O_ASYNC
- O_DIRECT
- O_DIRECTORY
- O_NOFOLLOW
- O_NOATIME
-
- These constants are GNU extensions and not present if they are not defined by
- the C library.
-
-
-.. data:: RTLD_LAZY
- RTLD_NOW
- RTLD_GLOBAL
- RTLD_LOCAL
- RTLD_NODELETE
- RTLD_NOLOAD
- RTLD_DEEPBIND
-
- See the Unix manual page :manpage:`dlopen(3)`.
+ Availability: Unix.
.. versionadded:: 3.3
@@ -1195,6 +1208,49 @@ Querying the size of a terminal
Height of the terminal window in characters.
+.. _fd_inheritance:
+
+Inheritance of File Descriptors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 3.4
+
+A file descriptor has an "inheritable" flag which indicates if the file descriptor
+can be inherited by child processes. Since Python 3.4, file descriptors
+created by Python are non-inheritable by default.
+
+On UNIX, non-inheritable file descriptors are closed in child processes at the
+execution of a new program, other file descriptors are inherited.
+
+On Windows, non-inheritable handles and file descriptors are closed in child
+processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout
+and stderr), which are always inherited. Using :func:`spawn\* <spawnl>` functions,
+all inheritable handles and all inheritable file descriptors are inherited.
+Using the :mod:`subprocess` module, all file descriptors except standard
+streams are closed, and inheritable handles are only inherited if the
+*close_fds* parameter is ``False``.
+
+.. function:: get_inheritable(fd)
+
+ Get the "inheritable" flag of the specified file descriptor (a boolean).
+
+.. function:: set_inheritable(fd, inheritable)
+
+ Set the "inheritable" flag of the specified file descriptor.
+
+.. function:: get_handle_inheritable(handle)
+
+ Get the "inheritable" flag of the specified handle (a boolean).
+
+ Availability: Windows.
+
+.. function:: set_handle_inheritable(handle, inheritable)
+
+ Set the "inheritable" flag of the specified handle.
+
+ Availability: Windows.
+
+
.. _os-file-dir:
Files and Directories
@@ -1551,7 +1607,7 @@ features:
The *dir_fd* argument.
-.. function:: makedirs(path, mode=0o777, exist_ok=False)
+.. function:: makedirs(name, mode=0o777, exist_ok=False)
.. index::
single: directory; creating
@@ -1707,7 +1763,7 @@ features:
The *dir_fd* argument.
-.. function:: removedirs(path)
+.. function:: removedirs(name)
.. index:: single: directory; deleting
@@ -1931,11 +1987,26 @@ features:
read-only, and if :const:`ST_NOSUID` is set, the semantics of
setuid/setgid bits are disabled or not supported.
+ Additional module-level constants are defined for GNU/glibc based systems.
+ These are :const:`ST_NODEV` (disallow access to device special files),
+ :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS`
+ (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS),
+ :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND`
+ (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME`
+ (do not update access times), :const:`ST_NODIRATIME` (do not update directory access
+ times), :const:`ST_RELATIME` (update atime relative to mtime/ctime).
+
This function can support :ref:`specifying a file descriptor <path_fd>`.
.. versionchanged:: 3.2
The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added.
+ .. versionchanged:: 3.4
+ The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`,
+ :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`,
+ :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`,
+ and :const:`ST_RELATIME` constants were added.
+
Availability: Unix.
.. versionadded:: 3.3
@@ -2585,7 +2656,7 @@ written in Python, such as a mail server's external command delivery program.
Fork a child process. Return ``0`` in the child and the child's process id in the
parent. If an error occurs :exc:`OSError` is raised.
- Note that some platforms including FreeBSD <= 6.3, Cygwin and OS/2 EMX have
+ Note that some platforms including FreeBSD <= 6.3 and Cygwin have
known issues when using fork() from a thread.
.. warning::
@@ -2831,7 +2902,6 @@ written in Python, such as a mail server's external command delivery program.
:manpage:`times(2)` or the corresponding Windows Platform API documentation.
On Windows, only :attr:`user` and :attr:`system` are known; the other
attributes are zero.
- On OS/2, only :attr:`elapsed` is known; the other attributes are zero.
Availability: Unix, Windows.
@@ -3053,7 +3123,7 @@ information, consult your Unix manpages.
.. versionadded:: 3.3
-The following scheduling policies are exposed if they are a supported by the
+The following scheduling policies are exposed if they are supported by the
operating system.
.. data:: SCHED_OTHER
@@ -3162,10 +3232,6 @@ operating system.
Return the set of CPUs the process with PID *pid* (or the current process
if zero) is restricted to.
- .. seealso::
- :func:`multiprocessing.cpu_count` returns the number of CPUs in the
- system.
-
.. _os-path:
@@ -3203,6 +3269,13 @@ Miscellaneous System Information
Availability: Unix.
+.. function:: cpu_count()
+
+ Return the number of CPUs in the system. Returns None if undetermined.
+
+ .. versionadded:: 3.4
+
+
.. function:: getloadavg()
Return the number of processes in the system run queue averaged over the last
@@ -3301,6 +3374,19 @@ Higher-level operations on pathnames are defined in the :mod:`os.path` module.
The file path of the null device. For example: ``'/dev/null'`` for
POSIX, ``'nul'`` for Windows. Also available via :mod:`os.path`.
+.. data:: RTLD_LAZY
+ RTLD_NOW
+ RTLD_GLOBAL
+ RTLD_LOCAL
+ RTLD_NODELETE
+ RTLD_NOLOAD
+ RTLD_DEEPBIND
+
+ Flags for use with the :func:`~sys.setdlopenflags` and
+ :func:`~sys.getdlopenflags` functions. See the Unix manual page
+ :manpage:`dlopen(3)` for what the different flags mean.
+
+ .. versionadded:: 3.3
.. _os-miscfunc:
diff --git a/Doc/library/pathlib-inheritance.png b/Doc/library/pathlib-inheritance.png
new file mode 100644
index 0000000..b81c3de
--- /dev/null
+++ b/Doc/library/pathlib-inheritance.png
Binary files differ
diff --git a/Doc/library/pathlib-inheritance.svg b/Doc/library/pathlib-inheritance.svg
new file mode 100644
index 0000000..9f42005
--- /dev/null
+++ b/Doc/library/pathlib-inheritance.svg
@@ -0,0 +1,4 @@
+<?xml version="1.0" standalone="yes"?>
+
+<svg version="1.1" viewBox="0.0 0.0 538.0 496.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><clipPath id="p.0"><path d="m0 0l538.0 0l0 496.0l-538.0 0l0 -496.0z" clip-rule="nonzero"></path></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l538.916 0l0 496.08398l-538.916 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m176.0 24.0l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m176.0 24.0l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path fill="#000000" d="m235.18527 66.877716l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm6.4296875 1.09375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m16.0 144.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.0 144.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path fill="#000000" d="m51.181374 186.87772l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm6.4296875 1.09375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm14.6953125 1.4375q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm8.7734375 -1.8125q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm8.4453125 -4.921875l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm10.007805 3.40625l2.4375 2.078125q0.4375 0.03125 0.65625 0.25q0.21875 0.21875 0.21875 0.5625q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.28125 0.171875 -0.5q0.1875 -0.21875 0.484375 -0.296875l-1.1875 -1.03125l-1.2187424 1.03125q0.35936737 0.09375 0.5156174 0.296875q0.171875 0.1875 0.171875 0.5q0 0.359375 -0.25 0.59375q-0.23436737 0.21875 -0.8124924 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.328125 0.21875 -0.546875q0.21875 -0.21875 0.65625 -0.25l2.375 -2.09375l-2.109375 -1.796875q-0.40625 -0.03125 -0.625 -0.25q-0.21875 -0.21875 -0.21875 -0.546875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.8124924 0.21875q0.25 0.21875 0.25 0.5625q0 0.46875 -0.43749237 0.75l0.9687424 0.8125l0.953125 -0.828125q-0.421875 -0.296875 -0.421875 -0.703125q0 -0.375 0.234375 -0.59375q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-2.109375 1.8125zm7.4765625 0.4375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m336.0 143.46457l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m336.0 143.46457l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path fill="#000000" d="m361.5798 186.34229l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm13.9296875 -2.234375l0 5.484375q0.515625 0 0.75 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.515625 0l0 -0.375q-0.6875 0.3125 -1.3125 0.46875q-0.625 0.171875 -1.1875 0.171875q-0.796875 0 -1.375 -0.328125q-0.578125 -0.34375 -0.90625 -0.9375q-0.25 -0.421875 -0.25 -1.046875l0 -3.453125l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.875 0l0 4.765625q0 0.5 0.234375 0.75q0.25 0.234375 0.765625 0.234375q0.484375 0 1.03125 -0.1875q0.5625 -0.203125 1.390625 -0.703125l0 -3.265625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm5.8671875 0l0 1.0q1.015625 -0.734375 1.59375 -0.96875q0.578125 -0.25 1.09375 -0.25q0.78125 0 1.515625 0.578125q0.5 0.390625 0.5 0.796875q0 0.34375 -0.25 0.59375q-0.234375 0.234375 -0.5625 0.234375q-0.296875 0 -0.625 -0.296875q-0.328125 -0.296875 -0.59375 -0.296875q-0.328125 0 -1.0 0.421875q-0.671875 0.421875 -1.671875 1.265625l0 2.40625l2.28125 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-4.828125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.953125 0l0 -3.890625l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.171875 0zm13.9609375 4.359375l-6.578125 0q0.25 0.625 0.890625 1.015625q0.640625 0.375 1.71875 0.375q0.890625 0 2.375 -0.390625q0.609375 -0.15625 0.84375 -0.15625q0.3125 0 0.53125 0.234375q0.21875 0.21875 0.21875 0.5625q0 0.3125 -0.234375 0.53125q-0.3125 0.296875 -1.53125 0.5625q-1.203125 0.265625 -2.3125 0.265625q-1.921875 0 -3.078125 -1.09375q-1.15625 -1.09375 -1.15625 -2.671875q0 -1.6875 1.25 -2.75q1.25 -1.0625 2.875 -1.0625q0.96875 0 1.78125 0.34375q0.828125 0.34375 1.21875 0.75q0.5625 0.578125 0.9375 1.421875q0.25 0.59375 0.25 1.375l0 0.6875zm-1.78125 -1.609375q-0.359375 -0.6875 -0.953125 -1.015625q-0.59375 -0.34375 -1.421875 -0.34375q-0.8125 0 -1.40625 0.34375q-0.59375 0.328125 -0.96875 1.015625l4.75 0zm7.3671875 -0.203125l-1.484375 4.546875l-1.796875 0l-0.953125 -7.875q-0.375 -0.046875 -0.5625 -0.25q-0.1875 -0.203125 -0.1875 -0.53125q0 -0.375 0.25 -0.59375q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-0.828125 0l0.53125 4.484375l1.25 -3.734375l1.671875 0l1.25 3.734375l0.53125 -4.484375l-0.84375 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.21875 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.296875 -0.203125 0.515625q-0.1875 0.21875 -0.5625 0.28125l-0.921875 7.875l-1.765625 0l-1.53125 -4.546875zm10.1640625 -5.59375l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm7.1953125 0l0 0.53125q0.4375 -0.375 0.953125 -0.5625q0.53125 -0.1875 1.15625 -0.1875q1.421875 0 2.25 0.890625q0.65625 0.703125 0.65625 1.84375l0 2.96875q0.5 0 0.734375 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.453125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.34375 0.234375 -0.5625q0.25 -0.234375 0.75 -0.234375l0 -3.015625q0 -0.53125 -0.28125 -0.765625q-0.359375 -0.3125 -1.09375 -0.3125q-0.5625 0 -0.984375 0.21875q-0.40625 0.203125 -1.046875 0.90625l0 2.96875q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -3.890625q-0.5 0 -0.75 -0.21875q-0.234375 -0.234375 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.53125 0zm14.8984375 -3.046875l0 8.53125l0.265625 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.875 0l0 -0.390625q-0.546875 0.3125 -1.140625 0.484375q-0.59375 0.171875 -1.234375 0.171875q-1.8125 0 -2.921875 -1.046875q-1.09375 -1.046875 -1.09375 -2.609375q0 -1.625 1.15625 -2.765625q1.15625 -1.15625 2.828125 -1.15625q0.625 0 1.21875 0.203125q0.609375 0.1875 1.1875 0.5625l0 -1.984375l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l1.875 0zm-1.609375 6.796875q0 -0.984375 -0.703125 -1.671875q-0.6875 -0.6875 -1.6875 -0.6875q-1.0 0 -1.703125 0.6875q-0.6875 0.6875 -0.6875 1.65625q0 0.875 0.625 1.453125q0.625 0.5625 1.765625 0.5625q1.125 0 1.75 -0.5625q0.640625 -0.578125 0.640625 -1.4375zm11.6953125 -0.078125q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm7.0546875 0.453125l-1.109375 2.953125l-1.5 0l-1.34375 -5.5q-0.4375 -0.015625 -0.671875 -0.234375q-0.21875 -0.234375 -0.21875 -0.5625q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.28125 0.609375q-0.21875 0.1875 -0.828125 0.1875l0.609375 2.546875l0.984375 -2.609375l1.421875 0l1.0 2.609375l0.625 -2.546875q-0.59375 0 -0.78125 -0.109375q-0.375 -0.25 -0.375 -0.6875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-1.3125 5.5l-1.484375 0l-1.171875 -2.953125zm11.3203125 -2.265625q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm6.9453125 1.96875l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m176.0 271.46457l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m176.0 271.46457l187.43307 0l0 80.53543l-187.43307 0z" fill-rule="nonzero"></path><path fill="#000000" d="m254.3884 314.3423l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m16.0 400.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m16.0 400.0l187.43306 0l0 80.53543l-187.43306 0z" fill-rule="nonzero"></path><path fill="#000000" d="m70.3845 442.87772l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm14.6953125 1.4375q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm8.7734375 -1.8125q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm8.4453125 -4.921875l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm10.0078125 3.40625l2.4375 2.078125q0.4375 0.03125 0.65625 0.25q0.21875 0.21875 0.21875 0.5625q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.28125 0.171875 -0.5q0.1875 -0.21875 0.484375 -0.296875l-1.1875 -1.03125l-1.21875 1.03125q0.359375 0.09375 0.515625 0.296875q0.171875 0.1875 0.171875 0.5q0 0.359375 -0.25 0.59375q-0.234375 0.21875 -0.8125 0.21875l-1.8125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.328125 0.21875 -0.546875q0.21875 -0.21875 0.65625 -0.25l2.375 -2.09375l-2.109375 -1.796875q-0.40625 -0.03125 -0.625 -0.25q-0.21875 -0.21875 -0.21875 -0.546875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.5625q0 0.46875 -0.4375 0.75l0.96875 0.8125l0.953125 -0.828125q-0.421875 -0.296875 -0.421875 -0.703125q0 -0.375 0.234375 -0.59375q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-2.109375 1.8125zm7.4765625 0.4375l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.976555 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3124924 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.8437424 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.8437424 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.6249924 -0.109375 1.1249924 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.9687424 0 -1.6718674 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.5781174 0 1.2968674 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#ffffff" d="m336.0 400.0l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m336.0 400.0l187.43304 0l0 80.53543l-187.43304 0z" fill-rule="nonzero"></path><path fill="#000000" d="m381.72043 441.58084l-1.484375 4.546875l-1.796875 0l-0.953125 -7.875q-0.375 -0.046875 -0.5625 -0.25q-0.1875 -0.203125 -0.1875 -0.53125q0 -0.375 0.25 -0.59375q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-0.828125 0l0.53125 4.484375l1.25 -3.734375l1.671875 0l1.25 3.734375l0.53125 -4.484375l-0.84375 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.21875 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.234375 0.828125 -0.234375l2.125 0.015625q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.296875 -0.203125 0.515625q-0.1875 0.21875 -0.5625 0.28125l-0.921875 7.875l-1.765625 0l-1.53125 -4.546875zm10.1640625 -5.59375l0 1.703125l-1.90625 0l0 -1.703125l1.90625 0zm0.21875 3.046875l0 5.484375l1.921875 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-5.4375 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.921875 0l0 -3.890625l-1.296875 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l2.890625 0zm7.1953125 0l0 0.53125q0.4375 -0.375 0.953125 -0.5625q0.53125 -0.1875 1.15625 -0.1875q1.421875 0 2.25 0.890625q0.65625 0.703125 0.65625 1.84375l0 2.96875q0.5 0 0.734375 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.453125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.34375 0.234375 -0.5625q0.25 -0.234375 0.75 -0.234375l0 -3.015625q0 -0.53125 -0.28125 -0.765625q-0.359375 -0.3125 -1.09375 -0.3125q-0.5625 0 -0.984375 0.21875q-0.40625 0.203125 -1.046875 0.90625l0 2.96875q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -3.890625q-0.5 0 -0.75 -0.21875q-0.234375 -0.234375 -0.234375 -0.578125q0 -0.359375 0.234375 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.53125 0zm14.8984375 -3.046875l0 8.53125l0.265625 0q0.578125 0 0.828125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.25 0.21875 -0.828125 0.21875l-1.875 0l0 -0.390625q-0.546875 0.3125 -1.140625 0.484375q-0.59375 0.171875 -1.234375 0.171875q-1.8125 0 -2.921875 -1.046875q-1.09375 -1.046875 -1.09375 -2.609375q0 -1.625 1.15625 -2.765625q1.15625 -1.15625 2.828125 -1.15625q0.625 0 1.21875 0.203125q0.609375 0.1875 1.1875 0.5625l0 -1.984375l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l1.875 0zm-1.609375 6.796875q0 -0.984375 -0.703125 -1.671875q-0.6875 -0.6875 -1.6875 -0.6875q-1.0 0 -1.703125 0.6875q-0.6875 0.6875 -0.6875 1.65625q0 0.875 0.625 1.453125q0.625 0.5625 1.765625 0.5625q1.125 0 1.75 -0.5625q0.640625 -0.578125 0.640625 -1.4375zm11.6953125 -0.078125q0 0.921875 -0.515625 1.796875q-0.515625 0.859375 -1.53125 1.375q-1.0 0.515625 -2.109375 0.515625q-1.09375 0 -2.09375 -0.5q-1.0 -0.515625 -1.53125 -1.375q-0.515625 -0.875 -0.515625 -1.828125q0 -0.953125 0.53125 -1.875q0.53125 -0.9375 1.53125 -1.46875q1.0 -0.53125 2.078125 -0.53125q1.09375 0 2.109375 0.546875q1.015625 0.546875 1.53125 1.46875q0.515625 0.90625 0.515625 1.875zm-1.609375 0.015625q0 -0.78125 -0.546875 -1.421875q-0.765625 -0.875 -2.0 -0.875q-1.078125 0 -1.8125 0.703125q-0.71875 0.6875 -0.71875 1.59375q0 0.75 0.734375 1.40625q0.734375 0.65625 1.796875 0.65625q1.078125 0 1.8125 -0.65625q0.734375 -0.65625 0.734375 -1.40625zm7.0546875 0.453125l-1.109375 2.953125l-1.5 0l-1.34375 -5.5q-0.4375 -0.015625 -0.671875 -0.234375q-0.21875 -0.234375 -0.21875 -0.5625q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.484375 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.28125 0.609375q-0.21875 0.1875 -0.828125 0.1875l0.609375 2.546875l0.984375 -2.609375l1.421875 0l1.0 2.609375l0.625 -2.546875q-0.59375 0 -0.78125 -0.109375q-0.375 -0.25 -0.375 -0.6875q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l1.5 0q0.578125 0 0.828125 0.21875q0.25 0.21875 0.25 0.578125q0 0.328125 -0.21875 0.546875q-0.21875 0.21875 -0.640625 0.25l-1.3125 5.5l-1.484375 0l-1.171875 -2.953125zm11.3203125 -2.265625q-0.390625 -0.25 -0.828125 -0.359375q-0.421875 -0.125 -0.890625 -0.125q-0.921875 0 -1.46875 0.296875q-0.25 0.140625 -0.25 0.296875q0 0.171875 0.328125 0.34375q0.25 0.125 1.125 0.25q1.59375 0.21875 2.21875 0.4375q0.8125 0.28125 1.25 0.859375q0.453125 0.5625 0.453125 1.203125q0 0.859375 -0.75 1.4375q-1.09375 0.84375 -2.828125 0.84375q-0.6875 0 -1.28125 -0.125q-0.59375 -0.125 -1.078125 -0.359375q-0.125 0.09375 -0.265625 0.15625q-0.125 0.046875 -0.265625 0.046875q-0.375 0 -0.59375 -0.234375q-0.21875 -0.25 -0.21875 -0.828125l0 -0.546875q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.296875 0 0.484375 0.15625q0.203125 0.15625 0.3125 0.546875q0.359375 0.3125 0.875 0.484375q0.515625 0.15625 1.1875 0.15625q1.109375 0 1.71875 -0.34375q0.28125 -0.171875 0.28125 -0.359375q0 -0.3125 -0.40625 -0.515625q-0.421875 -0.203125 -1.71875 -0.34375q-1.921875 -0.203125 -2.578125 -0.78125q-0.640625 -0.578125 -0.640625 -1.40625q0 -0.859375 0.71875 -1.4375q0.984375 -0.78125 2.578125 -0.78125q0.5625 0 1.0625 0.109375q0.515625 0.109375 0.984375 0.328125q0.15625 -0.109375 0.28125 -0.15625q0.125 -0.0625 0.234375 -0.0625q0.328125 0 0.546875 0.25q0.21875 0.234375 0.21875 0.8125l0 0.390625q0 0.53125 -0.125 0.71875q-0.25 0.359375 -0.671875 0.359375q-0.296875 0 -0.515625 -0.171875q-0.21875 -0.1875 -0.28125 -0.484375zm6.9453125 1.96875l0 1.640625l1.625 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-3.46875 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l0.25 0l0 -6.265625l-0.25 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.234375 0.828125 -0.234375l3.6875 0.015625q1.625 0 2.5625 0.890625q0.953125 0.875 0.953125 2.15625q0 0.703125 -0.3125 1.328125q-0.25 0.46875 -0.8125 0.921875q-0.5625 0.453125 -1.15625 0.6875q-0.59375 0.234375 -1.5625 0.234375l-1.515625 0zm0 -1.609375l1.484375 0q1.046875 0 1.65625 -0.46875q0.625 -0.46875 0.625 -1.140625q0 -0.5625 -0.5 -0.984375q-0.5 -0.421875 -1.421875 -0.421875l-1.84375 0l0 3.015625zm11.9765625 4.859375l0 -0.375q-0.609375 0.3125 -1.34375 0.46875q-0.71875 0.171875 -1.3125 0.171875q-1.28125 0 -2.09375 -0.6875q-0.796875 -0.6875 -0.796875 -1.515625q0 -1.0 1.015625 -1.859375q1.03125 -0.875 2.84375 -0.875q0.734375 0 1.6875 0.15625l0 -0.375q0 -0.359375 -0.3125 -0.578125q-0.3125 -0.234375 -1.171875 -0.234375q-0.71875 0 -1.84375 0.28125q-0.421875 0.09375 -0.65625 0.09375q-0.328125 0 -0.546875 -0.21875q-0.21875 -0.234375 -0.21875 -0.59375q0 -0.203125 0.078125 -0.34375q0.078125 -0.15625 0.21875 -0.25q0.140625 -0.09375 0.578125 -0.21875q0.59375 -0.15625 1.203125 -0.25q0.625 -0.109375 1.125 -0.109375q1.5 0 2.3125 0.65625q0.828125 0.640625 0.828125 1.75l0 3.296875l0.28125 0q0.578125 0 0.8125 0.234375q0.25 0.21875 0.25 0.578125q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.875 0zm0 -2.875q-0.96875 -0.1875 -1.78125 -0.1875q-0.96875 0 -1.671875 0.484375q-0.4375 0.296875 -0.4375 0.609375q0 0.234375 0.203125 0.375q0.390625 0.25 1.078125 0.25q0.578125 0 1.296875 -0.21875q0.734375 -0.234375 1.3125 -0.625l0 -0.6875zm7.7578125 -2.625l0 3.21875q0 0.515625 0.21875 0.671875q0.328125 0.265625 1.171875 0.265625q1.21875 0 2.265625 -0.53125q0.390625 -0.203125 0.625 -0.203125q0.3125 0 0.53125 0.234375q0.234375 0.234375 0.234375 0.578125q0 0.3125 -0.25 0.53125q-0.375 0.375 -1.515625 0.6875q-1.125 0.3125 -1.890625 0.3125q-1.5 0 -2.25 -0.640625q-0.734375 -0.65625 -0.734375 -1.59375l0 -3.53125l-0.578125 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.21875 -0.25 -0.578125q0 -0.359375 0.25 -0.578125q0.25 -0.21875 0.828125 -0.21875l0.578125 0l0 -1.453125q0 -0.578125 0.21875 -0.8125q0.21875 -0.25 0.578125 -0.25q0.359375 0 0.578125 0.25q0.21875 0.234375 0.21875 0.8125l0 1.453125l2.96875 0q0.578125 0 0.8125 0.21875q0.25 0.21875 0.25 0.578125q0 0.359375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-2.96875 0zm8.3515625 -4.640625l0 3.421875q0.5 -0.296875 1.0 -0.4375q0.515625 -0.15625 1.046875 -0.15625q0.828125 0 1.46875 0.28125q0.65625 0.28125 1.078125 0.890625q0.421875 0.609375 0.421875 1.53125l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.5625 0 -0.8125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.4375 0.375 -0.671875q0.203125 -0.125 0.796875 -0.125l0 -2.890625q0 -0.625 -0.28125 -0.875q-0.359375 -0.328125 -1.078125 -0.328125q-0.53125 0 -0.953125 0.203125q-0.40625 0.203125 -1.09375 0.890625l0 3.0q0.609375 0 0.796875 0.125q0.375 0.234375 0.375 0.6875q0 0.34375 -0.25 0.578125q-0.234375 0.21875 -0.8125 0.21875l-1.828125 0q-0.578125 0 -0.828125 -0.21875q-0.234375 -0.234375 -0.234375 -0.59375q0 -0.4375 0.375 -0.671875q0.1875 -0.125 0.796875 -0.125l0 -6.921875l-0.265625 0q-0.578125 0 -0.828125 -0.21875q-0.25 -0.234375 -0.25 -0.59375q0 -0.34375 0.25 -0.5625q0.25 -0.234375 0.828125 -0.234375l1.875 0z" fill-rule="nonzero"></path><path fill="#000000" fill-opacity="0.0" d="m96.0 144.0l0 -80.0l80.0 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m96.0 144.0l0 -80.0l68.0 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m164.0 67.30347l9.076202 -3.3034668l-9.076202 -3.303463z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m431.5013 143.08398l0 -78.99999l-68.0 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m431.5013 143.08398l0 -78.99999l-56.0 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m375.5013 60.780525l-9.076202 3.3034668l9.076202 3.3034592z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m269.71652 104.53543l0 166.92914" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m269.71652 116.53543l0 154.92914" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m273.02 116.53543l-3.3034668 -9.076195l-3.3034668 9.076195z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m65.0 223.08398l-1.0078735 176.91339" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m64.93163 235.08379l-0.93950653 164.91359" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m68.23505 235.10262l-3.2517014 -9.094864l-3.3551178 9.05722z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m471.0 224.38058l-1.0078735 176.91336" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m470.93164 236.38037l-0.93951416 164.9136" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m474.23505 236.3992l-3.251709 -9.094864l-3.3551025 9.05722z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m128.50131 400.00266l0 -86.168l47.496002 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m128.50131 400.00266l0 -86.168l35.496002 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m163.99731 317.13812l9.076202 -3.3034668l-9.076202 -3.3034668z" fill-rule="evenodd"></path><path fill="#000000" fill-opacity="0.0" d="m414.71902 400.00266l0 -86.168l-51.216003 0" fill-rule="nonzero"></path><path stroke="#000000" stroke-width="2.0" stroke-linejoin="round" stroke-linecap="butt" d="m414.71902 400.00266l0 -86.168l-39.216003 0" fill-rule="evenodd"></path><path stroke="#000000" stroke-width="2.0" stroke-linecap="butt" d="m375.50302 310.5312l-9.076172 3.3034668l9.076172 3.3034668z" fill-rule="evenodd"></path></g></svg>
+
diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
new file mode 100644
index 0000000..ec1dc4f
--- /dev/null
+++ b/Doc/library/pathlib.rst
@@ -0,0 +1,919 @@
+
+:mod:`pathlib` --- Object-oriented filesystem paths
+===================================================
+
+.. module:: pathlib
+ :synopsis: Object-oriented filesystem paths
+
+.. index:: single: path; operations
+
+.. versionadded:: 3.4
+
+This module offers classes representing filesystem paths with semantics
+appropriate for different operating systems. Path classes are divided
+between :ref:`pure paths <pure-paths>`, which provide purely computational
+operations without I/O, and :ref:`concrete paths <concrete-paths>`, which
+inherit from pure paths but also provide I/O operations.
+
+.. image:: pathlib-inheritance.png
+ :align: center
+
+If you've never used this module before or just aren't sure which class is
+right for your task, :class:`Path` is most likely what you need. It instantiates
+a :ref:`concrete path <concrete-paths>` for the platform the code is running on.
+
+Pure paths are useful in some special cases; for example:
+
+#. If you want to manipulate Windows paths on a Unix machine (or vice versa).
+ You cannot instantiate a :class:`WindowsPath` when running on Unix, but you
+ can instantiate :class:`PureWindowsPath`.
+#. You want to make sure that your code only manipulates paths without actually
+ accessing the OS. In this case, instantiating one of the pure classes may be
+ useful since those simply don't have any OS-accessing operations.
+
+.. note::
+ This module has been included in the standard library on a
+ :term:`provisional basis <provisional package>`. Backwards incompatible
+ changes (up to and including removal of the package) may occur if deemed
+ necessary by the core developers.
+
+.. seealso::
+ :pep:`428`: The pathlib module -- object-oriented filesystem paths.
+
+.. seealso::
+ For low-level path manipulation on strings, you can also use the
+ :mod:`os.path` module.
+
+
+Basic use
+---------
+
+Importing the main class::
+
+ >>> from pathlib import Path
+
+Listing subdirectories::
+
+ >>> p = Path('.')
+ >>> [x for x in p.iterdir() if x.is_dir()]
+ [PosixPath('.hg'), PosixPath('docs'), PosixPath('dist'),
+ PosixPath('__pycache__'), PosixPath('build')]
+
+Listing Python source files in this directory tree::
+
+ >>> list(p.glob('**/*.py'))
+ [PosixPath('test_pathlib.py'), PosixPath('setup.py'),
+ PosixPath('pathlib.py'), PosixPath('docs/conf.py'),
+ PosixPath('build/lib/pathlib.py')]
+
+Navigating inside a directory tree::
+
+ >>> p = Path('/etc')
+ >>> q = p / 'init.d' / 'reboot'
+ >>> q
+ PosixPath('/etc/init.d/reboot')
+ >>> q.resolve()
+ PosixPath('/etc/rc.d/init.d/halt')
+
+Querying path properties::
+
+ >>> q.exists()
+ True
+ >>> q.is_dir()
+ False
+
+Opening a file::
+
+ >>> with q.open() as f: f.readline()
+ ...
+ '#!/bin/bash\n'
+
+
+.. _pure-paths:
+
+Pure paths
+----------
+
+Pure path objects provide path-handling operations which don't actually
+access a filesystem. There are three ways to access these classes, which
+we also call *flavours*:
+
+.. class:: PurePath(*pathsegments)
+
+ A generic class that represents the system's path flavour (instantiating
+ it creates either a :class:`PurePosixPath` or a :class:`PureWindowsPath`)::
+
+ >>> PurePath('setup.py') # Running on a Unix machine
+ PurePosixPath('setup.py')
+
+ Each element of *pathsegments* can be either a string or bytes object
+ representing a path segment; it can also be another path object::
+
+ >>> PurePath('foo', 'some/path', 'bar')
+ PurePosixPath('foo/some/path/bar')
+ >>> PurePath(Path('foo'), Path('bar'))
+ PurePosixPath('foo/bar')
+
+ When *pathsegments* is empty, the current directory is assumed::
+
+ >>> PurePath()
+ PurePosixPath('.')
+
+ When several absolute paths are given, the last is taken as an anchor
+ (mimicking :func:`os.path.join`'s behaviour)::
+
+ >>> PurePath('/etc', '/usr', 'lib64')
+ PurePosixPath('/usr/lib64')
+ >>> PureWindowsPath('c:/Windows', 'd:bar')
+ PureWindowsPath('d:bar')
+
+ However, in a Windows path, changing the local root doesn't discard the
+ previous drive setting::
+
+ >>> PureWindowsPath('c:/Windows', '/Program Files')
+ PureWindowsPath('c:/Program Files')
+
+ Spurious slashes and single dots are collapsed, but double dots (``'..'``)
+ are not, since this would change the meaning of a path in the face of
+ symbolic links::
+
+ >>> PurePath('foo//bar')
+ PurePosixPath('foo/bar')
+ >>> PurePath('foo/./bar')
+ PurePosixPath('foo/bar')
+ >>> PurePath('foo/../bar')
+ PurePosixPath('foo/../bar')
+
+ (a naïve approach would make ``PurePosixPath('foo/../bar')`` equivalent
+ to ``PurePosixPath('bar')``, which is wrong if ``foo`` is a symbolic link
+ to another directory)
+
+.. class:: PurePosixPath(*pathsegments)
+
+ A subclass of :class:`PurePath`, this path flavour represents non-Windows
+ filesystem paths::
+
+ >>> PurePosixPath('/etc')
+ PurePosixPath('/etc')
+
+ *pathsegments* is specified similarly to :class:`PurePath`.
+
+.. class:: PureWindowsPath(*pathsegments)
+
+ A subclass of :class:`PurePath`, this path flavour represents Windows
+ filesystem paths::
+
+ >>> PureWindowsPath('c:/Program Files/')
+ PureWindowsPath('c:/Program Files')
+
+ *pathsegments* is specified similarly to :class:`PurePath`.
+
+Regardless of the system you're running on, you can instantiate all of
+these classes, since they don't provide any operation that does system calls.
+
+
+General properties
+^^^^^^^^^^^^^^^^^^
+
+Paths are immutable and hashable. Paths of a same flavour are comparable
+and orderable. These properties respect the flavour's case-folding
+semantics::
+
+ >>> PurePosixPath('foo') == PurePosixPath('FOO')
+ False
+ >>> PureWindowsPath('foo') == PureWindowsPath('FOO')
+ True
+ >>> PureWindowsPath('FOO') in { PureWindowsPath('foo') }
+ True
+ >>> PureWindowsPath('C:') < PureWindowsPath('d:')
+ True
+
+Paths of a different flavour compare unequal and cannot be ordered::
+
+ >>> PureWindowsPath('foo') == PurePosixPath('foo')
+ False
+ >>> PureWindowsPath('foo') < PurePosixPath('foo')
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ TypeError: unorderable types: PureWindowsPath() < PurePosixPath()
+
+
+Operators
+^^^^^^^^^
+
+The slash operator helps create child paths, similarly to :func:`os.path.join`::
+
+ >>> p = PurePath('/etc')
+ >>> p
+ PurePosixPath('/etc')
+ >>> p / 'init.d' / 'apache2'
+ PurePosixPath('/etc/init.d/apache2')
+ >>> q = PurePath('bin')
+ >>> '/usr' / q
+ PurePosixPath('/usr/bin')
+
+The string representation of a path is the raw filesystem path itself
+(in native form, e.g. with backslashes under Windows), which you can
+pass to any function taking a file path as a string::
+
+ >>> p = PurePath('/etc')
+ >>> str(p)
+ '/etc'
+ >>> p = PureWindowsPath('c:/Program Files')
+ >>> str(p)
+ 'c:\\Program Files'
+
+Similarly, calling :class:`bytes` on a path gives the raw filesystem path as a
+bytes object, as encoded by :func:`os.fsencode`::
+
+ >>> bytes(p)
+ b'/etc'
+
+.. note::
+ Calling :class:`bytes` is only recommended under Unix. Under Windows,
+ the unicode form is the canonical representation of filesystem paths.
+
+
+Accessing individual parts
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the individual "parts" (components) of a path, use the following
+property:
+
+.. data:: PurePath.parts
+
+ A tuple giving access to the path's various components::
+
+ >>> p = PurePath('/usr/bin/python3')
+ >>> p.parts
+ ('/', 'usr', 'bin', 'python3')
+
+ >>> p = PureWindowsPath('c:/Program Files/PSF')
+ >>> p.parts
+ ('c:\\', 'Program Files', 'PSF')
+
+ (note how the drive and local root are regrouped in a single part)
+
+
+Methods and properties
+^^^^^^^^^^^^^^^^^^^^^^
+
+Pure paths provide the following methods and properties:
+
+.. data:: PurePath.drive
+
+ A string representing the drive letter or name, if any::
+
+ >>> PureWindowsPath('c:/Program Files/').drive
+ 'c:'
+ >>> PureWindowsPath('/Program Files/').drive
+ ''
+ >>> PurePosixPath('/etc').drive
+ ''
+
+ UNC shares are also considered drives::
+
+ >>> PureWindowsPath('//host/share/foo.txt').drive
+ '\\\\host\\share'
+
+.. data:: PurePath.root
+
+ A string representing the (local or global) root, if any::
+
+ >>> PureWindowsPath('c:/Program Files/').root
+ '\\'
+ >>> PureWindowsPath('c:Program Files/').root
+ ''
+ >>> PurePosixPath('/etc').root
+ '/'
+
+ UNC shares always have a root::
+
+ >>> PureWindowsPath('//host/share').root
+ '\\'
+
+.. data:: PurePath.anchor
+
+ The concatenation of the drive and root::
+
+ >>> PureWindowsPath('c:/Program Files/').anchor
+ 'c:\\'
+ >>> PureWindowsPath('c:Program Files/').anchor
+ 'c:'
+ >>> PurePosixPath('/etc').anchor
+ '/'
+ >>> PureWindowsPath('//host/share').anchor
+ '\\\\host\\share\\'
+
+
+.. data:: PurePath.parents
+
+ An immutable sequence providing access to the logical ancestors of
+ the path::
+
+ >>> p = PureWindowsPath('c:/foo/bar/setup.py')
+ >>> p.parents[0]
+ PureWindowsPath('c:/foo/bar')
+ >>> p.parents[1]
+ PureWindowsPath('c:/foo')
+ >>> p.parents[2]
+ PureWindowsPath('c:/')
+
+
+.. data:: PurePath.parent
+
+ The logical parent of the path::
+
+ >>> p = PurePosixPath('/a/b/c/d')
+ >>> p.parent
+ PurePosixPath('/a/b/c')
+
+ You cannot go past an anchor, or empty path::
+
+ >>> p = PurePosixPath('/')
+ >>> p.parent
+ PurePosixPath('/')
+ >>> p = PurePosixPath('.')
+ >>> p.parent
+ PurePosixPath('.')
+
+ .. note::
+ This is a purely lexical operation, hence the following behaviour::
+
+ >>> p = PurePosixPath('foo/..')
+ >>> p.parent
+ PurePosixPath('foo')
+
+ If you want to walk an arbitrary filesystem path upwards, it is
+ recommended to first call :meth:`Path.resolve` so as to resolve
+ symlinks and eliminate `".."` components.
+
+
+.. data:: PurePath.name
+
+ A string representing the final path component, excluding the drive and
+ root, if any::
+
+ >>> PurePosixPath('my/library/setup.py').name
+ 'setup.py'
+
+ UNC drive names are not considered::
+
+ >>> PureWindowsPath('//some/share/setup.py').name
+ 'setup.py'
+ >>> PureWindowsPath('//some/share').name
+ ''
+
+
+.. data:: PurePath.suffix
+
+ The file extension of the final component, if any::
+
+ >>> PurePosixPath('my/library/setup.py').suffix
+ '.py'
+ >>> PurePosixPath('my/library.tar.gz').suffix
+ '.gz'
+ >>> PurePosixPath('my/library').suffix
+ ''
+
+
+.. data:: PurePath.suffixes
+
+ A list of the path's file extensions::
+
+ >>> PurePosixPath('my/library.tar.gar').suffixes
+ ['.tar', '.gar']
+ >>> PurePosixPath('my/library.tar.gz').suffixes
+ ['.tar', '.gz']
+ >>> PurePosixPath('my/library').suffixes
+ []
+
+
+.. data:: PurePath.stem
+
+ The final path component, without its suffix::
+
+ >>> PurePosixPath('my/library.tar.gz').stem
+ 'library.tar'
+ >>> PurePosixPath('my/library.tar').stem
+ 'library'
+ >>> PurePosixPath('my/library').stem
+ 'library'
+
+
+.. method:: PurePath.as_posix()
+
+ Return a string representation of the path with forward slashes (``/``)::
+
+ >>> p = PureWindowsPath('c:\\windows')
+ >>> str(p)
+ 'c:\\windows'
+ >>> p.as_posix()
+ 'c:/windows'
+
+
+.. method:: PurePath.as_uri()
+
+ Represent the path as a ``file`` URI. :exc:`ValueError` is raised if
+ the path isn't absolute.
+
+ >>> p = PurePosixPath('/etc/passwd')
+ >>> p.as_uri()
+ 'file:///etc/passwd'
+ >>> p = PureWindowsPath('c:/Windows')
+ >>> p.as_uri()
+ 'file:///c:/Windows'
+
+
+.. method:: PurePath.is_absolute()
+
+ Return whether the path is absolute or not. A path is considered absolute
+ if it has both a root and (if the flavour allows) a drive::
+
+ >>> PurePosixPath('/a/b').is_absolute()
+ True
+ >>> PurePosixPath('a/b').is_absolute()
+ False
+
+ >>> PureWindowsPath('c:/a/b').is_absolute()
+ True
+ >>> PureWindowsPath('/a/b').is_absolute()
+ False
+ >>> PureWindowsPath('c:').is_absolute()
+ False
+ >>> PureWindowsPath('//some/share').is_absolute()
+ True
+
+
+.. method:: PurePath.is_reserved()
+
+ With :class:`PureWindowsPath`, return ``True`` if the path is considered
+ reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`,
+ ``False`` is always returned.
+
+ >>> PureWindowsPath('nul').is_reserved()
+ True
+ >>> PurePosixPath('nul').is_reserved()
+ False
+
+ File system calls on reserved paths can fail mysteriously or have
+ unintended effects.
+
+
+.. method:: PurePath.joinpath(*other)
+
+ Calling this method is equivalent to combining the path with each of
+ the *other* arguments in turn::
+
+ >>> PurePosixPath('/etc').joinpath('passwd')
+ PurePosixPath('/etc/passwd')
+ >>> PurePosixPath('/etc').joinpath(PurePosixPath('passwd'))
+ PurePosixPath('/etc/passwd')
+ >>> PurePosixPath('/etc').joinpath('init.d', 'apache2')
+ PurePosixPath('/etc/init.d/apache2')
+ >>> PureWindowsPath('c:').joinpath('/Program Files')
+ PureWindowsPath('c:/Program Files')
+
+
+.. method:: PurePath.match(pattern)
+
+ Match this path against the provided glob-style pattern. Return ``True``
+ if matching is successful, ``False`` otherwise.
+
+ If *pattern* is relative, the path can be either relative or absolute,
+ and matching is done from the right::
+
+ >>> PurePath('a/b.py').match('*.py')
+ True
+ >>> PurePath('/a/b/c.py').match('b/*.py')
+ True
+ >>> PurePath('/a/b/c.py').match('a/*.py')
+ False
+
+ If *pattern* is absolute, the path must be absolute, and the whole path
+ must match::
+
+ >>> PurePath('/a.py').match('/*.py')
+ True
+ >>> PurePath('a/b.py').match('/*.py')
+ False
+
+ As with other methods, case-sensitivity is observed::
+
+ >>> PureWindowsPath('b.py').match('*.PY')
+ True
+
+
+.. method:: PurePath.relative_to(*other)
+
+ Compute a version of this path relative to the path represented by
+ *other*. If it's impossible, ValueError is raised::
+
+ >>> p = PurePosixPath('/etc/passwd')
+ >>> p.relative_to('/')
+ PurePosixPath('etc/passwd')
+ >>> p.relative_to('/etc')
+ PurePosixPath('passwd')
+ >>> p.relative_to('/usr')
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "pathlib.py", line 694, in relative_to
+ .format(str(self), str(formatted)))
+ ValueError: '/etc/passwd' does not start with '/usr'
+
+
+.. method:: PurePath.with_name(name)
+
+ Return a new path with the :attr:`name` changed. If the original path
+ doesn't have a name, ValueError is raised::
+
+ >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz')
+ >>> p.with_name('setup.py')
+ PureWindowsPath('c:/Downloads/setup.py')
+ >>> p = PureWindowsPath('c:/')
+ >>> p.with_name('setup.py')
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "/home/antoine/cpython/default/Lib/pathlib.py", line 751, in with_name
+ raise ValueError("%r has an empty name" % (self,))
+ ValueError: PureWindowsPath('c:/') has an empty name
+
+
+.. method:: PurePath.with_suffix(suffix)
+
+ Return a new path with the :attr:`suffix` changed. If the original path
+ doesn't have a suffix, the new *suffix* is appended instead::
+
+ >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz')
+ >>> p.with_suffix('.bz2')
+ PureWindowsPath('c:/Downloads/pathlib.tar.bz2')
+ >>> p = PureWindowsPath('README')
+ >>> p.with_suffix('.txt')
+ PureWindowsPath('README.txt')
+
+
+.. _concrete-paths:
+
+
+Concrete paths
+--------------
+
+Concrete paths are subclasses of the pure path classes. In addition to
+operations provided by the latter, they also provide methods to do system
+calls on path objects. There are three ways to instantiate concrete paths:
+
+.. class:: Path(*pathsegments)
+
+ A subclass of :class:`PurePath`, this class represents concrete paths of
+ the system's path flavour (instantiating it creates either a
+ :class:`PosixPath` or a :class:`WindowsPath`)::
+
+ >>> Path('setup.py')
+ PosixPath('setup.py')
+
+ *pathsegments* is specified similarly to :class:`PurePath`.
+
+.. class:: PosixPath(*pathsegments)
+
+ A subclass of :class:`Path` and :class:`PurePosixPath`, this class
+ represents concrete non-Windows filesystem paths::
+
+ >>> PosixPath('/etc')
+ PosixPath('/etc')
+
+ *pathsegments* is specified similarly to :class:`PurePath`.
+
+.. class:: WindowsPath(*pathsegments)
+
+ A subclass of :class:`Path` and :class:`PureWindowsPath`, this class
+ represents concrete Windows filesystem paths::
+
+ >>> WindowsPath('c:/Program Files/')
+ WindowsPath('c:/Program Files')
+
+ *pathsegments* is specified similarly to :class:`PurePath`.
+
+You can only instantiate the class flavour that corresponds to your system
+(allowing system calls on non-compatible path flavours could lead to
+bugs or failures in your application)::
+
+ >>> import os
+ >>> os.name
+ 'posix'
+ >>> Path('setup.py')
+ PosixPath('setup.py')
+ >>> PosixPath('setup.py')
+ PosixPath('setup.py')
+ >>> WindowsPath('setup.py')
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "pathlib.py", line 798, in __new__
+ % (cls.__name__,))
+ NotImplementedError: cannot instantiate 'WindowsPath' on your system
+
+
+Methods
+^^^^^^^
+
+Concrete paths provide the following methods in addition to pure paths
+methods. Many of these methods can raise an :exc:`OSError` if a system
+call fails (for example because the path doesn't exist):
+
+.. classmethod:: Path.cwd()
+
+ Return a new path object representing the current directory (as returned
+ by :func:`os.getcwd`)::
+
+ >>> Path.cwd()
+ PosixPath('/home/antoine/pathlib')
+
+
+.. method:: Path.stat()
+
+ Return information about this path (similarly to :func:`os.stat`).
+ The result is looked up at each call to this method.
+
+ >>> p = Path('setup.py')
+ >>> p.stat().st_size
+ 956
+ >>> p.stat().st_mtime
+ 1327883547.852554
+
+
+.. method:: Path.chmod(mode)
+
+ Change the file mode and permissions, like :func:`os.chmod`::
+
+ >>> p = Path('setup.py')
+ >>> p.stat().st_mode
+ 33277
+ >>> p.chmod(0o444)
+ >>> p.stat().st_mode
+ 33060
+
+
+.. method:: Path.exists()
+
+ Whether the path points to an existing file or directory::
+
+ >>> Path('.').exists()
+ True
+ >>> Path('setup.py').exists()
+ True
+ >>> Path('/etc').exists()
+ True
+ >>> Path('nonexistentfile').exists()
+ False
+
+ .. note::
+ If the path points to a symlink, :meth:`exists` returns whether the
+ symlink *points to* an existing file or directory.
+
+
+.. method:: Path.glob(pattern)
+
+ Glob the given *pattern* in the directory represented by this path,
+ yielding all matching files (of any kind)::
+
+ >>> sorted(Path('.').glob('*.py'))
+ [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')]
+ >>> sorted(Path('.').glob('*/*.py'))
+ [PosixPath('docs/conf.py')]
+
+ The "``**``" pattern means "this directory and all subdirectories,
+ recursively". In other words, it enables recursive globbing::
+
+ >>> sorted(Path('.').glob('**/*.py'))
+ [PosixPath('build/lib/pathlib.py'),
+ PosixPath('docs/conf.py'),
+ PosixPath('pathlib.py'),
+ PosixPath('setup.py'),
+ PosixPath('test_pathlib.py')]
+
+ .. note::
+ Using the "``**``" pattern in large directory trees may consume
+ an inordinate amount of time.
+
+
+.. method:: Path.group()
+
+ Return the name of the group owning the file. :exc:`KeyError` is raised
+ if the file's gid isn't found in the system database.
+
+
+.. method:: Path.is_dir()
+
+ Return ``True`` if the path points to a directory (or a symbolic link
+ pointing to a directory), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.is_file()
+
+ Return ``True`` if the path points to a regular file (or a symbolic link
+ pointing to a regular file), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.is_symlink()
+
+ Return ``True`` if the path points to a symbolic link, ``False`` otherwise.
+
+ ``False`` is also returned if the path doesn't exist; other errors (such
+ as permission errors) are propagated.
+
+
+.. method:: Path.is_socket()
+
+ Return ``True`` if the path points to a Unix socket (or a symbolic link
+ pointing to a Unix socket), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.is_fifo()
+
+ Return ``True`` if the path points to a FIFO (or a symbolic link
+ pointing to a FIFO), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.is_block_device()
+
+ Return ``True`` if the path points to a block device (or a symbolic link
+ pointing to a block device), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.is_char_device()
+
+ Return ``True`` if the path points to a character device (or a symbolic link
+ pointing to a character device), ``False`` if it points to another kind of file.
+
+ ``False`` is also returned if the path doesn't exist or is a broken symlink;
+ other errors (such as permission errors) are propagated.
+
+
+.. method:: Path.iterdir()
+
+ When the path points to a directory, yield path objects of the directory
+ contents::
+
+ >>> p = Path('docs')
+ >>> for child in p.iterdir(): child
+ ...
+ PosixPath('docs/conf.py')
+ PosixPath('docs/_templates')
+ PosixPath('docs/make.bat')
+ PosixPath('docs/index.rst')
+ PosixPath('docs/_build')
+ PosixPath('docs/_static')
+ PosixPath('docs/Makefile')
+
+.. method:: Path.lchmod(mode)
+
+ Like :meth:`Path.chmod` but, if the path points to a symbolic link, the
+ symbolic link's mode is changed rather than its target's.
+
+
+.. method:: Path.lstat()
+
+ Like :meth:`Path.stat` but, if the path points to a symbolic link, return
+ the symbolic link's information rather than its target's.
+
+
+.. method:: Path.mkdir(mode=0o777, parents=False)
+
+ Create a new directory at this given path. If *mode* is given, it is
+ combined with the process' ``umask`` value to determine the file mode
+ and access flags. If the path already exists, :exc:`FileExistsError`
+ is raised.
+
+ If *parents* is true, any missing parents of this path are created
+ as needed; they are created with the default permissions without taking
+ *mode* into account (mimicking the POSIX ``mkdir -p`` command).
+
+ If *parents* is false (the default), a missing parent raises
+ :exc:`FileNotFoundError`.
+
+
+.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None)
+
+ Open the file pointed to by the path, like the built-in :func:`open`
+ function does::
+
+ >>> p = Path('setup.py')
+ >>> with p.open() as f:
+ ... f.readline()
+ ...
+ '#!/usr/bin/env python3\n'
+
+
+.. method:: Path.owner()
+
+ Return the name of the user owning the file. :exc:`KeyError` is raised
+ if the file's uid isn't found in the system database.
+
+
+.. method:: Path.rename(target)
+
+ Rename this file or directory to the given *target*. *target* can be
+ either a string or another path object::
+
+ >>> p = Path('foo')
+ >>> p.open('w').write('some text')
+ 9
+ >>> target = Path('bar')
+ >>> p.rename(target)
+ >>> target.open().read()
+ 'some text'
+
+
+.. method:: Path.replace(target)
+
+ Rename this file or directory to the given *target*. If *target* points
+ to an existing file or directory, it will be unconditionally replaced.
+
+
+.. method:: Path.resolve()
+
+ Make the path absolute, resolving any symlinks. A new path object is
+ returned::
+
+ >>> p = Path()
+ >>> p
+ PosixPath('.')
+ >>> p.resolve()
+ PosixPath('/home/antoine/pathlib')
+
+ `".."` components are also eliminated (this is the only method to do so)::
+
+ >>> p = Path('docs/../setup.py')
+ >>> p.resolve()
+ PosixPath('/home/antoine/pathlib/setup.py')
+
+ If the path doesn't exist, :exc:`FileNotFoundError` is raised. If an
+ infinite loop is encountered along the resolution path,
+ :exc:`RuntimeError` is raised.
+
+
+.. method:: Path.rglob(pattern)
+
+ This is like calling :meth:`glob` with "``**``" added in front of the
+ given *pattern*:
+
+ >>> sorted(Path().rglob("*.py"))
+ [PosixPath('build/lib/pathlib.py'),
+ PosixPath('docs/conf.py'),
+ PosixPath('pathlib.py'),
+ PosixPath('setup.py'),
+ PosixPath('test_pathlib.py')]
+
+
+.. method:: Path.rmdir()
+
+ Remove this directory. The directory must be empty.
+
+
+.. method:: Path.symlink_to(target, target_is_directory=False)
+
+ Make this path a symbolic link to *target*. Under Windows,
+ *target_is_directory* must be true (default ``False``) if the link's target
+ is a directory. Under POSIX, *target_is_directory*'s value is ignored.
+
+ >>> p = Path('mylink')
+ >>> p.symlink_to('setup.py')
+ >>> p.resolve()
+ PosixPath('/home/antoine/pathlib/setup.py')
+ >>> p.stat().st_size
+ 956
+ >>> p.lstat().st_size
+ 8
+
+ .. note::
+ The order of arguments (link, target) is the reverse
+ of :func:`os.symlink`'s.
+
+
+.. method:: Path.touch(mode=0o777, exist_ok=True)
+
+ Create a file at this given path. If *mode* is given, it is combined
+ with the process' ``umask`` value to determine the file mode and access
+ flags. If the file already exists, the function succeeds if *exist_ok*
+ is true (and its modification time is updated to the current time),
+ otherwise :exc:`FileExistsError` is raised.
+
+
+.. method:: Path.unlink()
+
+ Remove this file or symbolic link. If the path points to a directory,
+ use :func:`Path.rmdir` instead.
diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst
index f4e37ac..48a8a6b 100644
--- a/Doc/library/pdb.rst
+++ b/Doc/library/pdb.rst
@@ -41,7 +41,7 @@ of the debugger is::
.. versionchanged:: 3.3
Tab-completion via the :mod:`readline` module is available for commands and
command arguments, e.g. the current global and local names are offered as
- arguments of the ``print`` command.
+ arguments of the ``p`` command.
:file:`pdb.py` can also be invoked as a script to debug other scripts. For
example::
@@ -309,7 +309,7 @@ by the local file.
``end`` to terminate the commands. An example::
(Pdb) commands 1
- (com) print some_variable
+ (com) p some_variable
(com) end
(Pdb)
@@ -403,13 +403,19 @@ by the local file.
Print the argument list of the current function.
-.. pdbcommand:: p(rint) expression
+.. pdbcommand:: p expression
Evaluate the *expression* in the current context and print its value.
+ .. note::
+
+ ``print()`` can also be used, but is not a debugger command --- this executes the
+ Python :func:`print` function.
+
+
.. pdbcommand:: pp expression
- Like the :pdbcmd:`print` command, except the value of the expression is
+ Like the :pdbcmd:`p` command, except the value of the expression is
pretty-printed using the :mod:`pprint` module.
.. pdbcommand:: whatis expression
diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index 273fb34..ce5467f 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -94,6 +94,9 @@ There are fundamental differences between the pickle protocols and
The :mod:`json` module: a standard library module allowing JSON
serialization and deserialization.
+
+.. _pickle-protocols:
+
Data stream format
------------------
@@ -113,7 +116,9 @@ The module :mod:`pickletools` contains tools for analyzing data streams
generated by :mod:`pickle`. :mod:`pickletools` source code has extensive
comments about opcodes used by pickle protocols.
-There are currently 4 different protocols which can be used for pickling.
+There are currently 5 different protocols which can be used for pickling.
+The higher the protocol used, the more recent the version of Python needed
+to read the pickle produced.
* Protocol version 0 is the original "human-readable" protocol and is
backwards compatible with earlier versions of Python.
@@ -125,10 +130,15 @@ There are currently 4 different protocols which can be used for pickling.
efficient pickling of :term:`new-style class`\es. Refer to :pep:`307` for
information about improvements brought by protocol 2.
-* Protocol version 3 was added in Python 3. It has explicit support for
+* Protocol version 3 was added in Python 3.0. It has explicit support for
:class:`bytes` objects and cannot be unpickled by Python 2.x. This is
- the default as well as the current recommended protocol; use it whenever
- possible.
+ the default protocol, and the recommended protocol when compatibility with
+ other Python 3 versions is required.
+
+* Protocol version 4 was added in Python 3.4. It adds support for very large
+ objects, pickling more kinds of objects, and some data format
+ optimizations. Refer to :pep:`3154` for information about improvements
+ brought by protocol 4.
.. note::
Serialization is a more primitive notion than persistence; although
@@ -156,13 +166,16 @@ The :mod:`pickle` module provides the following constants:
.. data:: HIGHEST_PROTOCOL
- The highest protocol version available. This value can be passed as a
- *protocol* value.
+ An integer, the highest :ref:`protocol version <pickle-protocols>`
+ available. This value can be passed as a *protocol* value to functions
+ :func:`dump` and :func:`dumps` as well as the :class:`Pickler`
+ constructor.
.. data:: DEFAULT_PROTOCOL
- The default protocol used for pickling. May be less than HIGHEST_PROTOCOL.
- Currently the default protocol is 3, a new protocol designed for Python 3.0.
+ An integer, the default :ref:`protocol version <pickle-protocols>` used
+ for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the
+ default protocol is 3, a new protocol designed for Python 3.
The :mod:`pickle` module provides the following functions to make the pickling
@@ -173,13 +186,10 @@ process more convenient:
Write a pickled representation of *obj* to the open :term:`file object` *file*.
This is equivalent to ``Pickler(file, protocol).dump(obj)``.
- The optional *protocol* argument tells the pickler to use the given protocol;
- supported protocols are 0, 1, 2, 3. The default protocol is 3; a
- backward-incompatible protocol designed for Python 3.0.
-
- Specifying a negative protocol version selects the highest protocol version
- supported. The higher the protocol used, the more recent the version of
- Python needed to read the pickle produced.
+ The optional *protocol* argument, an integer, tells the pickler to use
+ the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`.
+ If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative
+ number is specified, :data:`HIGHEST_PROTOCOL` is selected.
The *file* argument must have a write() method that accepts a single bytes
argument. It can thus be an on-disk file opened for binary writing, a
@@ -187,64 +197,57 @@ process more convenient:
interface.
If *fix_imports* is true and *protocol* is less than 3, pickle will try to
- map the new Python 3.x names to the old module names used in Python 2.x,
- so that the pickle data stream is readable with Python 2.x.
+ map the new Python 3 names to the old module names used in Python 2, so
+ that the pickle data stream is readable with Python 2.
.. function:: dumps(obj, protocol=None, \*, fix_imports=True)
- Return the pickled representation of the object as a :class:`bytes`
- object, instead of writing it to a file.
-
- The optional *protocol* argument tells the pickler to use the given protocol;
- supported protocols are 0, 1, 2, 3. The default protocol is 3; a
- backward-incompatible protocol designed for Python 3.0.
+ Return the pickled representation of the object as a :class:`bytes` object,
+ instead of writing it to a file.
- Specifying a negative protocol version selects the highest protocol version
- supported. The higher the protocol used, the more recent the version of
- Python needed to read the pickle produced.
-
- If *fix_imports* is true and *protocol* is less than 3, pickle will try to
- map the new Python 3.x names to the old module names used in Python 2.x,
- so that the pickle data stream is readable with Python 2.x.
+ Arguments *protocol* and *fix_imports* have the same meaning as in
+ :func:`dump`.
.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict")
- Read a pickled object representation from the open :term:`file object` *file*
- and return the reconstituted object hierarchy specified therein. This is
- equivalent to ``Unpickler(file).load()``.
+ Read a pickled object representation from the open :term:`file object`
+ *file* and return the reconstituted object hierarchy specified therein.
+ This is equivalent to ``Unpickler(file).load()``.
- The protocol version of the pickle is detected automatically, so no protocol
- argument is needed. Bytes past the pickled object's representation are
- ignored.
+ The protocol version of the pickle is detected automatically, so no
+ protocol argument is needed. Bytes past the pickled object's
+ representation are ignored.
The argument *file* must have two methods, a read() method that takes an
integer argument, and a readline() method that requires no arguments. Both
- methods should return bytes. Thus *file* can be an on-disk file opened
- for binary reading, a :class:`io.BytesIO` object, or any other custom object
+ methods should return bytes. Thus *file* can be an on-disk file opened for
+ binary reading, a :class:`io.BytesIO` object, or any other custom object
that meets this interface.
Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
which are used to control compatibility support for pickle stream generated
- by Python 2.x. If *fix_imports* is true, pickle will try to map the old
- Python 2.x names to the new names used in Python 3.x. The *encoding* and
+ by Python 2. If *fix_imports* is true, pickle will try to map the old
+ Python 2 names to the new names used in Python 3. The *encoding* and
*errors* tell pickle how to decode 8-bit string instances pickled by Python
- 2.x; these default to 'ASCII' and 'strict', respectively.
+ 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
+ be 'bytes' to read these 8-bit string instances as bytes objects.
.. function:: loads(bytes_object, \*, fix_imports=True, encoding="ASCII", errors="strict")
Read a pickled object hierarchy from a :class:`bytes` object and return the
reconstituted object hierarchy specified therein
- The protocol version of the pickle is detected automatically, so no protocol
- argument is needed. Bytes past the pickled object's representation are
- ignored.
+ The protocol version of the pickle is detected automatically, so no
+ protocol argument is needed. Bytes past the pickled object's
+ representation are ignored.
Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
which are used to control compatibility support for pickle stream generated
- by Python 2.x. If *fix_imports* is true, pickle will try to map the old
- Python 2.x names to the new names used in Python 3.x. The *encoding* and
+ by Python 2. If *fix_imports* is true, pickle will try to map the old
+ Python 2 names to the new names used in Python 3. The *encoding* and
*errors* tell pickle how to decode 8-bit string instances pickled by Python
- 2.x; these default to 'ASCII' and 'strict', respectively.
+ 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
+ be 'bytes' to read these 8-bit string instances as bytes objects.
The :mod:`pickle` module defines three exceptions:
@@ -279,21 +282,19 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and
This takes a binary file for writing a pickle data stream.
- The optional *protocol* argument tells the pickler to use the given protocol;
- supported protocols are 0, 1, 2, 3. The default protocol is 3; a
- backward-incompatible protocol designed for Python 3.0.
-
- Specifying a negative protocol version selects the highest protocol version
- supported. The higher the protocol used, the more recent the version of
- Python needed to read the pickle produced.
+ The optional *protocol* argument, an integer, tells the pickler to use
+ the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`.
+ If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative
+ number is specified, :data:`HIGHEST_PROTOCOL` is selected.
The *file* argument must have a write() method that accepts a single bytes
argument. It can thus be an on-disk file opened for binary writing, a
- :class:`io.BytesIO` instance, or any other custom object that meets this interface.
+ :class:`io.BytesIO` instance, or any other custom object that meets this
+ interface.
If *fix_imports* is true and *protocol* is less than 3, pickle will try to
- map the new Python 3.x names to the old module names used in Python 2.x,
- so that the pickle data stream is readable with Python 2.x.
+ map the new Python 3 names to the old module names used in Python 2, so
+ that the pickle data stream is readable with Python 2.
.. method:: dump(obj)
@@ -355,16 +356,17 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and
The argument *file* must have two methods, a read() method that takes an
integer argument, and a readline() method that requires no arguments. Both
- methods should return bytes. Thus *file* can be an on-disk file object opened
- for binary reading, a :class:`io.BytesIO` object, or any other custom object
- that meets this interface.
+ methods should return bytes. Thus *file* can be an on-disk file object
+ opened for binary reading, a :class:`io.BytesIO` object, or any other
+ custom object that meets this interface.
Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
which are used to control compatibility support for pickle stream generated
- by Python 2.x. If *fix_imports* is true, pickle will try to map the old
- Python 2.x names to the new names used in Python 3.x. The *encoding* and
+ by Python 2. If *fix_imports* is true, pickle will try to map the old
+ Python 2 names to the new names used in Python 3. The *encoding* and
*errors* tell pickle how to decode 8-bit string instances pickled by Python
- 2.x; these default to 'ASCII' and 'strict', respectively.
+ 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can
+ be 'bytes' to read these ß8-bit string instances as bytes objects.
.. method:: load()
@@ -409,7 +411,8 @@ The following types can be pickled:
* tuples, lists, sets, and dictionaries containing only picklable objects
-* functions defined at the top level of a module
+* functions defined at the top level of a module (using :keyword:`def`, not
+ :keyword:`lambda`)
* built-in functions defined at the top level of a module
@@ -427,7 +430,7 @@ raised in this case. You can carefully raise this limit with
:func:`sys.setrecursionlimit`.
Note that functions (built-in and user-defined) are pickled by "fully qualified"
-name reference, not by value. This means that only the function name is
+name reference, not by value. [#]_ This means that only the function name is
pickled, along with the name of the module the function is defined in. Neither
the function's code, nor any of its function attributes are pickled. Thus the
defining module must be importable in the unpickling environment, and the module
@@ -483,12 +486,29 @@ implementation of this behaviour::
Classes can alter the default behaviour by providing one or several special
methods:
+.. method:: object.__getnewargs_ex__()
+
+ In protocols 4 and newer, classes that implements the
+ :meth:`__getnewargs_ex__` method can dictate the values passed to the
+ :meth:`__new__` method upon unpickling. The method must return a pair
+ ``(args, kwargs)`` where *args* is a tuple of positional arguments
+ and *kwargs* a dictionary of named arguments for constructing the
+ object. Those will be passed to the :meth:`__new__` method upon
+ unpickling.
+
+ You should implement this method if the :meth:`__new__` method of your
+ class requires keyword-only arguments. Otherwise, it is recommended for
+ compatibility to implement :meth:`__getnewargs__`.
+
+
.. method:: object.__getnewargs__()
- In protocol 2 and newer, classes that implements the :meth:`__getnewargs__`
- method can dictate the values passed to the :meth:`__new__` method upon
- unpickling. This is often needed for classes whose :meth:`__new__` method
- requires arguments.
+ This method serve a similar purpose as :meth:`__getnewargs_ex__` but
+ for protocols 2 and newer. It must return a tuple of arguments `args`
+ which will be passed to the :meth:`__new__` method upon unpickling.
+
+ In protocols 4 and newer, :meth:`__getnewargs__` will not be called if
+ :meth:`__getnewargs_ex__` is defined.
.. method:: object.__getstate__()
@@ -520,10 +540,10 @@ the methods :meth:`__getstate__` and :meth:`__setstate__`.
At unpickling time, some methods like :meth:`__getattr__`,
:meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the
- instance. In case those methods rely on some internal invariant being true,
- the type should implement :meth:`__getnewargs__` to establish such an
- invariant; otherwise, neither :meth:`__new__` nor :meth:`__init__` will be
- called.
+ instance. In case those methods rely on some internal invariant being
+ true, the type should implement :meth:`__getnewargs__` or
+ :meth:`__getnewargs_ex__` to establish such an invariant; otherwise,
+ neither :meth:`__new__` nor :meth:`__init__` will be called.
.. index:: pair: copy; protocol
@@ -535,7 +555,7 @@ objects. [#]_
Although powerful, implementing :meth:`__reduce__` directly in your classes is
error prone. For this reason, class designers should use the high-level
-interface (i.e., :meth:`__getnewargs__`, :meth:`__getstate__` and
+interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and
:meth:`__setstate__`) whenever possible. We will show, however, cases where
using :meth:`__reduce__` is the only option or leads to more efficient pickling
or both.
@@ -883,6 +903,9 @@ The following example reads the resulting pickled data. ::
.. [#] Don't confuse this with the :mod:`marshal` module
+.. [#] This is why :keyword:`lambda` functions cannot be pickled: all
+ :keyword:`lambda` functions share the same name: ``<lambda>``.
+
.. [#] The exception raised will likely be an :exc:`ImportError` or an
:exc:`AttributeError` but it could be something else.
diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst
index 7c4f912..13ea7b9 100644
--- a/Doc/library/pkgutil.rst
+++ b/Doc/library/pkgutil.rst
@@ -74,15 +74,17 @@ support.
Retrieve a :pep:`302` module loader for the given *fullname*.
- This is a convenience wrapper around :func:`importlib.find_loader` that
- sets the *path* argument correctly when searching for submodules, and
- also ensures parent packages (if any) are imported before searching for
- submodules.
+ This is a backwards compatibility wrapper around
+ :func:`importlib.util.find_spec` that converts most failures to
+ :exc:`ImportError` and only returns the loader rather than the full
+ :class:`ModuleSpec`.
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
on the package internal PEP 302 import emulation.
+ .. versionchanged:: 3.4
+ Updated to be based on :pep:`451`
.. function:: get_importer(path_item)
@@ -109,14 +111,13 @@ support.
not already imported, its containing package (if any) is imported, in order
to establish the package ``__path__``.
- This function uses :func:`iter_importers`, and is thus subject to the same
- limitations regarding platform-specific special import locations such as the
- Windows registry.
-
.. versionchanged:: 3.3
Updated to be based directly on :mod:`importlib` rather than relying
on the package internal PEP 302 import emulation.
+ .. versionchanged:: 3.4
+ Updated to be based on :pep:`451`
+
.. function:: iter_importers(fullname='')
diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
index a267368..6a2d6b4 100644
--- a/Doc/library/plistlib.rst
+++ b/Doc/library/plistlib.rst
@@ -16,26 +16,24 @@
--------------
This module provides an interface for reading and writing the "property list"
-XML files used mainly by Mac OS X.
+files used mainly by Mac OS X and supports both binary and XML plist files.
-The property list (``.plist``) file format is a simple XML pickle supporting
+The property list (``.plist``) file format is a simple serialization supporting
basic object types, like dictionaries, lists, numbers and strings. Usually the
top level object is a dictionary.
-To write out and to parse a plist file, use the :func:`writePlist` and
-:func:`readPlist` functions.
+To write out and to parse a plist file, use the :func:`dump` and
+:func:`load` functions.
-To work with plist data in bytes objects, use :func:`writePlistToBytes`
-and :func:`readPlistFromBytes`.
+To work with plist data in bytes objects, use :func:`dumps`
+and :func:`loads`.
Values can be strings, integers, floats, booleans, tuples, lists, dictionaries
-(but only with string keys), :class:`Data` or :class:`datetime.datetime`
-objects. String values (including dictionary keys) have to be unicode strings --
-they will be written out as UTF-8.
+(but only with string keys), :class:`Data`, :class:`bytes`, :class:`bytesarray`
+or :class:`datetime.datetime` objects.
-The ``<data>`` plist type is supported through the :class:`Data` class. This is
-a thin wrapper around a Python bytes object. Use :class:`Data` if your strings
-contain control characters.
+.. versionchanged:: 3.4
+ New API, old API deprecated. Support for binary format plists added.
.. seealso::
@@ -45,37 +43,147 @@ contain control characters.
This module defines the following functions:
-.. function:: readPlist(pathOrFile)
+.. function:: load(fp, \*, fmt=None, use_builtin_types=True, dict_type=dict)
- Read a plist file. *pathOrFile* may either be a file name or a (readable and
- binary) file object. Return the unpacked root object (which usually is a
+ Read a plist file. *fp* should be a readable and binary file object.
+ Return the unpacked root object (which usually is a
dictionary).
- The XML data is parsed using the Expat parser from :mod:`xml.parsers.expat`
- -- see its documentation for possible exceptions on ill-formed XML.
- Unknown elements will simply be ignored by the plist parser.
+ The *fmt* is the format of the file and the following values are valid:
+
+ * :data:`None`: Autodetect the file format
+
+ * :data:`FMT_XML`: XML file format
+
+ * :data:`FMT_BINARY`: Binary plist format
+
+ If *use_builtin_types* is true (the default) binary data will be returned
+ as instances of :class:`bytes`, otherwise it is returned as instances of
+ :class:`Data`.
+
+ The *dict_type* is the type used for dictionaries that are read from the
+ plist file. The exact structure of the plist can be recovered by using
+ :class:`collections.OrderedDict` (although the order of keys shouldn't be
+ important in plist files).
+
+ XML data for the :data:`FMT_XML` format is parsed using the Expat parser
+ from :mod:`xml.parsers.expat` -- see its documentation for possible
+ exceptions on ill-formed XML. Unknown elements will simply be ignored
+ by the plist parser.
+
+ The parser for the binary format raises :exc:`InvalidFileException`
+ when the file cannot be parsed.
+
+ .. versionadded:: 3.4
+
+
+.. function:: loads(data, \*, fmt=None, use_builtin_types=True, dict_type=dict)
+
+ Load a plist from a bytes object. See :func:`load` for an explanation of
+ the keyword arguments.
+
+ .. versionadded:: 3.4
+
+
+.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False)
+
+ Write *value* to a plist file. *Fp* should be a writable, binary
+ file object.
+
+ The *fmt* argument specifies the format of the plist file and can be
+ one of the following values:
+
+ * :data:`FMT_XML`: XML formatted plist file
+
+ * :data:`FMT_BINARY`: Binary formatted plist file
+
+ When *sort_keys* is true (the default) the keys for dictionaries will be
+ written to the plist in sorted order, otherwise they will be written in
+ the iteration order of the dictionary.
+
+ When *skipkeys* is false (the default) the function raises :exc:`TypeError`
+ when a key of a dictionary is not a string, otherwise such keys are skipped.
+
+ A :exc:`TypeError` will be raised if the object is of an unsupported type or
+ a container that contains objects of unsupported types.
+
+ An :exc:`OverflowError` will be raised for integer values that cannot
+ be represented in (binary) plist files.
+
+ .. versionadded:: 3.4
+
+
+.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False)
+
+ Return *value* as a plist-formatted bytes object. See
+ the documentation for :func:`dump` for an explanation of the keyword
+ arguments of this function.
+
+ .. versionadded:: 3.4
+
+The following functions are deprecated:
+
+.. function:: readPlist(pathOrFile)
+
+ Read a plist file. *pathOrFile* may be either a file name or a (readable
+ and binary) file object. Returns the unpacked root object (which usually
+ is a dictionary).
+
+ This function calls :func:`load` to do the actual work, the the documentation
+ of :func:`that function <load>` for an explanation of the keyword arguments.
+
+ .. note::
+
+ Dict values in the result have a ``__getattr__`` method that defers
+ to ``__getitem_``. This means that you can use attribute access to
+ access items of these dictionaries.
+
+ .. deprecated:: 3.4 Use :func:`load` instead.
.. function:: writePlist(rootObject, pathOrFile)
- Write *rootObject* to a plist file. *pathOrFile* may either be a file name
- or a (writable and binary) file object.
+ Write *rootObject* to an XML plist file. *pathOrFile* may be either a file name
+ or a (writable and binary) file object
- A :exc:`TypeError` will be raised if the object is of an unsupported type or
- a container that contains objects of unsupported types.
+ .. deprecated:: 3.4 Use :func:`dump` instead.
.. function:: readPlistFromBytes(data)
Read a plist data from a bytes object. Return the root object.
+ See :func:`load` for a description of the keyword arguments.
+
+ .. note::
+
+ Dict values in the result have a ``__getattr__`` method that defers
+ to ``__getitem_``. This means that you can use attribute access to
+ access items of these dictionaries.
+
+ .. deprecated:: 3.4 Use :func:`loads` instead.
+
.. function:: writePlistToBytes(rootObject)
- Return *rootObject* as a plist-formatted bytes object.
+ Return *rootObject* as an XML plist-formatted bytes object.
+
+ .. deprecated:: 3.4 Use :func:`dumps` instead.
+
+The following classes are available:
+
+.. class:: Dict([dict]):
+
+ Return an extended mapping object with the same value as dictionary
+ *dict*.
+
+ This class is a subclass of :class:`dict` where attribute access can
+ be used to access items. That is, ``aDict.key`` is the same as
+ ``aDict['key']`` for getting, setting and deleting items in the mapping.
+
+ .. deprecated:: 3.0
-The following class is available:
.. class:: Data(data)
@@ -86,6 +194,24 @@ The following class is available:
It has one attribute, :attr:`data`, that can be used to retrieve the Python
bytes object stored in it.
+ .. deprecated:: 3.4 Use a :class:`bytes` object instead
+
+
+The following constants are available:
+
+.. data:: FMT_XML
+
+ The XML format for plist files.
+
+ .. versionadded:: 3.4
+
+
+.. data:: FMT_BINARY
+
+ The binary format for plist files
+
+ .. versionadded:: 3.4
+
Examples
--------
@@ -103,13 +229,15 @@ Generating a plist::
aTrueValue = True,
aFalseValue = False,
),
- someData = Data(b"<binary gunk>"),
- someMoreData = Data(b"<lots of binary gunk>" * 10),
+ someData = b"<binary gunk>",
+ someMoreData = b"<lots of binary gunk>" * 10,
aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())),
)
- writePlist(pl, fileName)
+ with open(fileName, 'wb') as fp:
+ dump(pl, fp)
Parsing a plist::
- pl = readPlist(pathOrFile)
+ with open(fileName, 'rb') as fp:
+ pl = load(fp)
print(pl["aKey"])
diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst
index b4080d6..bc7b3e7 100644
--- a/Doc/library/poplib.rst
+++ b/Doc/library/poplib.rst
@@ -13,8 +13,11 @@
--------------
This module defines a class, :class:`POP3`, which encapsulates a connection to a
-POP3 server and implements the protocol as defined in :rfc:`1725`. The
-:class:`POP3` class supports both the minimal and optional command sets.
+POP3 server and implements the protocol as defined in :rfc:`1939`. The
+:class:`POP3` class supports both the minimal and optional command sets from
+:rfc:`1939`. The :class:`POP3` class also supports the `STLS` command introduced
+in :rfc:`2595` to enable encrypted communication on an already established connection.
+
Additionally, this module provides a class :class:`POP3_SSL`, which provides
support for connecting to POP3 servers that use SSL as an underlying protocol
layer.
@@ -40,16 +43,23 @@ The :mod:`poplib` module provides two classes:
This is a subclass of :class:`POP3` that connects to the server over an SSL
encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL
- port is used. *keyfile* and *certfile* are also optional - they can contain a
- PEM formatted private key and certificate chain file for the SSL connection.
- *timeout* works as in the :class:`POP3` constructor. *context* parameter is a
- :class:`ssl.SSLContext` object which allows bundling SSL configuration
- options, certificates and private keys into a single (potentially long-lived)
- structure.
+ port is used. *timeout* works as in the :class:`POP3` constructor.
+ *context* is an optional :class:`ssl.SSLContext` object which allows
+ bundling SSL configuration options, certificates and private keys into a
+ single (potentially long-lived) structure. Please read :ref:`ssl-security`
+ for best practices.
+
+ *keyfile* and *certfile* are a legacy alternative to *context* - they can
+ point to PEM-formatted private key and certificate chain files,
+ respectively, for the SSL connection.
.. versionchanged:: 3.2
*context* parameter added.
+ .. versionchanged:: 3.4
+ The class now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
One exception is defined as an attribute of the :mod:`poplib` module:
@@ -97,6 +107,14 @@ An :class:`POP3` instance has the following methods:
Returns the greeting string sent by the POP3 server.
+.. method:: POP3.capa()
+
+ Query the server's capabilities as specified in :rfc:`2449`.
+ Returns a dictionary in the form ``{'name': ['param'...]}``.
+
+ .. versionadded:: 3.4
+
+
.. method:: POP3.user(username)
Send user command, response should indicate that a password is required.
@@ -176,6 +194,23 @@ An :class:`POP3` instance has the following methods:
the unique id for that message in the form ``'response mesgnum uid``, otherwise
result is list ``(response, ['mesgnum uid', ...], octets)``.
+.. method:: POP3.stls(context=None)
+
+ Start a TLS session on the active connection as specified in :rfc:`2595`.
+ This is only allowed before user authentication
+
+ *context* parameter is a :class:`ssl.SSLContext` object which allows
+ bundling SSL configuration options, certificates and private keys into
+ a single (potentially long-lived) structure. Please read :ref:`ssl-security`
+ for best practices.
+
+ This method supports hostname checking via
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
+
+ .. versionadded:: 3.4
+
+
Instances of :class:`POP3_SSL` have no additional methods. The interface of this
subclass is identical to its parent.
diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst
index 3a86331..447f8f7 100644
--- a/Doc/library/pprint.rst
+++ b/Doc/library/pprint.rst
@@ -14,8 +14,8 @@ The :mod:`pprint` module provides a capability to "pretty-print" arbitrary
Python data structures in a form which can be used as input to the interpreter.
If the formatted structures include objects which are not fundamental Python
types, the representation may not be loadable. This may be the case if objects
-such as files, sockets, classes, or instances are included, as well as many
-other built-in objects which are not representable as Python constants.
+such as files, sockets or classes are included, as well as many other
+objects which are not representable as Python literals.
The formatted representation keeps objects on a single line if it can, and
breaks them onto multiple lines if they don't fit within the allowed width.
@@ -29,14 +29,14 @@ The :mod:`pprint` module defines one class:
.. First the implementation class:
-.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None)
+.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \
+ compact=False)
Construct a :class:`PrettyPrinter` instance. This constructor understands
several keyword parameters. An output stream may be set using the *stream*
keyword; the only method used on the stream object is the file protocol's
:meth:`write` method. If not specified, the :class:`PrettyPrinter` adopts
- ``sys.stdout``. Three additional parameters may be used to control the
- formatted representation. The keywords are *indent*, *depth*, and *width*. The
+ ``sys.stdout``. The
amount of indentation added for each recursive level is specified by *indent*;
the default is one. Other values can cause output to look a little odd, but can
make nesting easier to spot. The number of levels which may be printed is
@@ -45,7 +45,12 @@ The :mod:`pprint` module defines one class:
the depth of the objects being formatted. The desired output width is
constrained using the *width* parameter; the default is 80 characters. If a
structure cannot be formatted within the constrained width, a best effort will
- be made.
+ be made. If *compact* is false (the default) each item of a long sequence
+ will be formatted on a separate line. If *compact* is true, as many items
+ as will fit within the *width* will be formatted on each output line.
+
+ .. versionchanged:: 3.4
+ Added the *compact* parameter.
>>> import pprint
>>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni']
@@ -58,6 +63,12 @@ The :mod:`pprint` module defines one class:
'lumberjack',
'knights',
'ni']
+ >>> pp = pprint.PrettyPrinter(width=41, compact=True)
+ >>> pp.pprint(stuff)
+ [['spam', 'eggs', 'lumberjack',
+ 'knights', 'ni'],
+ 'spam', 'eggs', 'lumberjack', 'knights',
+ 'ni']
>>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead',
... ('parrot', ('fresh fruit',))))))))
>>> pp = pprint.PrettyPrinter(depth=6)
@@ -65,23 +76,30 @@ The :mod:`pprint` module defines one class:
('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...)))))))
-The :class:`PrettyPrinter` class supports several derivative functions:
+The :mod:`pprint` module also provides several shortcut functions:
-.. function:: pformat(object, indent=1, width=80, depth=None)
+.. function:: pformat(object, indent=1, width=80, depth=None, *, compact=False)
- Return the formatted representation of *object* as a string. *indent*, *width*
- and *depth* will be passed to the :class:`PrettyPrinter` constructor as
- formatting parameters.
+ Return the formatted representation of *object* as a string. *indent*,
+ *width*, *depth* and *compact* will be passed to the :class:`PrettyPrinter`
+ constructor as formatting parameters.
+ .. versionchanged:: 3.4
+ Added the *compact* parameter.
-.. function:: pprint(object, stream=None, indent=1, width=80, depth=None)
+
+.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \
+ compact=False)
Prints the formatted representation of *object* on *stream*, followed by a
newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used
in the interactive interpreter instead of the :func:`print` function for
inspecting values (you can even reassign ``print = pprint.pprint`` for use
- within a scope). *indent*, *width* and *depth* will be passed to the
- :class:`PrettyPrinter` constructor as formatting parameters.
+ within a scope). *indent*, *width*, *depth* and *compact* will be passed
+ to the :class:`PrettyPrinter` constructor as formatting parameters.
+
+ .. versionchanged:: 3.4
+ Added the *compact* parameter.
>>> import pprint
>>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni']
@@ -193,101 +211,141 @@ Example
-------
To demonstrate several uses of the :func:`pprint` function and its parameters,
-let's fetch information about a project from PyPI::
+let's fetch information about a project from `PyPI <https://pypi.python.org>`_::
>>> import json
>>> import pprint
>>> from urllib.request import urlopen
- >>> with urlopen('http://pypi.python.org/pypi/configparser/json') as url:
+ >>> with urlopen('http://pypi.python.org/pypi/Twisted/json') as url:
... http_info = url.info()
... raw_data = url.read().decode(http_info.get_content_charset())
>>> project_info = json.loads(raw_data)
- >>> result = {'headers': http_info.items(), 'body': project_info}
In its basic form, :func:`pprint` shows the whole object::
- >>> pprint.pprint(result)
- {'body': {'info': {'_pypi_hidden': False,
- '_pypi_ordering': 12,
- 'classifiers': ['Development Status :: 4 - Beta',
- 'Intended Audience :: Developers',
- 'License :: OSI Approved :: MIT License',
- 'Natural Language :: English',
- 'Operating System :: OS Independent',
- 'Programming Language :: Python',
- 'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.6',
- 'Programming Language :: Python :: 2.7',
- 'Topic :: Software Development :: Libraries',
- 'Topic :: Software Development :: Libraries :: Python Modules'],
- 'download_url': 'UNKNOWN',
- 'home_page': 'http://docs.python.org/py3k/library/configparser.html',
- 'keywords': 'configparser ini parsing conf cfg configuration file',
- 'license': 'MIT',
- 'name': 'configparser',
- 'package_url': 'http://pypi.python.org/pypi/configparser',
- 'platform': 'any',
- 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3',
- 'requires_python': None,
- 'stable_version': None,
- 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.',
- 'version': '3.2.0r3'},
- 'urls': [{'comment_text': '',
- 'downloads': 47,
- 'filename': 'configparser-3.2.0r3.tar.gz',
- 'has_sig': False,
- 'md5_digest': '8500fd87c61ac0de328fc996fce69b96',
- 'packagetype': 'sdist',
- 'python_version': 'source',
- 'size': 32281,
- 'upload_time': '2011-05-10T16:28:50',
- 'url': 'http://pypi.python.org/packages/source/c/configparser/configparser-3.2.0r3.tar.gz'}]},
- 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'),
- ('Server', 'Apache/2.2.16 (Debian)'),
- ('Content-Disposition', 'inline'),
- ('Connection', 'close'),
- ('Transfer-Encoding', 'chunked'),
- ('Content-Type', 'application/json; charset="UTF-8"')]}
+ >>> pprint.pprint(project_info)
+ {'info': {'_pypi_hidden': False,
+ '_pypi_ordering': 125,
+ 'author': 'Glyph Lefkowitz',
+ 'author_email': 'glyph@twistedmatrix.com',
+ 'bugtrack_url': '',
+ 'cheesecake_code_kwalitee_id': None,
+ 'cheesecake_documentation_id': None,
+ 'cheesecake_installability_id': None,
+ 'classifiers': ['Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 2 :: Only'],
+ 'description': 'An extensible framework for Python programming, '
+ 'with special focus\r\n'
+ 'on event-based network programming and '
+ 'multiprotocol integration.',
+ 'docs_url': '',
+ 'download_url': 'UNKNOWN',
+ 'home_page': 'http://twistedmatrix.com/',
+ 'keywords': '',
+ 'license': 'MIT',
+ 'maintainer': '',
+ 'maintainer_email': '',
+ 'name': 'Twisted',
+ 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'platform': 'UNKNOWN',
+ 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'requires_python': None,
+ 'stable_version': None,
+ 'summary': 'An asynchronous networking framework written in Python',
+ 'version': '12.3.0'},
+ 'urls': [{'comment_text': '',
+ 'downloads': 71844,
+ 'filename': 'Twisted-12.3.0.tar.bz2',
+ 'has_sig': False,
+ 'md5_digest': '6e289825f3bf5591cfd670874cc0862d',
+ 'packagetype': 'sdist',
+ 'python_version': 'source',
+ 'size': 2615733,
+ 'upload_time': '2012-12-26T12:47:03',
+ 'url': 'https://pypi.python.org/packages/source/T/Twisted/Twisted-12.3.0.tar.bz2'},
+ {'comment_text': '',
+ 'downloads': 5224,
+ 'filename': 'Twisted-12.3.0.win32-py2.7.msi',
+ 'has_sig': False,
+ 'md5_digest': '6b778f5201b622a5519a2aca1a2fe512',
+ 'packagetype': 'bdist_msi',
+ 'python_version': '2.7',
+ 'size': 2916352,
+ 'upload_time': '2012-12-26T12:48:15',
+ 'url': 'https://pypi.python.org/packages/2.7/T/Twisted/Twisted-12.3.0.win32-py2.7.msi'}]}
The result can be limited to a certain *depth* (ellipsis is used for deeper
contents)::
- >>> pprint.pprint(result, depth=3)
- {'body': {'info': {'_pypi_hidden': False,
- '_pypi_ordering': 12,
- 'classifiers': [...],
- 'download_url': 'UNKNOWN',
- 'home_page': 'http://docs.python.org/py3k/library/configparser.html',
- 'keywords': 'configparser ini parsing conf cfg configuration file',
- 'license': 'MIT',
- 'name': 'configparser',
- 'package_url': 'http://pypi.python.org/pypi/configparser',
- 'platform': 'any',
- 'release_url': 'http://pypi.python.org/pypi/configparser/3.2.0r3',
- 'requires_python': None,
- 'stable_version': None,
- 'summary': 'This library brings the updated configparser from Python 3.2+ to Python 2.6-2.7.',
- 'version': '3.2.0r3'},
- 'urls': [{...}]},
- 'headers': [('Date', 'Sat, 14 May 2011 12:48:52 GMT'),
- ('Server', 'Apache/2.2.16 (Debian)'),
- ('Content-Disposition', 'inline'),
- ('Connection', 'close'),
- ('Transfer-Encoding', 'chunked'),
- ('Content-Type', 'application/json; charset="UTF-8"')]}
-
-Additionally, maximum *width* can be suggested. If a long object cannot be
-split, the specified width will be exceeded::
-
- >>> pprint.pprint(result['headers'], width=30)
- [('Date',
- 'Sat, 14 May 2011 12:48:52 GMT'),
- ('Server',
- 'Apache/2.2.16 (Debian)'),
- ('Content-Disposition',
- 'inline'),
- ('Connection', 'close'),
- ('Transfer-Encoding',
- 'chunked'),
- ('Content-Type',
- 'application/json; charset="UTF-8"')]
+ >>> pprint.pprint(project_info, depth=2)
+ {'info': {'_pypi_hidden': False,
+ '_pypi_ordering': 125,
+ 'author': 'Glyph Lefkowitz',
+ 'author_email': 'glyph@twistedmatrix.com',
+ 'bugtrack_url': '',
+ 'cheesecake_code_kwalitee_id': None,
+ 'cheesecake_documentation_id': None,
+ 'cheesecake_installability_id': None,
+ 'classifiers': [...],
+ 'description': 'An extensible framework for Python programming, '
+ 'with special focus\r\n'
+ 'on event-based network programming and '
+ 'multiprotocol integration.',
+ 'docs_url': '',
+ 'download_url': 'UNKNOWN',
+ 'home_page': 'http://twistedmatrix.com/',
+ 'keywords': '',
+ 'license': 'MIT',
+ 'maintainer': '',
+ 'maintainer_email': '',
+ 'name': 'Twisted',
+ 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'platform': 'UNKNOWN',
+ 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'requires_python': None,
+ 'stable_version': None,
+ 'summary': 'An asynchronous networking framework written in Python',
+ 'version': '12.3.0'},
+ 'urls': [{...}, {...}]}
+
+Additionally, maximum character *width* can be suggested. If a long object
+cannot be split, the specified width will be exceeded::
+
+ >>> pprint.pprint(project_info, depth=2, width=50)
+ {'info': {'_pypi_hidden': False,
+ '_pypi_ordering': 125,
+ 'author': 'Glyph Lefkowitz',
+ 'author_email': 'glyph@twistedmatrix.com',
+ 'bugtrack_url': '',
+ 'cheesecake_code_kwalitee_id': None,
+ 'cheesecake_documentation_id': None,
+ 'cheesecake_installability_id': None,
+ 'classifiers': [...],
+ 'description': 'An extensible '
+ 'framework for '
+ 'Python programming, '
+ 'with special '
+ 'focus\r\n'
+ 'on event-based '
+ 'network programming '
+ 'and multiprotocol '
+ 'integration.',
+ 'docs_url': '',
+ 'download_url': 'UNKNOWN',
+ 'home_page': 'http://twistedmatrix.com/',
+ 'keywords': '',
+ 'license': 'MIT',
+ 'maintainer': '',
+ 'maintainer_email': '',
+ 'name': 'Twisted',
+ 'package_url': 'http://pypi.python.org/pypi/Twisted',
+ 'platform': 'UNKNOWN',
+ 'release_url': 'http://pypi.python.org/pypi/Twisted/12.3.0',
+ 'requires_python': None,
+ 'stable_version': None,
+ 'summary': 'An asynchronous '
+ 'networking framework '
+ 'written in Python',
+ 'version': '12.3.0'},
+ 'urls': [{...}, {...}]}
diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst
index 2b9385b..90baec5 100644
--- a/Doc/library/pty.rst
+++ b/Doc/library/pty.rst
@@ -45,6 +45,9 @@ The :mod:`pty` module defines the following functions:
a file descriptor. The defaults try to read 1024 bytes each time they are
called.
+ .. versionchanged:: 3.4
+ :func:`spawn` now returns the status value from :func:`os.waitpid`
+ on the child process.
Example
-------
diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst
index 07ddc25..bae8450 100644
--- a/Doc/library/py_compile.rst
+++ b/Doc/library/py_compile.rst
@@ -28,7 +28,7 @@ byte-code cache files in the directory containing the source code.
.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1)
- Compile a source file to byte-code and write out the byte-code cache file.
+ Compile a source file to byte-code and write out the byte-code cache file.
The source code is loaded from the file name *file*. The byte-code is
written to *cfile*, which defaults to the :PEP:`3147` path, ending in
``.pyc`` (``.pyo`` if optimization is enabled in the current interpreter).
@@ -41,6 +41,13 @@ byte-code cache files in the directory containing the source code.
is raised. This function returns the path to byte-compiled file, i.e.
whatever *cfile* value was used.
+ If the path that *cfile* becomes (either explicitly specified or computed)
+ is a symlink or non-regular file, :exc:`FileExistsError` will be raised.
+ This is to act as a warning that import will turn those paths into regular
+ files if it is allowed to write byte-compiled files to those paths. This is
+ a side-effect of import using file renaming to place the final byte-compiled
+ file into place to prevent concurrent file writing issues.
+
*optimize* controls the optimization level and is passed to the built-in
:func:`compile` function. The default of ``-1`` selects the optimization
level of the current interpreter.
@@ -50,6 +57,13 @@ byte-code cache files in the directory containing the source code.
default was *file* + ``'c'`` (``'o'`` if optimization was enabled).
Also added the *optimize* parameter.
+ .. versionchanged:: 3.4
+ Changed code to use :mod:`importlib` for the byte-code cache file writing.
+ This means file creation/writing semantics now match what :mod:`importlib`
+ does, e.g. permissions, write-and-move semantics, etc. Also added the
+ caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or
+ non-regular file.
+
.. function:: main(args=None)
diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst
index e100865..3f520e8 100644
--- a/Doc/library/pydoc.rst
+++ b/Doc/library/pydoc.rst
@@ -84,3 +84,8 @@ Reference Manual pages.
.. versionchanged:: 3.2
Added the ``-b`` option, deprecated the ``-g`` option.
+
+.. versionchanged:: 3.4
+ :mod:`pydoc` now uses :func:`inspect.signature` rather than
+ :func:`inspect.getfullargspec` to extract signature information from
+ callables.
diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst
index 3d88d85..cb8ab65 100644
--- a/Doc/library/pyexpat.rst
+++ b/Doc/library/pyexpat.rst
@@ -100,6 +100,11 @@ The :mod:`xml.parsers.expat` module contains two functions:
http://www.python.org/ns/ elem1
elem2
+ Due to limitations in the ``Expat`` library used by :mod:`pyexpat`,
+ the :class:`xmlparser` instance returned can only be used to parse a single
+ XML document. Call ``ParserCreate`` for each document to provide unique
+ parser instances.
+
.. seealso::
@@ -119,7 +124,9 @@ XMLParser Objects
Parses the contents of the string *data*, calling the appropriate handler
functions to process the parsed data. *isfinal* must be true on the final call
- to this method. *data* can be the empty string at any time.
+ to this method; it allows the parsing of a single file in fragments,
+ not the submission of multiple files.
+ *data* can be the empty string at any time.
.. method:: xmlparser.ParseFile(file)
diff --git a/Doc/library/python.rst b/Doc/library/python.rst
index b67fbfc..f307d7d 100644
--- a/Doc/library/python.rst
+++ b/Doc/library/python.rst
@@ -25,4 +25,3 @@ overview:
inspect.rst
site.rst
fpectl.rst
- distutils.rst
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index dd790e7..1fa1cc0 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -481,7 +481,7 @@ form.
.. note::
The compiled versions of the most recent patterns passed to
- :func:`re.match`, :func:`re.search` or :func:`re.compile` are cached, so
+ :func:`re.compile` and the module-level matching functions are cached, so
programs that use only a few regular expressions at a time needn't worry
about compiling regular expressions.
@@ -584,6 +584,16 @@ form.
instead (see also :ref:`search-vs-match`).
+.. function:: fullmatch(pattern, string, flags=0)
+
+ If the whole *string* matches the regular expression *pattern*, return a
+ corresponding :ref:`match object <match-objects>`. Return ``None`` if the
+ string does not match the pattern; note that this is different from a
+ zero-length match.
+
+ .. versionadded:: 3.4
+
+
.. function:: split(pattern, string, maxsplit=0, flags=0)
Split *string* by the occurrences of *pattern*. If capturing parentheses are
@@ -755,7 +765,7 @@ attributes:
>>> pattern = re.compile("d")
>>> pattern.search("dog") # Match at index 0
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 1), match='d'>
>>> pattern.search("dog", 1) # No match; search doesn't include the "d"
@@ -772,12 +782,30 @@ attributes:
>>> pattern = re.compile("o")
>>> pattern.match("dog") # No match as "o" is not at the start of "dog".
>>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog".
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(1, 2), match='o'>
If you want to locate a match anywhere in *string*, use
:meth:`~regex.search` instead (see also :ref:`search-vs-match`).
+.. method:: regex.fullmatch(string[, pos[, endpos]])
+
+ If the whole *string* matches this regular expression, return a corresponding
+ :ref:`match object <match-objects>`. Return ``None`` if the string does not
+ match the pattern; note that this is different from a zero-length match.
+
+ The optional *pos* and *endpos* parameters have the same meaning as for the
+ :meth:`~regex.search` method.
+
+ >>> pattern = re.compile("o[gh]")
+ >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog".
+ >>> pattern.fullmatch("ogre") # No match as not the full string matches.
+ >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits.
+ <_sre.SRE_Match object; span=(1, 3), match='og'>
+
+ .. versionadded:: 3.4
+
+
.. method:: regex.split(string, maxsplit=0)
Identical to the :func:`split` function, using the compiled pattern.
@@ -1139,7 +1167,7 @@ For example::
>>> re.match("c", "abcdef") # No match
>>> re.search("c", "abcdef") # Match
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(2, 3), match='c'>
Regular expressions beginning with ``'^'`` can be used with :func:`search` to
restrict the match at the beginning of the string::
@@ -1147,7 +1175,7 @@ restrict the match at the beginning of the string::
>>> re.match("c", "abcdef") # No match
>>> re.search("^c", "abcdef") # No match
>>> re.search("^a", "abcdef") # Match
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 1), match='a'>
Note however that in :const:`MULTILINE` mode :func:`match` only matches at the
beginning of the string, whereas using :func:`search` with a regular expression
@@ -1155,7 +1183,7 @@ beginning with ``'^'`` will match at the beginning of each line.
>>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match
>>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(4, 5), match='X'>
Making a Phonebook
@@ -1274,9 +1302,9 @@ another one to escape it. For example, the two following lines of code are
functionally identical:
>>> re.match(r"\W(.)\1\W", " ff ")
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 4), match=' ff '>
>>> re.match("\\W(.)\\1\\W", " ff ")
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 4), match=' ff '>
When one wants to match a literal backslash, it must be escaped in the regular
expression. With raw string notation, this means ``r"\\"``. Without raw string
@@ -1284,9 +1312,9 @@ notation, one must use ``"\\\\"``, making the following lines of code
functionally identical:
>>> re.match(r"\\", r"\\")
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 1), match='\\'>
>>> re.match("\\\\", r"\\")
- <_sre.SRE_Match object at ...>
+ <_sre.SRE_Match object; span=(0, 1), match='\\'>
Writing a Tokenizer
diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst
index 1134619..692310b 100644
--- a/Doc/library/readline.rst
+++ b/Doc/library/readline.rst
@@ -190,28 +190,32 @@ Example
The following example demonstrates how to use the :mod:`readline` module's
history reading and writing functions to automatically load and save a history
-file named :file:`.pyhist` from the user's home directory. The code below would
-normally be executed automatically during interactive sessions from the user's
-:envvar:`PYTHONSTARTUP` file. ::
+file named :file:`.python_history` from the user's home directory. The code
+below would normally be executed automatically during interactive sessions
+from the user's :envvar:`PYTHONSTARTUP` file. ::
+ import atexit
import os
import readline
- histfile = os.path.join(os.path.expanduser("~"), ".pyhist")
+
+ histfile = os.path.join(os.path.expanduser("~"), ".python_history")
try:
readline.read_history_file(histfile)
except FileNotFoundError:
pass
- import atexit
+
atexit.register(readline.write_history_file, histfile)
- del os, histfile
+
+This code is actually automatically run when Python is run in
+:ref:`interactive mode <tut-interactive>` (see :ref:`rlcompleter-config`).
The following example extends the :class:`code.InteractiveConsole` class to
support history save/restore. ::
- import code
- import readline
import atexit
+ import code
import os
+ import readline
class HistoryConsole(code.InteractiveConsole):
def __init__(self, locals=None, filename="<console>",
diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst
index ed85850..f8112cc 100644
--- a/Doc/library/resource.rst
+++ b/Doc/library/resource.rst
@@ -74,6 +74,27 @@ this module for those platforms.
``setrlimit`` may also raise :exc:`error` if the underlying system call
fails.
+.. function:: prlimit(pid, resource[, limits])
+
+ Combines :func:`setrlimit` and :func:`getrlimit` in one function and
+ supports to get and set the resources limits of an arbitrary process. If
+ *pid* is 0, then the call applies to the current process. *resource* and
+ *limits* have the same meaning as in :func:`setrlimit`, except that
+ *limits* is optional.
+
+ When *limits* is not given the function returns the *resource* limit of the
+ process *pid*. When *limits* is given the *resource* limit of the process is
+ set and the former resource limit is returned.
+
+ Raises :exc:`ProcessLookupError` when *pid* can't be found and
+ :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for
+ the process.
+
+ Availability: Linux 2.6.36 or later with glibc 2.13 or later
+
+ .. versionadded:: 3.4
+
+
These symbols define resources whose consumption can be controlled using the
:func:`setrlimit` and :func:`getrlimit` functions described below. The values of
these symbols are exactly the constants used by C programs.
@@ -151,6 +172,80 @@ platform.
The maximum area (in bytes) of address space which may be taken by the process.
+.. data:: RLIMIT_MSGQUEUE
+
+ The number of bytes that can be allocated for POSIX message queues.
+
+ Availability: Linux 2.6.8 or later.
+
+ .. versionadded:: 3.4
+
+
+.. data:: RLIMIT_NICE
+
+ The ceiling for the process's nice level (calculated as 20 - rlim_cur).
+
+ Availability: Linux 2.6.12 or later.
+
+ .. versionadded:: 3.4
+
+
+.. data:: RLIMIT_RTPRIO
+
+ The ceiling of the real-time priority.
+
+ Availability: Linux 2.6.12 or later.
+
+ .. versionadded:: 3.4
+
+
+.. data:: RLIMIT_RTTIME
+
+ The time limit (in microseconds) on CPU time that a process can spend
+ under real-time scheduling without making a blocking syscall.
+
+ Availability: Linux 2.6.25 or later.
+
+ .. versionadded:: 3.4
+
+
+.. data:: RLIMIT_SIGPENDING
+
+ The number of signals which the process may queue.
+
+ Availability: Linux 2.6.8 or later.
+
+ .. versionadded:: 3.4
+
+.. data:: RLIMIT_SBSIZE
+
+ The maximum size (in bytes) of socket buffer usage for this user.
+ This limits the amount of network memory, and hence the amount of mbufs,
+ that this user may hold at any time.
+
+ Availability: FreeBSD 9 or later.
+
+ .. versionadded:: 3.4
+
+.. data:: RLIMIT_SWAP
+
+ The maximum size (in bytes) of the swap space that may be reserved or
+ used by all of this user id's processes.
+ This limit is enforced only if bit 1 of the vm.overcommit sysctl is set.
+ Please see :manpage:`tuning(7)` for a complete description of this sysctl.
+
+ Availability: FreeBSD 9 or later.
+
+ .. versionadded:: 3.4
+
+.. data:: RLIMIT_NPTS
+
+ The maximum number of pseudo-terminals created by this user id.
+
+ Availability: FreeBSD 9 or later.
+
+ .. versionadded:: 3.4
+
Resource Usage
--------------
diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst
index 633088d..9ed01c7 100644
--- a/Doc/library/rlcompleter.rst
+++ b/Doc/library/rlcompleter.rst
@@ -27,18 +27,10 @@ Example::
readline.__name__ readline.parse_and_bind(
>>> readline.
-The :mod:`rlcompleter` module is designed for use with Python's interactive
-mode. A user can add the following lines to his or her initialization file
-(identified by the :envvar:`PYTHONSTARTUP` environment variable) to get
-automatic :kbd:`Tab` completion::
-
- try:
- import readline
- except ImportError:
- print("Module readline not available.")
- else:
- import rlcompleter
- readline.parse_and_bind("tab: complete")
+The :mod:`rlcompleter` module is designed for use with Python's
+:ref:`interactive mode <tut-interactive>`. Unless Python is run with the
+:option:`-S` option, the module is automatically imported and configured
+(see :ref:`rlcompleter-config`).
On platforms without :mod:`readline`, the :class:`Completer` class defined by
this module can still be used for custom purposes.
diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst
index 6919bc0..ee9fbcf 100644
--- a/Doc/library/runpy.rst
+++ b/Doc/library/runpy.rst
@@ -44,28 +44,22 @@ The :mod:`runpy` module provides two functions:
below are defined in the supplied dictionary, those definitions are
overridden by :func:`run_module`.
- The special global variables ``__name__``, ``__file__``, ``__cached__``,
- ``__loader__``
- and ``__package__`` are set in the globals dictionary before the module
- code is executed (Note that this is a minimal set of variables - other
- variables may be set implicitly as an interpreter implementation detail).
+ The special global variables ``__name__``, ``__spec__``, ``__file__``,
+ ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+ dictionary before the module code is executed (Note that this is a
+ minimal set of variables - other variables may be set implicitly as an
+ interpreter implementation detail).
``__name__`` is set to *run_name* if this optional argument is not
:const:`None`, to ``mod_name + '.__main__'`` if the named module is a
package and to the *mod_name* argument otherwise.
- ``__file__`` is set to the name provided by the module loader. If the
- loader does not make filename information available, this variable is set
- to :const:`None`.
+ ``__spec__`` will be set appropriately for the *actually* imported
+ module (that is, ``__spec__.name`` will always be *mod_name* or
+ ``mod_name + '.__main__``, never *run_name*).
- ``__cached__`` will be set to ``None``.
-
- ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
- code for the module (This loader may be a wrapper around the standard
- import mechanism).
-
- ``__package__`` is set to *mod_name* if the named module is a package and
- to ``mod_name.rpartition('.')[0]`` otherwise.
+ ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are
+ :ref:`set as normal <import-mod-attrs>` based on the module spec.
If the argument *alter_sys* is supplied and evaluates to :const:`True`,
then ``sys.argv[0]`` is updated with the value of ``__file__`` and
@@ -83,8 +77,13 @@ The :mod:`runpy` module provides two functions:
Added ability to execute packages by looking for a ``__main__`` submodule.
.. versionchanged:: 3.2
- Added ``__cached__`` global variable (see :PEP:`3147`).
+ Added ``__cached__`` global variable (see :pep:`3147`).
+ .. versionchanged:: 3.4
+ Updated to take advantage of the module spec feature added by
+ :pep:`451`. This allows ``__cached__`` to be set correctly for modules
+ run this way, as well as ensuring the real module name is always
+ accessible as ``__spec__.name``.
.. function:: run_path(file_path, init_globals=None, run_name=None)
@@ -108,23 +107,28 @@ The :mod:`runpy` module provides two functions:
below are defined in the supplied dictionary, those definitions are
overridden by :func:`run_path`.
- The special global variables ``__name__``, ``__file__``, ``__loader__``
- and ``__package__`` are set in the globals dictionary before the module
- code is executed (Note that this is a minimal set of variables - other
- variables may be set implicitly as an interpreter implementation detail).
+ The special global variables ``__name__``, ``__spec__``, ``__file__``,
+ ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
+ dictionary before the module code is executed (Note that this is a
+ minimal set of variables - other variables may be set implicitly as an
+ interpreter implementation detail).
``__name__`` is set to *run_name* if this optional argument is not
:const:`None` and to ``'<run_path>'`` otherwise.
- ``__file__`` is set to the name provided by the module loader. If the
- loader does not make filename information available, this variable is set
- to :const:`None`. For a simple script, this will be set to ``file_path``.
+ If the supplied path directly references a script file (whether as source
+ or as precompiled byte code), then ``__file__`` will be set to the
+ supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and
+ ``__package__`` will all be set to :const:`None`.
- ``__loader__`` is set to the :pep:`302` module loader used to retrieve the
- code for the module (This loader may be a wrapper around the standard
- import mechanism). For a simple script, this will be set to :const:`None`.
+ ``__spec__`` will be set to :const:`None` if the supplied path is a
+ direct path to a script (as source or as precompiled bytecode).
- ``__package__`` is set to ``__name__.rpartition('.')[0]``.
+ If the supplied path is a reference to a valid sys.path entry, then
+ ``__spec__`` will be set appropriately for the imported ``__main__``
+ module (that is, ``__spec__.name`` will always be ``__main__``).
+ ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be
+ :ref:`set as normal <import-mod-attrs>` based on the module spec.
A number of alterations are also made to the :mod:`sys` module. Firstly,
``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated
@@ -141,6 +145,12 @@ The :mod:`runpy` module provides two functions:
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ Updated to take advantage of the module spec feature added by
+ :pep:`451`. This allows ``__cached__`` to be set correctly in the
+ case where ``__main__`` is imported from a valid sys.path entry rather
+ than being executed directly.
+
.. seealso::
:pep:`338` - Executing modules as scripts
@@ -149,6 +159,9 @@ The :mod:`runpy` module provides two functions:
:pep:`366` - Main module explicit relative imports
PEP written and implemented by Nick Coghlan.
+ :pep:`451` - A ModuleSpec Type for the Import System
+ PEP written and implemented by Eric Snow
+
:ref:`using-on-general` - CPython command line details
The :func:`importlib.import_module` function
diff --git a/Doc/library/select.rst b/Doc/library/select.rst
index b1fd3a8..4fae87e 100644
--- a/Doc/library/select.rst
+++ b/Doc/library/select.rst
@@ -14,6 +14,14 @@ it also works for other file types (in particular, on Unix, it works on pipes).
It cannot be used on regular files to determine whether a file has grown since
it was last read.
+.. note::
+
+ The :mod:`selectors` module allows high-level and efficient I/O
+ multiplexing, built upon the :mod:`select` module primitives. Users are
+ encouraged to use the :mod:`selectors` module instead, unless they want
+ precise control over the OS-level primitives used.
+
+
The module defines the following:
@@ -37,21 +45,37 @@ The module defines the following:
increases this value, :c:func:`devpoll` may return an
incomplete list of active file descriptors.
+ The new file descriptor is :ref:`non-inheritable <fd_inheritance>`.
+
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ The new file descriptor is now non-inheritable.
+
.. function:: epoll(sizehint=-1, flags=0)
(Only supported on Linux 2.5.44 and newer.) Return an edge polling object,
which can be used as Edge or Level Triggered interface for I/O
events. *sizehint* is deprecated and completely ignored. *flags* can be set
to :const:`EPOLL_CLOEXEC`, which causes the epoll descriptor to be closed
- automatically when :func:`os.execve` is called. See section
- :ref:`epoll-objects` below for the methods supported by epolling objects.
+ automatically when :func:`os.execve` is called.
+
+ See the :ref:`epoll-objects` section below for the methods supported by
+ epolling objects.
+
+ ``epoll`` objects support the context management protocol: when used in a
+ :keyword:`with` statement, the new file descriptor is automatically closed
+ at the end of the block.
+ The new file descriptor is :ref:`non-inheritable <fd_inheritance>`.
.. versionchanged:: 3.3
Added the *flags* parameter.
+ .. versionchanged:: 3.4
+ Support for the :keyword:`with` statement was added.
+ The new file descriptor is now non-inheritable.
+
.. function:: poll()
@@ -66,6 +90,11 @@ The module defines the following:
(Only supported on BSD.) Returns a kernel queue object; see section
:ref:`kqueue-objects` below for the methods supported by kqueue objects.
+ The new file descriptor is :ref:`non-inheritable <fd_inheritance>`.
+
+ .. versionchanged:: 3.4
+ The new file descriptor is now non-inheritable.
+
.. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0)
@@ -144,6 +173,27 @@ descriptors), ``/dev/poll`` is O(active file descriptors).
object.
+.. method:: devpoll.close()
+
+ Close the file descriptor of the polling object.
+
+ .. versionadded:: 3.4
+
+
+.. attribute:: devpoll.closed
+
+ ``True`` if the polling object is closed.
+
+ .. versionadded:: 3.4
+
+
+.. method:: devpoll.fileno()
+
+ Return the file descriptor number of the polling object.
+
+ .. versionadded:: 3.4
+
+
.. method:: devpoll.register(fd[, eventmask])
Register a file descriptor with the polling object. Future calls to the
@@ -241,6 +291,11 @@ Edge and Level Trigger Polling (epoll) Objects
Close the control file descriptor of the epoll object.
+.. attribute:: epoll.closed
+
+ ``True`` if the epoll object is closed.
+
+
.. method:: epoll.fileno()
Return the file descriptor number of the control fd.
@@ -258,7 +313,7 @@ Edge and Level Trigger Polling (epoll) Objects
.. method:: epoll.modify(fd, eventmask)
- Modify a register file descriptor.
+ Modify a registered file descriptor.
.. method:: epoll.unregister(fd)
@@ -360,6 +415,11 @@ Kqueue Objects
Close the control file descriptor of the kqueue object.
+.. attribute:: kqueue.closed
+
+ ``True`` if the kqueue object is closed.
+
+
.. method:: kqueue.fileno()
Return the file descriptor number of the control fd.
diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst
new file mode 100644
index 0000000..98377c8
--- /dev/null
+++ b/Doc/library/selectors.rst
@@ -0,0 +1,257 @@
+:mod:`selectors` -- High-level I/O multiplexing
+===============================================
+
+.. module:: selectors
+ :synopsis: High-level I/O multiplexing.
+
+.. versionadded:: 3.4
+
+
+Introduction
+------------
+
+This module allows high-level and efficient I/O multiplexing, built upon the
+:mod:`select` module primitives. Users are encouraged to use this module
+instead, unless they want precise control over the OS-level primitives used.
+
+It defines a :class:`BaseSelector` abstract base class, along with several
+concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...),
+that can be used to wait for I/O readiness notification on multiple file
+objects. In the following, "file object" refers to any object with a
+:meth:`fileno()` method, or a raw file descriptor. See :term:`file object`.
+
+:class:`DefaultSelector` is an alias to the most efficient implementation
+available on the current platform: this should be the default choice for most
+users.
+
+.. note::
+ The type of file objects supported depends on the platform: on Windows,
+ sockets are supported, but not pipes, whereas on Unix, both are supported
+ (some other types may be supported as well, such as fifos or special file
+ devices).
+
+.. seealso::
+
+ :mod:`select`
+ Low-level I/O multiplexing module.
+
+
+Classes
+-------
+
+Classes hierarchy::
+
+ BaseSelector
+ +-- SelectSelector
+ +-- PollSelector
+ +-- EpollSelector
+ +-- KqueueSelector
+
+
+In the following, *events* is a bitwise mask indicating which I/O events should
+be waited for on a given file object. It can be a combination of the constants
+below:
+
+ +-----------------------+-----------------------------------------------+
+ | Constant | Meaning |
+ +=======================+===============================================+
+ | :const:`EVENT_READ` | Available for read |
+ +-----------------------+-----------------------------------------------+
+ | :const:`EVENT_WRITE` | Available for write |
+ +-----------------------+-----------------------------------------------+
+
+
+.. class:: SelectorKey
+
+ A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to
+ associate a file object to its underlying file decriptor, selected event
+ mask and attached data. It is returned by several :class:`BaseSelector`
+ methods.
+
+ .. attribute:: fileobj
+
+ File object registered.
+
+ .. attribute:: fd
+
+ Underlying file descriptor.
+
+ .. attribute:: events
+
+ Events that must be waited for on this file object.
+
+ .. attribute:: data
+
+ Optional opaque data associated to this file object: for example, this
+ could be used to store a per-client session ID.
+
+
+.. class:: BaseSelector
+
+ A :class:`BaseSelector` is used to wait for I/O event readiness on multiple
+ file objects. It supports file stream registration, unregistration, and a
+ method to wait for I/O events on those streams, with an optional timeout.
+ It's an abstract base class, so cannot be instantiated. Use
+ :class:`DefaultSelector` instead, or one of :class:`SelectSelector`,
+ :class:`KqueueSelector` etc. if you want to specifically use an
+ implementation, and your platform supports it.
+ :class:`BaseSelector` and its concrete implementations support the
+ :term:`context manager` protocol.
+
+ .. method:: register(fileobj, events, data=None)
+
+ Register a file object for selection, monitoring it for I/O events.
+
+ *fileobj* is the file object to monitor. It may either be an integer
+ file descriptor or an object with a ``fileno()`` method.
+ *events* is a bitwise mask of events to monitor.
+ *data* is an opaque object.
+
+ This returns a new :class:`SelectorKey` instance, or raises a
+ :exc:`ValueError` in case of invalid event mask or file descriptor, or
+ :exc:`KeyError` if the file object is already registered.
+
+ .. method:: unregister(fileobj)
+
+ Unregister a file object from selection, removing it from monitoring. A
+ file object shall be unregistered prior to being closed.
+
+ *fileobj* must be a file object previously registered.
+
+ This returns the associated :class:`SelectorKey` instance, or raises a
+ :exc:`KeyError` if *fileobj* is not registered. It will raise
+ :exc:`ValueError` if *fileobj* is invalid (e.g. it has no ``fileno()``
+ method or its ``fileno()`` method has an invalid return value).
+
+ .. method:: modify(fileobj, events, data=None)
+
+ Change a registered file object's monitored events or attached data.
+
+ This is equivalent to :meth:`BaseSelector.unregister(fileobj)` followed
+ by :meth:`BaseSelector.register(fileobj, events, data)`, except that it
+ can be implemented more efficiently.
+
+ This returns a new :class:`SelectorKey` instance, or raises a
+ :exc:`ValueError` in case of invalid event mask or file descriptor, or
+ :exc:`KeyError` if the file object is not registered.
+
+ .. method:: select(timeout=None)
+
+ Wait until some registered file objects become ready, or the timeout
+ expires.
+
+ If ``timeout > 0``, this specifies the maximum wait time, in seconds.
+ If ``timeout <= 0``, the call won't block, and will report the currently
+ ready file objects.
+ If *timeout* is ``None``, the call will block until a monitored file object
+ becomes ready.
+
+ This returns a list of ``(key, events)`` tuples, one for each ready file
+ object.
+
+ *key* is the :class:`SelectorKey` instance corresponding to a ready file
+ object.
+ *events* is a bitmask of events ready on this file object.
+
+ .. note::
+ This method can return before any file object becomes ready or the
+ timeout has elapsed if the current process receives a signal: in this
+ case, an empty list will be returned.
+
+ .. method:: close()
+
+ Close the selector.
+
+ This must be called to make sure that any underlying resource is freed.
+ The selector shall not be used once it has been closed.
+
+ .. method:: get_key(fileobj)
+
+ Return the key associated with a registered file object.
+
+ This returns the :class:`SelectorKey` instance associated to this file
+ object, or raises :exc:`KeyError` if the file object is not registered.
+
+ .. method:: get_map()
+
+ Return a mapping of file objects to selector keys.
+
+ This returns a :class:`~collections.abc.Mapping` instance mapping
+ registered file objects to their associated :class:`SelectorKey`
+ instance.
+
+
+.. class:: DefaultSelector()
+
+ The default selector class, using the most efficient implementation
+ available on the current platform. This should be the default choice for
+ most users.
+
+
+.. class:: SelectSelector()
+
+ :func:`select.select`-based selector.
+
+
+.. class:: PollSelector()
+
+ :func:`select.poll`-based selector.
+
+
+.. class:: EpollSelector()
+
+ :func:`select.epoll`-based selector.
+
+ .. method:: fileno()
+
+ This returns the file descriptor used by the underlying
+ :func:`select.epoll` object.
+
+
+.. class:: KqueueSelector()
+
+ :func:`select.kqueue`-based selector.
+
+ .. method:: fileno()
+
+ This returns the file descriptor used by the underlying
+ :func:`select.kqueue` object.
+
+
+Examples
+--------
+
+Here is a simple echo server implementation::
+
+ import selectors
+ import socket
+
+ sel = selectors.DefaultSelector()
+
+ def accept(sock, mask):
+ conn, addr = sock.accept() # Should be ready
+ print('accepted', conn, 'from', addr)
+ conn.setblocking(False)
+ sel.register(conn, selectors.EVENT_READ, read)
+
+ def read(conn, mask):
+ data = conn.recv(1000) # Should be ready
+ if data:
+ print('echoing', repr(data), 'to', conn)
+ conn.send(data) # Hope it won't block
+ else:
+ print('closing', conn)
+ sel.unregister(conn)
+ conn.close()
+
+ sock = socket.socket()
+ sock.bind(('localhost', 1234))
+ sock.listen(100)
+ sock.setblocking(False)
+ sel.register(sock, selectors.EVENT_READ, accept)
+
+ while True:
+ events = sel.select()
+ for key, mask in events:
+ callback = key.data
+ callback(key.fileobj, mask)
diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst
index 048f80a..22e202d 100644
--- a/Doc/library/shelve.rst
+++ b/Doc/library/shelve.rst
@@ -44,8 +44,11 @@ lots of shared sub-objects. The keys are ordinary strings.
.. note::
Do not rely on the shelf being closed automatically; always call
- :meth:`close` explicitly when you don't need it any more, or use a
- :keyword:`with` statement with :func:`contextlib.closing`.
+ :meth:`~Shelf.close` explicitly when you don't need it any more, or
+ use :func:`shelve.open` as a context manager::
+
+ with shelve.open('spam') as db:
+ db['eggs'] = 'eggs'
.. warning::
@@ -118,10 +121,16 @@ Restrictions
The *keyencoding* parameter is the encoding used to encode keys before they
are used with the underlying dict.
- .. versionadded:: 3.2
- The *keyencoding* parameter; previously, keys were always encoded in
+ A :class:`Shelf` object can also be used as a context manager, in which
+ case it will be automatically closed when the :keyword:`with` block ends.
+
+ .. versionchanged:: 3.2
+ Added the *keyencoding* parameter; previously, keys were always encoded in
UTF-8.
+ .. versionchanged:: 3.4
+ Added context manager support.
+
.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8')
diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index a1457d0..e4f348c 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -53,7 +53,7 @@ Directory and files operations
*dst* and return *dst*. *src* and *dst* are path names given as strings.
*dst* must be the complete target file name; look at :func:`shutil.copy`
for a copy that accepts a target directory path. If *src* and *dst*
- specify the same file, :exc:`Error` is raised.
+ specify the same file, :exc:`SameFileError` is raised.
The destination location must be writable; otherwise, an :exc:`OSError`
exception will be raised. If *dst* already exists, it will be replaced.
@@ -69,6 +69,19 @@ Directory and files operations
Added *follow_symlinks* argument.
Now returns *dst*.
+ .. versionchanged:: 3.4
+ Raise :exc:`SameFileError` instead of :exc:`Error`. Since the former is
+ a subclass of the latter, this change is backward compatible.
+
+
+.. exception:: SameFileError
+
+ This exception is raised if source and destination in :func:`copyfile`
+ are the same file.
+
+ .. versionadded:: 3.4
+
+
.. function:: copymode(src, dst, *, follow_symlinks=True)
Copy the permission bits from *src* to *dst*. The file contents, owner, and
@@ -380,11 +393,10 @@ provided by this module. ::
errors.extend(err.args[0])
try:
copystat(src, dst)
- except WindowsError:
- # can't copy file access times on Windows
- pass
except OSError as why:
- errors.extend((src, dst, str(why)))
+ # can't copy file access times on Windows
+ if why.winerror is None:
+ errors.extend((src, dst, str(why)))
if errors:
raise Error(errors)
diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index 36b80c3..2fdf303 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -38,6 +38,9 @@ Unix and Macintosh). For each of the distinct head-tail combinations, it sees
if it refers to an existing directory, and if so, adds it to ``sys.path`` and
also inspects the newly added path for configuration files.
+.. deprecated:: 3.4
+ Support for the "site-python" directory will be removed in 3.5.
+
If a file named "pyvenv.cfg" exists one directory above sys.executable,
sys.prefix and sys.exec_prefix are set to that directory and
it is also checked for site-packages and site-python (sys.base_prefix and
@@ -111,6 +114,27 @@ empty, and the path manipulations are skipped; however the import of
:mod:`sitecustomize` and :mod:`usercustomize` is still attempted.
+.. _rlcompleter-config:
+
+Readline configuration
+----------------------
+
+On systems that support :mod:`readline`, this module will also import and
+configure the :mod:`rlcompleter` module, if Python is started in
+:ref:`interactive mode <tut-interactive>` and without the :option:`-S` option.
+The default behavior is enable tab-completion and to use
+:file:`~/.python_history` as the history save file. To disable it, delete (or
+override) the :data:`sys.__interactivehook__` attribute in your
+:mod:`sitecustomize` or :mod:`usercustomize` module or your
+:envvar:`PYTHONSTARTUP` file.
+
+.. versionchanged:: 3.4
+ Activation of rlcompleter and history was made automatic.
+
+
+Module contents
+---------------
+
.. data:: PREFIXES
A list of prefixes for site-packages directories.
@@ -153,8 +177,7 @@ empty, and the path manipulations are skipped; however the import of
Adds all the standard site-specific directories to the module search
path. This function is called automatically when this module is imported,
- unless the :program:`python` interpreter was started with the :option:`-S`
- flag.
+ unless the Python interpreter was started with the :option:`-S` flag.
.. versionchanged:: 3.3
This function used to be called unconditionnally.
diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst
index f305818..3ebed06 100644
--- a/Doc/library/smtpd.rst
+++ b/Doc/library/smtpd.rst
@@ -27,7 +27,8 @@ SMTPServer Objects
------------------
-.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432)
+.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\
+ map=None)
Create a new :class:`SMTPServer` object, which binds to local address
*localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. It
@@ -38,6 +39,8 @@ SMTPServer Objects
accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no
limit.
+ A dictionary can be specified in *map* to avoid using a global socket map.
+
.. method:: process_message(peer, mailfrom, rcpttos, data)
Raise :exc:`NotImplementedError` exception. Override this in subclasses to
@@ -53,6 +56,9 @@ SMTPServer Objects
Override this in subclasses to use a custom :class:`SMTPChannel` for
managing SMTP clients.
+ .. versionchanged:: 3.4
+ The *map* argument was added.
+
DebuggingServer Objects
-----------------------
@@ -90,11 +96,20 @@ MailmanProxy Objects
SMTPChannel Objects
-------------------
-.. class:: SMTPChannel(server, conn, addr)
+.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\
+ map=None))
Create a new :class:`SMTPChannel` object which manages the communication
between the server and a single SMTP client.
+ *conn* and *addr* are as per the instance variables described below.
+
+ *data_size_limit* specifies the maximum number of bytes that will be
+ accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no
+ limit.
+
+ A dictionary can be specified in *map* to avoid using a global socket map.
+
To use a custom SMTPChannel implementation you need to override the
:attr:`SMTPServer.channel_class` of your :class:`SMTPServer`.
diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index addc6be..46cfa36 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -69,20 +69,15 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
required from the beginning of the connection and using :meth:`starttls` is
not appropriate. If *host* is not specified, the local host is used. If
*port* is zero, the standard SMTP-over-SSL port (465) is used. The optional
- arguments *local_hostname* and *source_address* have the same meaning as
- they do in the :class:`SMTP` class. *keyfile* and *certfile* are also
- optional, and can contain a PEM formatted private key and certificate chain
- file for the SSL connection. *context* also optional, can contain a
- SSLContext, and is an alternative to keyfile and certfile; If it is
- specified both keyfile and certfile must be None. The optional *timeout*
- parameter specifies a timeout in seconds for blocking operations like the
- connection attempt (if not specified, the global default timeout setting
- will be used). The optional source_address parameter allows to bind to some
- specific source address in a machine with multiple network interfaces,
- and/or to some specific source tcp port. It takes a 2-tuple (host, port),
- for the socket to bind to as its source address before connecting. If
- omitted (or if host or port are ``''`` and/or 0 respectively) the OS default
- behavior will be used.
+ arguments *local_hostname*, *timeout* and *source_address* have the same
+ meaning as they do in the :class:`SMTP` class. *context*, also optional,
+ can contain a :class:`~ssl.SSLContext` and allows to configure various
+ aspects of the secure connection. Please read :ref:`ssl-security` for
+ best practices.
+
+ *keyfile* and *certfile* are a legacy alternative to *context*, and can
+ point to a PEM formatted private key and certificate chain file for the
+ SSL connection.
.. versionchanged:: 3.3
*context* was added.
@@ -90,6 +85,10 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions).
.. versionchanged:: 3.3
source_address argument was added.
+ .. versionchanged:: 3.4
+ The class now supports hostname check with
+ :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see
+ :data:`ssl.HAS_SNI`).
.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None)
@@ -110,8 +109,11 @@ A nice selection of exceptions is defined as well:
.. exception:: SMTPException
- The base exception class for all the other exceptions provided by this
- module.
+ Subclass of :exc:`OSError` that is the base exception class for all
+ the other exceptions provided by this module.
+
+ .. versionchanged:: 3.4
+ SMTPException became subclass of :exc:`OSError`
.. exception:: SMTPServerDisconnected
@@ -316,6 +318,11 @@ An :class:`SMTP` instance has the following methods:
.. versionchanged:: 3.3
*context* was added.
+ .. versionchanged:: 3.4
+ The method now supports hostname check with
+ :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see
+ :data:`~ssl.HAS_SNI`).
+
.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[])
diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 3a49eed..54c6bad 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -6,8 +6,7 @@
This module provides access to the BSD *socket* interface. It is available on
-all modern Unix systems, Windows, MacOS, OS/2, and probably additional
-platforms.
+all modern Unix systems, Windows, MacOS, and probably additional platforms.
.. note::
@@ -107,8 +106,8 @@ created. Socket addresses are represented as follows:
.. versionadded:: 3.3
-- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`)
- support specific representations.
+- Certain other address families (:const:`AF_BLUETOOTH`, :const:`AF_PACKET`,
+ :const:`AF_CAN`) support specific representations.
.. XXX document them!
@@ -264,6 +263,16 @@ Constants
.. versionadded:: 3.3
+.. data:: CAN_BCM
+ CAN_BCM_*
+
+ CAN_BCM, in the CAN protocol family, is the broadcast manager (BCM) protocol.
+ Broadcast manager constants, documented in the Linux documentation, are also
+ defined in the socket module.
+
+ Availability: Linux >= 2.6.25.
+
+ .. versionadded:: 3.4
.. data:: AF_RDS
PF_RDS
@@ -290,6 +299,11 @@ Constants
TIPC related constants, matching the ones exported by the C socket API. See
the TIPC documentation for more information.
+.. data:: AF_LINK
+
+ Availability: BSD, OSX.
+
+ .. versionadded:: 3.4
.. data:: has_ipv6
@@ -306,20 +320,29 @@ Creating sockets
The following functions all create :ref:`socket objects <socket-objects>`.
-.. function:: socket([family[, type[, proto]]])
+.. function:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None)
Create a new socket using the given address family, socket type and protocol
number. The address family should be :const:`AF_INET` (the default),
:const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN` or :const:`AF_RDS`. The
socket type should be :const:`SOCK_STREAM` (the default),
:const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_``
- constants. The protocol number is usually zero and may be omitted in that
- case or :const:`CAN_RAW` in case the address family is :const:`AF_CAN`.
+ constants. The protocol number is usually zero and may be omitted or in the
+ case where the address family is :const:`AF_CAN` the protocol should be one
+ of :const:`CAN_RAW` or :const:`CAN_BCM`.
+
+ The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
.. versionchanged:: 3.3
The AF_CAN family was added.
The AF_RDS family was added.
+ .. versionchanged:: 3.4
+ The CAN_BCM protocol was added.
+
+ .. versionchanged:: 3.4
+ The returned socket is now non-inheritable.
+
.. function:: socketpair([family[, type[, proto]]])
@@ -329,10 +352,15 @@ The following functions all create :ref:`socket objects <socket-objects>`.
if defined on the platform; otherwise, the default is :const:`AF_INET`.
Availability: Unix.
+ The newly created sockets are :ref:`non-inheritable <fd_inheritance>`.
+
.. versionchanged:: 3.2
The returned socket objects now support the whole socket API, rather
than a subset.
+ .. versionchanged:: 3.4
+ The returned sockets are now non-inheritable.
+
.. function:: create_connection(address[, timeout[, source_address]])
@@ -360,7 +388,7 @@ The following functions all create :ref:`socket objects <socket-objects>`.
support for the :keyword:`with` statement was added.
-.. function:: fromfd(fd, family, type[, proto])
+.. function:: fromfd(fd, family, type, proto=0)
Duplicate the file descriptor *fd* (an integer as returned by a file object's
:meth:`fileno` method) and build a socket object from the result. Address
@@ -371,6 +399,11 @@ The following functions all create :ref:`socket objects <socket-objects>`.
a socket passed to a program as standard input or output (such as a server
started by the Unix inet daemon). The socket is assumed to be in blocking mode.
+ The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
+
+ .. versionchanged:: 3.4
+ The returned socket is now non-inheritable.
+
.. function:: fromshare(data)
@@ -598,7 +631,10 @@ The :mod:`socket` module also offers various network-related services:
both the value of *address_family* and the underlying implementation of
:c:func:`inet_pton`.
- Availability: Unix (maybe not all platforms).
+ Availability: Unix (maybe not all platforms), Windows.
+
+ .. versionchanged:: 3.4
+ Windows support added
.. function:: inet_ntop(address_family, packed_ip)
@@ -614,7 +650,10 @@ The :mod:`socket` module also offers various network-related services:
specified address family, :exc:`ValueError` will be raised. A
:exc:`OSError` is raised for errors from the call to :func:`inet_ntop`.
- Availability: Unix (maybe not all platforms).
+ Availability: Unix (maybe not all platforms), Windows.
+
+ .. versionchanged:: 3.4
+ Windows support added
..
@@ -735,6 +774,11 @@ to sockets.
*new* socket object usable to send and receive data on the connection, and
*address* is the address bound to the socket on the other end of the connection.
+ The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
+
+ .. versionchanged:: 3.4
+ The socket is now non-inheritable.
+
.. method:: socket.bind(address)
@@ -787,6 +831,16 @@ to sockets.
.. versionadded:: 3.2
+.. method:: socket.dup()
+
+ Duplicate the socket.
+
+ The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
+
+ .. versionchanged:: 3.4
+ The socket is now non-inheritable.
+
+
.. method:: socket.fileno()
Return the socket's file descriptor (a small integer). This is useful with
@@ -797,6 +851,15 @@ to sockets.
this limitation.
+.. method:: socket.get_inheritable()
+
+ Get the :ref:`inheritable flag <fd_inheritance>` of the socket's file
+ descriptor or socket's handle: ``True`` if the socket can be inherited in
+ child processes, ``False`` if it cannot.
+
+ .. versionadded:: 3.4
+
+
.. method:: socket.getpeername()
Return the remote address to which the socket is connected. This is useful to
@@ -1083,6 +1146,14 @@ to sockets.
.. versionadded:: 3.3
+.. method:: socket.set_inheritable(inheritable)
+
+ Set the :ref:`inheritable flag <fd_inheritance>` of the socket's file
+ descriptor or socket's handle.
+
+ .. versionadded:: 3.4
+
+
.. method:: socket.setblocking(flag)
Set blocking or non-blocking mode of the socket: if *flag* is false, the
@@ -1364,7 +1435,16 @@ the interface::
s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
The last example shows how to use the socket interface to communicate to a CAN
-network. This example might require special priviledge::
+network using the raw socket protocol. To use CAN with the broadcast
+manager protocol instead, open a socket with::
+
+ socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM)
+
+After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you
+can use the :meth:`socket.send`, and the :meth:`socket.recv` operations (and
+their counterparts) on the socket object as usual.
+
+This example might require special priviledge::
import socket
import struct
diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst
index add611f..58be78f 100644
--- a/Doc/library/spwd.rst
+++ b/Doc/library/spwd.rst
@@ -19,9 +19,9 @@ below, see ``<shadow.h>``):
+-------+---------------+---------------------------------+
| Index | Attribute | Meaning |
+=======+===============+=================================+
-| 0 | ``sp_nam`` | Login name |
+| 0 | ``sp_namp`` | Login name |
+-------+---------------+---------------------------------+
-| 1 | ``sp_pwd`` | Encrypted password |
+| 1 | ``sp_pwdp`` | Encrypted password |
+-------+---------------+---------------------------------+
| 2 | ``sp_lstchg`` | Date of last change |
+-------+---------------+---------------------------------+
@@ -36,15 +36,15 @@ below, see ``<shadow.h>``):
+-------+---------------+---------------------------------+
| 6 | ``sp_inact`` | Number of days after password |
| | | expires until account is |
-| | | blocked |
+| | | disabled |
+-------+---------------+---------------------------------+
| 7 | ``sp_expire`` | Number of days since 1970-01-01 |
-| | | until account is disabled |
+| | | when account expires |
+-------+---------------+---------------------------------+
| 8 | ``sp_flag`` | Reserved |
+-------+---------------+---------------------------------+
-The sp_nam and sp_pwd items are strings, all others are integers.
+The sp_namp and sp_pwdp items are strings, all others are integers.
:exc:`KeyError` is raised if the entry asked for cannot be found.
The following functions are defined:
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 8a50f87..1ec8398 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -159,7 +159,7 @@ Module functions and constants
first blank for the column name: the column name would simply be "x".
-.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements])
+.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri])
Opens a connection to the SQLite database file *database*. You can use
``":memory:"`` to open a database connection to a database that resides in RAM
@@ -195,6 +195,18 @@ Module functions and constants
for the connection, you can set the *cached_statements* parameter. The currently
implemented default is to cache 100 statements.
+ If *uri* is true, *database* is interpreted as a URI. This allows you
+ to specify options. For example, to open a database in read-only mode
+ you can use::
+
+ db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)
+
+ More information about this feature, including a list of recognized options, can
+ be found in the `SQLite URI documentation <http://www.sqlite.org/uri.html>`_.
+
+ .. versionchanged:: 3.4
+ Added the *uri* parameter.
+
.. function:: register_converter(typename, callable)
diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index aa5a52f..94a0c81 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -26,7 +26,8 @@ probably additional platforms, as long as OpenSSL is installed on that platform.
Some behavior may be platform dependent, since calls are made to the
operating system socket APIs. The installed version of OpenSSL may also
- cause variations in behavior.
+ cause variations in behavior. For example, TLSv1.1 and TLSv1.2 come with
+ openssl version 1.0.1.
.. warning::
Don't use this module without reading the :ref:`ssl-security`. Doing so
@@ -186,14 +187,16 @@ instead.
.. table::
- ======================== ========= ========= ========== =========
- *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1**
- ------------------------ --------- --------- ---------- ---------
- *SSLv2* yes no yes no
- *SSLv3* no yes yes no
- *SSLv23* yes no yes no
- *TLSv1* no no yes yes
- ======================== ========= ========= ========== =========
+ ======================== ========= ========= ========== ========= =========== ===========
+ *client* / **server** **SSLv2** **SSLv3** **SSLv23** **TLSv1** **TLSv1.1** **TLSv1.2**
+ ------------------------ --------- --------- ---------- --------- ----------- -----------
+ *SSLv2* yes no yes no no no
+ *SSLv3* no yes yes no no no
+ *SSLv23* yes no yes no no no
+ *TLSv1* no no yes yes no no
+ *TLSv1.1* no no yes no yes no
+ *TLSv1.2* no no yes no no yes
+ ======================== ========= ========= ========== ========= =========== ===========
.. note::
@@ -227,6 +230,58 @@ instead.
.. versionchanged:: 3.2
New optional argument *ciphers*.
+
+Context creation
+^^^^^^^^^^^^^^^^
+
+A convenience function helps create :class:`SSLContext` objects for common
+purposes.
+
+.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
+
+ Return a new :class:`SSLContext` object with default settings for
+ the given *purpose*. The settings are chosen by the :mod:`ssl` module,
+ and usually represent a higher security level than when calling the
+ :class:`SSLContext` constructor directly.
+
+ *cafile*, *capath*, *cadata* represent optional CA certificates to
+ trust for certificate verification, as in
+ :meth:`SSLContext.load_verify_locations`. If all three are
+ :const:`None`, this function can choose to trust the system's default
+ CA certificates instead.
+
+ The settings in Python 3.4 are: :data:`PROTOCOL_SSLv23`, :data:`OP_NO_SSLv2`,
+ and :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and
+ without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH`
+ as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED`
+ and either loads CA certificates (when at least one of *cafile*, *capath* or
+ *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load
+ default CA certificates.
+
+ .. note::
+ The protocol, options, cipher and other settings may change to more
+ restrictive values anytime without prior deprecation. The values
+ represent a fair balance between compatibility and security.
+
+ If your application needs specific settings, you should create a
+ :class:`SSLContext` and apply the settings yourself.
+
+ .. note::
+ If you find that when certain older clients or servers attempt to connect
+ with a :class:`SSLContext` created by this function that they get an
+ error stating "Protocol or cipher suite mismatch", it may be that they
+ only support SSL3.0 which this function excludes using the
+ :data:`OP_NO_SSLv3`. SSL3.0 has problematic security due to a number of
+ poor implementations and it's reliance on MD5 within the protocol. If you
+ wish to continue to use this function but still allow SSL 3.0 connections
+ you can re-enable them using::
+
+ ctx = ssl.create_default_context(Purpose.CLIENT_AUTH)
+ ctx.options &= ~ssl.OP_NO_SSLv3
+
+ .. versionadded:: 3.4
+
+
Random generation
^^^^^^^^^^^^^^^^^
@@ -356,6 +411,61 @@ Certificate handling
Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of
bytes for that same certificate.
+.. function:: get_default_verify_paths()
+
+ Returns a named tuple with paths to OpenSSL's default cafile and capath.
+ The paths are the same as used by
+ :meth:`SSLContext.set_default_verify_paths`. The return value is a
+ :term:`named tuple` ``DefaultVerifyPaths``:
+
+ * :attr:`cafile` - resolved path to cafile or None if the file doesn't exist,
+ * :attr:`capath` - resolved path to capath or None if the directory doesn't exist,
+ * :attr:`openssl_cafile_env` - OpenSSL's environment key that points to a cafile,
+ * :attr:`openssl_cafile` - hard coded path to a cafile,
+ * :attr:`openssl_capath_env` - OpenSSL's environment key that points to a capath,
+ * :attr:`openssl_capath` - hard coded path to a capath directory
+
+ .. versionadded:: 3.4
+
+.. function:: enum_certificates(store_name)
+
+ Retrieve certificates from Windows' system cert store. *store_name* may be
+ one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert
+ stores, too.
+
+ The function returns a list of (cert_bytes, encoding_type, trust) tuples.
+ The encoding_type specifies the encoding of cert_bytes. It is either
+ :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for
+ PKCS#7 ASN.1 data. Trust specifies the purpose of the certificate as a set
+ of OIDS or exactly ``True`` if the certificate is trustworthy for all
+ purposes.
+
+ Example::
+
+ >>> ssl.enum_certificates("CA")
+ [(b'data...', 'x509_asn', {'1.3.6.1.5.5.7.3.1', '1.3.6.1.5.5.7.3.2'}),
+ (b'data...', 'x509_asn', True)]
+
+ Availability: Windows.
+
+ .. versionadded:: 3.4
+
+.. function:: enum_crls(store_name)
+
+ Retrieve CRLs from Windows' system cert store. *store_name* may be
+ one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert
+ stores, too.
+
+ The function returns a list of (cert_bytes, encoding_type, trust) tuples.
+ The encoding_type specifies the encoding of cert_bytes. It is either
+ :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for
+ PKCS#7 ASN.1 data.
+
+ Availability: Windows.
+
+ .. versionadded:: 3.4
+
+
Constants
^^^^^^^^^
@@ -392,6 +502,38 @@ Constants
be passed, either to :meth:`SSLContext.load_verify_locations` or as a
value of the ``ca_certs`` parameter to :func:`wrap_socket`.
+.. data:: VERIFY_DEFAULT
+
+ Possible value for :attr:`SSLContext.verify_flags`. In this mode,
+ certificate revocation lists (CRLs) are not checked. By default OpenSSL
+ does neither require nor verify CRLs.
+
+ .. versionadded:: 3.4
+
+.. data:: VERIFY_CRL_CHECK_LEAF
+
+ Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the
+ peer cert is check but non of the intermediate CA certificates. The mode
+ requires a valid CRL that is signed by the peer cert's issuer (its direct
+ ancestor CA). If no proper has been loaded
+ :attr:`SSLContext.load_verify_locations`, validation will fail.
+
+ .. versionadded:: 3.4
+
+.. data:: VERIFY_CRL_CHECK_CHAIN
+
+ Possible value for :attr:`SSLContext.verify_flags`. In this mode, CRLs of
+ all certificates in the peer cert chain are checked.
+
+ .. versionadded:: 3.4
+
+.. data:: VERIFY_X509_STRICT
+
+ Possible value for :attr:`SSLContext.verify_flags` to disable workarounds
+ for broken X.509 certificates.
+
+ .. versionadded:: 3.4
+
.. data:: PROTOCOL_SSLv2
Selects SSL version 2 as the channel encryption protocol.
@@ -417,9 +559,22 @@ Constants
.. data:: PROTOCOL_TLSv1
- Selects TLS version 1 as the channel encryption protocol. This is the most
+ Selects TLS version 1.0 as the channel encryption protocol.
+
+.. data:: PROTOCOL_TLSv1_1
+
+ Selects TLS version 1.1 as the channel encryption protocol.
+ Available only with openssl version 1.0.1+.
+
+ .. versionadded:: 3.4
+
+.. data:: PROTOCOL_TLSv1_2
+
+ Selects TLS version 1.2 as the channel encryption protocol. This is the most
modern version, and probably the best choice for maximum protection, if both
- sides can speak it.
+ sides can speak it. Available only with openssl version 1.0.1+.
+
+ .. versionadded:: 3.4
.. data:: OP_ALL
@@ -453,6 +608,22 @@ Constants
.. versionadded:: 3.2
+.. data:: OP_NO_TLSv1_1
+
+ Prevents a TLSv1.1 connection. This option is only applicable in conjunction
+ with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.1 as
+ the protocol version. Available only with openssl version 1.0.1+.
+
+ .. versionadded:: 3.4
+
+.. data:: OP_NO_TLSv1_2
+
+ Prevents a TLSv1.2 connection. This option is only applicable in conjunction
+ with :const:`PROTOCOL_SSLv23`. It prevents the peers from choosing TLSv1.2 as
+ the protocol version. Available only with openssl version 1.0.1+.
+
+ .. versionadded:: 3.4
+
.. data:: OP_CIPHER_SERVER_PREFERENCE
Use the server's cipher ordering preference, rather than the client's.
@@ -549,6 +720,37 @@ Constants
.. versionadded:: 3.2
+.. data:: ALERT_DESCRIPTION_HANDSHAKE_FAILURE
+ ALERT_DESCRIPTION_INTERNAL_ERROR
+ ALERT_DESCRIPTION_*
+
+ Alert Descriptions from :rfc:`5246` and others. The `IANA TLS Alert Registry
+ <http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6>`_
+ contains this list and references to the RFCs where their meaning is defined.
+
+ Used as the return value of the callback function in
+ :meth:`SSLContext.set_servername_callback`.
+
+ .. versionadded:: 3.4
+
+.. data:: Purpose.SERVER_AUTH
+
+ Option for :func:`create_default_context` and
+ :meth:`SSLContext.load_default_certs`. This value indicates that the
+ context may be used to authenticate Web servers (therefore, it will
+ be used to create client-side sockets).
+
+ .. versionadded:: 3.4
+
+.. data:: Purpose.CLIENT_AUTH
+
+ Option for :func:`create_default_context` and
+ :meth:`SSLContext.load_default_certs`. This value indicates that the
+ context may be used to authenticate Web clients (therefore, it will
+ be used to create server-side sockets).
+
+ .. versionadded:: 3.4
+
SSL Sockets
-----------
@@ -584,10 +786,16 @@ SSL sockets also have the following additional methods and attributes:
Perform the SSL setup handshake.
+ .. versionchanged:: 3.4
+ The handshake method also performce :func:`match_hostname` when the
+ :attr:`~SSLContext.check_hostname` attribute of the socket's
+ :attr:`~SSLSocket.context` is true.
+
.. method:: SSLSocket.getpeercert(binary_form=False)
If there is no certificate for the peer on the other end of the connection,
- returns ``None``.
+ return ``None``. If the SSL handshake hasn't been done yet, raise
+ :exc:`ValueError`.
If the ``binary_form`` parameter is :const:`False`, and a certificate was
received from the peer, this method returns a :class:`dict` instance. If the
@@ -645,6 +853,11 @@ SSL sockets also have the following additional methods and attributes:
The returned dictionary includes additional items such as ``issuer``
and ``notBefore``.
+ .. versionchanged:: 3.4
+ :exc:`ValueError` is raised when the handshake isn't done.
+ The returned dictionary includes additional X509v3 extension items
+ such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs.
+
.. method:: SSLSocket.cipher()
Returns a three-value tuple containing the name of the cipher being used, the
@@ -715,11 +928,30 @@ to speed up repeated connections from the same clients.
Create a new SSL context. You must pass *protocol* which must be one
of the ``PROTOCOL_*`` constants defined in this module.
- :data:`PROTOCOL_SSLv23` is recommended for maximum interoperability.
+ :data:`PROTOCOL_SSLv23` is currently recommended for maximum
+ interoperability.
+
+ .. seealso::
+ :func:`create_default_context` lets the :mod:`ssl` module choose
+ security settings for a given purpose.
:class:`SSLContext` objects have the following methods and attributes:
+.. method:: SSLContext.cert_store_stats()
+
+ Get statistics about quantities of loaded X.509 certificates, count of
+ X.509 certificates flagged as CA certificates and certificate revocation
+ lists as dictionary.
+
+ Example for a context with one CA cert and one other cert::
+
+ >>> context.cert_store_stats()
+ {'crl': 0, 'x509_ca': 1, 'x509': 2}
+
+ .. versionadded:: 3.4
+
+
.. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None)
Load a private key and the corresponding certificate. The *certfile*
@@ -750,12 +982,32 @@ to speed up repeated connections from the same clients.
.. versionchanged:: 3.3
New optional argument *password*.
-.. method:: SSLContext.load_verify_locations(cafile=None, capath=None)
+.. method:: SSLContext.load_default_certs(purpose=Purpose.SERVER_AUTH)
+
+ Load a set of default "certification authority" (CA) certificates from
+ default locations. On Windows it loads CA certs from the ``CA`` and
+ ``ROOT`` system stores. On other systems it calls
+ :meth:`SSLContext.set_default_verify_paths`. In the future the method may
+ load CA certificates from other locations, too.
+
+ The *purpose* flag specifies what kind of CA certificates are loaded. The
+ default settings :data:`Purpose.SERVER_AUTH` loads certificates, that are
+ flagged and trusted for TLS web server authentication (client side
+ sockets). :data:`Purpose.CLIENT_AUTH` loads CA certificates for client
+ certificate verification on the server side.
+
+ .. versionadded:: 3.4
+
+.. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None)
Load a set of "certification authority" (CA) certificates used to validate
other peers' certificates when :data:`verify_mode` is other than
:data:`CERT_NONE`. At least one of *cafile* or *capath* must be specified.
+ This method can also load certification revocation lists (CRLs) in PEM or
+ or DER format. In order to make use of CRLs, :attr:`SSLContext.verify_flags`
+ must be configured properly.
+
The *cafile* string, if present, is the path to a file of concatenated
CA certificates in PEM format. See the discussion of
:ref:`ssl-certificates` for more information about how to arrange the
@@ -766,6 +1018,25 @@ to speed up repeated connections from the same clients.
following an `OpenSSL specific layout
<http://www.openssl.org/docs/ssl/SSL_CTX_load_verify_locations.html>`_.
+ The *cadata* object, if present, is either an ASCII string of one or more
+ PEM-encoded certificates or a bytes-like object of DER-encoded
+ certificates. Like with *capath* extra lines around PEM-encoded
+ certificates are ignored but at least one certificate must be present.
+
+ .. versionchanged:: 3.4
+ New optional argument *cadata*
+
+.. method:: SSLContext.get_ca_certs(binary_form=False)
+
+ Get a list of loaded "certification authority" (CA) certificates. If the
+ ``binary_form`` parameter is :const:`False` each list
+ entry is a dict like the output of :meth:`SSLSocket.getpeercert`. Otherwise
+ the method returns a list of DER-encoded certificates. The returned list
+ does not contain certificates from *capath* unless a certificate was
+ requested and loaded by a SSL connection.
+
+ .. versionadded:: 3.4
+
.. method:: SSLContext.set_default_verify_paths()
Load a set of default "certification authority" (CA) certificates from
@@ -803,6 +1074,56 @@ to speed up repeated connections from the same clients.
.. versionadded:: 3.3
+.. method:: SSLContext.set_servername_callback(server_name_callback)
+
+ Register a callback function that will be called after the TLS Client Hello
+ handshake message has been received by the SSL/TLS server when the TLS client
+ specifies a server name indication. The server name indication mechanism
+ is specified in :rfc:`6066` section 3 - Server Name Indication.
+
+ Only one callback can be set per ``SSLContext``. If *server_name_callback*
+ is ``None`` then the callback is disabled. Calling this function a
+ subsequent time will disable the previously registered callback.
+
+ The callback function, *server_name_callback*, will be called with three
+ arguments; the first being the :class:`ssl.SSLSocket`, the second is a string
+ that represents the server name that the client is intending to communicate
+ (or :const:`None` if the TLS Client Hello does not contain a server name)
+ and the third argument is the original :class:`SSLContext`. The server name
+ argument is the IDNA decoded server name.
+
+ A typical use of this callback is to change the :class:`ssl.SSLSocket`'s
+ :attr:`SSLSocket.context` attribute to a new object of type
+ :class:`SSLContext` representing a certificate chain that matches the server
+ name.
+
+ Due to the early negotiation phase of the TLS connection, only limited
+ methods and attributes are usable like
+ :meth:`SSLSocket.selected_npn_protocol` and :attr:`SSLSocket.context`.
+ :meth:`SSLSocket.getpeercert`, :meth:`SSLSocket.getpeercert`,
+ :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compress` methods require that
+ the TLS connection has progressed beyond the TLS Client Hello and therefore
+ will not contain return meaningful values nor can they be called safely.
+
+ The *server_name_callback* function must return ``None`` to allow the
+ TLS negotiation to continue. If a TLS failure is required, a constant
+ :const:`ALERT_DESCRIPTION_* <ALERT_DESCRIPTION_INTERNAL_ERROR>` can be
+ returned. Other return values will result in a TLS fatal error with
+ :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`.
+
+ If there is a IDNA decoding error on the server name, the TLS connection
+ will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS
+ alert message to the client.
+
+ If an exception is raised from the *server_name_callback* function the TLS
+ connection will terminate with a fatal TLS alert message
+ :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`.
+
+ This method will raise :exc:`NotImplementedError` if the OpenSSL library
+ had OPENSSL_NO_TLSEXT defined when it was built.
+
+ .. versionadded:: 3.4
+
.. method:: SSLContext.load_dh_params(dhfile)
Load the key generation parameters for Diffie-Helman (DH) key exchange.
@@ -869,6 +1190,45 @@ to speed up repeated connections from the same clients.
>>> stats['hits'], stats['misses']
(0, 0)
+.. method:: SSLContext.get_ca_certs(binary_form=False)
+
+ Returns a list of dicts with information of loaded CA certs. If the
+ optional argument is true, returns a DER-encoded copy of the CA
+ certificate.
+
+ .. note::
+ Certificates in a capath directory aren't loaded unless they have
+ been used at least once.
+
+ .. versionadded:: 3.4
+
+.. attribute:: SSLContext.check_hostname
+
+ Wether to match the peer cert's hostname with :func:`match_hostname` in
+ :meth:`SSLSocket.do_handshake`. The context's
+ :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or
+ :data:`CERT_REQUIRED`, and you must pass *server_hostname* to
+ :meth:`~SSLContext.wrap_socket` in order to match the hostname.
+
+ Example::
+
+ import socket, ssl
+
+ context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+ context.verify_mode = ssl.CERT_REQUIRED
+ context.check_hostname = True
+ context.load_default_certs()
+
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ ssl_sock = context.wrap_socket(s, server_hostname='www.verisign.com'):
+ ssl_sock.connect(('www.verisign.com', 443))
+
+ .. versionadded:: 3.4
+
+ .. note::
+
+ This features requires OpenSSL 0.9.8f or newer.
+
.. attribute:: SSLContext.options
An integer representing the set of SSL options enabled on this context.
@@ -885,6 +1245,15 @@ to speed up repeated connections from the same clients.
The protocol version chosen when constructing the context. This attribute
is read-only.
+.. attribute:: SSLContext.verify_flags
+
+ The flags for certificate verification operations. You can set flags like
+ :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL
+ does neither require nor verify certificate revocation lists (CRLs).
+ Available only with openssl version 0.9.8+.
+
+ .. versionadded:: 3.4
+
.. attribute:: SSLContext.verify_mode
Whether to try to verify other peers' certificates and how to behave
@@ -970,20 +1339,9 @@ If you are going to require validation of the other side of the connection's
certificate, you need to provide a "CA certs" file, filled with the certificate
chains for each issuer you are willing to trust. Again, this file just contains
these chains concatenated together. For validation, Python will use the first
-chain it finds in the file which matches. Some "standard" root certificates are
-available from various certification authorities: `CACert.org
-<http://www.cacert.org/index.php?id=3>`_, `Thawte
-<http://www.thawte.com/roots/>`_, `Verisign
-<http://www.verisign.com/support/roots.html>`_, `Positive SSL
-<http://www.PositiveSSL.com/ssl-certificate-support/cert_installation/UTN-USERFirst-Hardware.crt>`_
-(used by python.org), `Equifax and GeoTrust
-<http://www.geotrust.com/resources/root_certificates/index.asp>`_.
-
-In general, if you are using SSL3 or TLS1, you don't need to put the full chain
-in your "CA certs" file; you only need the root certificates, and the remote
-peer is supposed to furnish the other certificates necessary to chain from its
-certificate to a root certificate. See :rfc:`4158` for more discussion of the
-way in which certification chains can be built.
+chain it finds in the file which matches. The platform's certificates file can
+be used by calling :meth:`SSLContext.load_default_certs`, this is done
+automatically with :func:`.create_default_context`.
Combined key and certificate
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1186,7 +1544,7 @@ waiting for clients to connect::
import socket, ssl
- context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.load_cert_chain(certfile="mycertfile", keyfile="mykeyfile")
bindsocket = socket.socket()
@@ -1263,9 +1621,40 @@ to be aware of:
Security considerations
-----------------------
+Best defaults
+^^^^^^^^^^^^^
+
+For **client use**, if you don't have any special requirements for your
+security policy, it is highly recommended that you use the
+:func:`create_default_context` function to create your SSL context.
+It will load the system's trusted CA certificates, enable certificate
+validation and hostname checking, and try to choose reasonably secure
+protocol and cipher settings.
+
+For example, here is how you would use the :class:`smtplib.SMTP` class to
+create a trusted, secure connection to a SMTP server::
+
+ >>> import ssl, smtplib
+ >>> smtp = smtplib.SMTP("mail.python.org", port=587)
+ >>> context = ssl.create_default_context()
+ >>> smtp.starttls(context=context)
+ (220, b'2.0.0 Ready to start TLS')
+
+If a client certificate is needed for the connection, it can be added with
+:meth:`SSLContext.load_cert_chain`.
+
+By contrast, if you create the SSL context by calling the :class:`SSLContext`
+constructor yourself, it will not have certificate validation nor hostname
+checking enabled by default. If you do so, please read the paragraphs below
+to achieve a good security level.
+
+Manual settings
+^^^^^^^^^^^^^^^
+
Verifying certificates
-^^^^^^^^^^^^^^^^^^^^^^
+''''''''''''''''''''''
+When calling the the :class:`SSLContext` constructor directly,
:const:`CERT_NONE` is the default. Since it does not authenticate the other
peer, it can be insecure, especially in client mode where most of time you
would like to ensure the authenticity of the server you're talking to.
@@ -1274,7 +1663,9 @@ Therefore, when in client mode, it is highly recommended to use
have to check that the server certificate, which can be obtained by calling
:meth:`SSLSocket.getpeercert`, matches the desired service. For many
protocols and applications, the service can be identified by the hostname;
-in this case, the :func:`match_hostname` function can be used.
+in this case, the :func:`match_hostname` function can be used. This common
+check is automatically performed when :attr:`SSLContext.check_hostname` is
+enabled.
In server mode, if you want to authenticate your clients using the SSL layer
(rather than using a higher-level authentication mechanism), you'll also have
@@ -1287,7 +1678,7 @@ to specify :const:`CERT_REQUIRED` and similarly check the client certificate.
by default).
Protocol versions
-^^^^^^^^^^^^^^^^^
+'''''''''''''''''
SSL version 2 is considered insecure and is therefore dangerous to use. If
you want maximum compatibility between clients and servers, it is recommended
@@ -1297,27 +1688,20 @@ SSLv2 explicitly using the :data:`SSLContext.options` attribute::
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
-The SSL context created above will allow SSLv3 and TLSv1 connections, but
-not SSLv2.
+The SSL context created above will allow SSLv3 and TLSv1 (and later, if
+supported by your system) connections, but not SSLv2.
Cipher selection
-^^^^^^^^^^^^^^^^
+''''''''''''''''
If you have advanced security requirements, fine-tuning of the ciphers
enabled when negotiating a SSL session is possible through the
:meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the
ssl module disables certain weak ciphers by default, but you may want
-to further restrict the cipher choice. For example::
-
- context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
- context.set_ciphers('HIGH:!aNULL:!eNULL')
-
-The ``!aNULL:!eNULL`` part of the cipher spec is necessary to disable ciphers
-which don't provide both encryption and authentication. Be sure to read
-OpenSSL's documentation about the `cipher list
-format <http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT>`_.
-If you want to check which ciphers are enabled by a given cipher list,
-use the ``openssl ciphers`` command on your system.
+to further restrict the cipher choice. Be sure to read OpenSSL's documentation
+about the `cipher list format <http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT>`_.
+If you want to check which ciphers are enabled by a given cipher list, use the
+``openssl ciphers`` command on your system.
Multi-processing
^^^^^^^^^^^^^^^^
@@ -1350,3 +1734,12 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or
`RFC 4366: Transport Layer Security (TLS) Extensions <http://www.ietf.org/rfc/rfc4366>`_
Blake-Wilson et. al.
+
+ `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <http://www.ietf.org/rfc/rfc5246>`_
+ T. Dierks et. al.
+
+ `RFC 6066: Transport Layer Security (TLS) Extensions <http://www.ietf.org/rfc/rfc6066>`_
+ D. Eastlake
+
+ `IANA TLS: Transport Layer Security (TLS) Parameters <http://www.iana.org/assignments/tls-parameters/tls-parameters.xml>`_
+ IANA
diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst
index 6c20aa2..24769f6 100644
--- a/Doc/library/stat.rst
+++ b/Doc/library/stat.rst
@@ -6,7 +6,8 @@
os.lstat() and os.fstat().
.. sectionauthor:: Skip Montanaro <skip@automatrix.com>
-**Source code:** :source:`Lib/stat.py`
+**Source code:** :source:`Modules/_stat.c`
+ :source:`Lib/stat.py`
--------------
@@ -15,6 +16,9 @@ results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they
exist). For complete details about the :c:func:`stat`, :c:func:`fstat` and
:c:func:`lstat` calls, consult the documentation for your system.
+.. versionchanged:: 3.4
+ The stat module is backed by a C implementation.
+
The :mod:`stat` module defines the following functions to test for specific file
types:
@@ -53,6 +57,24 @@ types:
Return non-zero if the mode is from a socket.
+.. function:: S_ISDOOR(mode)
+
+ Return non-zero if the mode is from a door.
+
+ .. versionadded:: 3.4
+
+.. function:: S_ISPORT(mode)
+
+ Return non-zero if the mode is from an event port.
+
+ .. versionadded:: 3.4
+
+.. function:: S_ISWHT(mode)
+
+ Return non-zero if the mode is from a whiteout.
+
+ .. versionadded:: 3.4
+
Two additional functions are defined for more general manipulation of the file's
mode:
@@ -113,6 +135,10 @@ readable string:
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ The function supports :data:`S_IFDOOR`, :data:`S_IFPORT` and
+ :data:`S_IFWHT`.
+
All the variables below are simply symbolic indexes into the 10-tuple returned
by :func:`os.stat`, :func:`os.fstat` or :func:`os.lstat`.
@@ -210,6 +236,29 @@ Use of the functions above is more portable than use of the first set of flags:
FIFO.
+.. data:: S_IFDOOR
+
+ Door.
+
+ .. versionadded:: 3.4
+
+.. data:: S_IFPORT
+
+ Event port.
+
+ .. versionadded:: 3.4
+
+.. data:: S_IFWHT
+
+ Whiteout.
+
+ .. versionadded:: 3.4
+
+.. note::
+
+ :data:`S_IFDOOR`, :data:`S_IFPORT` or :data:`S_IFWHT` are defined as
+ 0 when the platform does not have support for the file types.
+
The following flags can also be used in the *mode* argument of :func:`os.chmod`:
.. data:: S_ISUID
diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
new file mode 100644
index 0000000..4e77838
--- /dev/null
+++ b/Doc/library/statistics.rst
@@ -0,0 +1,418 @@
+:mod:`statistics` --- Mathematical statistics functions
+=======================================================
+
+.. module:: statistics
+ :synopsis: mathematical statistics functions
+.. moduleauthor:: Steven D'Aprano <steve+python@pearwood.info>
+.. sectionauthor:: Steven D'Aprano <steve+python@pearwood.info>
+
+.. versionadded:: 3.4
+
+.. testsetup:: *
+
+ from statistics import *
+ __name__ = '<doctest>'
+
+**Source code:** :source:`Lib/statistics.py`
+
+--------------
+
+This module provides functions for calculating mathematical statistics of
+numeric (:class:`Real`-valued) data.
+
+.. note::
+
+ Unless explicitly noted otherwise, these functions support :class:`int`,
+ :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
+ Behaviour with other types (whether in the numeric tower or not) is
+ currently unsupported. Mixed types are also undefined and
+ implementation-dependent. If your input data consists of mixed types,
+ you may be able to use :func:`map` to ensure a consistent result, e.g.
+ ``map(float, input_data)``.
+
+Averages and measures of central location
+-----------------------------------------
+
+These functions calculate an average or typical value from a population
+or sample.
+
+======================= =============================================
+:func:`mean` Arithmetic mean ("average") of data.
+:func:`median` Median (middle value) of data.
+:func:`median_low` Low median of data.
+:func:`median_high` High median of data.
+:func:`median_grouped` Median, or 50th percentile, of grouped data.
+:func:`mode` Mode (most common value) of discrete data.
+======================= =============================================
+
+Measures of spread
+------------------
+
+These functions calculate a measure of how much the population or sample
+tends to deviate from the typical or average values.
+
+======================= =============================================
+:func:`pstdev` Population standard deviation of data.
+:func:`pvariance` Population variance of data.
+:func:`stdev` Sample standard deviation of data.
+:func:`variance` Sample variance of data.
+======================= =============================================
+
+
+Function details
+----------------
+
+Note: The functions do not require the data given to them to be sorted.
+However, for reading convenience, most of the examples show sorted sequences.
+
+.. function:: mean(data)
+
+ Return the sample arithmetic mean of *data*, a sequence or iterator of
+ real-valued numbers.
+
+ The arithmetic mean is the sum of the data divided by the number of data
+ points. It is commonly called "the average", although it is only one of many
+ different mathematical averages. It is a measure of the central location of
+ the data.
+
+ If *data* is empty, :exc:`StatisticsError` will be raised.
+
+ Some examples of use:
+
+ .. doctest::
+
+ >>> mean([1, 2, 3, 4, 4])
+ 2.8
+ >>> mean([-1.0, 2.5, 3.25, 5.75])
+ 2.625
+
+ >>> from fractions import Fraction as F
+ >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)])
+ Fraction(13, 21)
+
+ >>> from decimal import Decimal as D
+ >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")])
+ Decimal('0.5625')
+
+ .. note::
+
+ The mean is strongly affected by outliers and is not a robust estimator
+ for central location: the mean is not necessarily a typical example of the
+ data points. For more robust, although less efficient, measures of
+ central location, see :func:`median` and :func:`mode`. (In this case,
+ "efficient" refers to statistical efficiency rather than computational
+ efficiency.)
+
+ The sample mean gives an unbiased estimate of the true population mean,
+ which means that, taken on average over all the possible samples,
+ ``mean(sample)`` converges on the true mean of the entire population. If
+ *data* represents the entire population rather than a sample, then
+ ``mean(data)`` is equivalent to calculating the true population mean μ.
+
+
+.. function:: median(data)
+
+ Return the median (middle value) of numeric data, using the common "mean of
+ middle two" method. If *data* is empty, :exc:`StatisticsError` is raised.
+
+ The median is a robust measure of central location, and is less affected by
+ the presence of outliers in your data. When the number of data points is
+ odd, the middle data point is returned:
+
+ .. doctest::
+
+ >>> median([1, 3, 5])
+ 3
+
+ When the number of data points is even, the median is interpolated by taking
+ the average of the two middle values:
+
+ .. doctest::
+
+ >>> median([1, 3, 5, 7])
+ 4.0
+
+ This is suited for when your data is discrete, and you don't mind that the
+ median may not be an actual data point.
+
+ .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped`
+
+
+.. function:: median_low(data)
+
+ Return the low median of numeric data. If *data* is empty,
+ :exc:`StatisticsError` is raised.
+
+ The low median is always a member of the data set. When the number of data
+ points is odd, the middle value is returned. When it is even, the smaller of
+ the two middle values is returned.
+
+ .. doctest::
+
+ >>> median_low([1, 3, 5])
+ 3
+ >>> median_low([1, 3, 5, 7])
+ 3
+
+ Use the low median when your data are discrete and you prefer the median to
+ be an actual data point rather than interpolated.
+
+
+.. function:: median_high(data)
+
+ Return the high median of data. If *data* is empty, :exc:`StatisticsError`
+ is raised.
+
+ The high median is always a member of the data set. When the number of data
+ points is odd, the middle value is returned. When it is even, the larger of
+ the two middle values is returned.
+
+ .. doctest::
+
+ >>> median_high([1, 3, 5])
+ 3
+ >>> median_high([1, 3, 5, 7])
+ 5
+
+ Use the high median when your data are discrete and you prefer the median to
+ be an actual data point rather than interpolated.
+
+
+.. function:: median_grouped(data, interval=1)
+
+ Return the median of grouped continuous data, calculated as the 50th
+ percentile, using interpolation. If *data* is empty, :exc:`StatisticsError`
+ is raised.
+
+ .. doctest::
+
+ >>> median_grouped([52, 52, 53, 54])
+ 52.5
+
+ In the following example, the data are rounded, so that each value represents
+ the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5-1.5, 2
+ is the midpoint of 1.5-2.5, 3 is the midpoint of 2.5-3.5, etc. With the data
+ given, the middle value falls somewhere in the class 3.5-4.5, and
+ interpolation is used to estimate it:
+
+ .. doctest::
+
+ >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5])
+ 3.7
+
+ Optional argument *interval* represents the class interval, and defaults
+ to 1. Changing the class interval naturally will change the interpolation:
+
+ .. doctest::
+
+ >>> median_grouped([1, 3, 3, 5, 7], interval=1)
+ 3.25
+ >>> median_grouped([1, 3, 3, 5, 7], interval=2)
+ 3.5
+
+ This function does not check whether the data points are at least
+ *interval* apart.
+
+ .. impl-detail::
+
+ Under some circumstances, :func:`median_grouped` may coerce data points to
+ floats. This behaviour is likely to change in the future.
+
+ .. seealso::
+
+ * "Statistics for the Behavioral Sciences", Frederick J Gravetter and
+ Larry B Wallnau (8th Edition).
+
+ * Calculating the `median <http://www.ualberta.ca/~opscan/median.html>`_.
+
+ * The `SSMEDIAN
+ <https://projects.gnome.org/gnumeric/doc/gnumeric-function-SSMEDIAN.shtml>`_
+ function in the Gnome Gnumeric spreadsheet, including `this discussion
+ <https://mail.gnome.org/archives/gnumeric-list/2011-April/msg00018.html>`_.
+
+
+.. function:: mode(data)
+
+ Return the most common data point from discrete or nominal *data*. The mode
+ (when it exists) is the most typical value, and is a robust measure of
+ central location.
+
+ If *data* is empty, or if there is not exactly one most common value,
+ :exc:`StatisticsError` is raised.
+
+ ``mode`` assumes discrete data, and returns a single value. This is the
+ standard treatment of the mode as commonly taught in schools:
+
+ .. doctest::
+
+ >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
+ 3
+
+ The mode is unique in that it is the only statistic which also applies
+ to nominal (non-numeric) data:
+
+ .. doctest::
+
+ >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
+ 'red'
+
+
+.. function:: pstdev(data, mu=None)
+
+ Return the population standard deviation (the square root of the population
+ variance). See :func:`pvariance` for arguments and other details.
+
+ .. doctest::
+
+ >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75])
+ 0.986893273527251
+
+
+.. function:: pvariance(data, mu=None)
+
+ Return the population variance of *data*, a non-empty iterable of real-valued
+ numbers. Variance, or second moment about the mean, is a measure of the
+ variability (spread or dispersion) of data. A large variance indicates that
+ the data is spread out; a small variance indicates it is clustered closely
+ around the mean.
+
+ If the optional second argument *mu* is given, it should be the mean of
+ *data*. If it is missing or ``None`` (the default), the mean is
+ automatically calculated.
+
+ Use this function to calculate the variance from the entire population. To
+ estimate the variance from a sample, the :func:`variance` function is usually
+ a better choice.
+
+ Raises :exc:`StatisticsError` if *data* is empty.
+
+ Examples:
+
+ .. doctest::
+
+ >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25]
+ >>> pvariance(data)
+ 1.25
+
+ If you have already calculated the mean of your data, you can pass it as the
+ optional second argument *mu* to avoid recalculation:
+
+ .. doctest::
+
+ >>> mu = mean(data)
+ >>> pvariance(data, mu)
+ 1.25
+
+ This function does not attempt to verify that you have passed the actual mean
+ as *mu*. Using arbitrary values for *mu* may lead to invalid or impossible
+ results.
+
+ Decimals and Fractions are supported:
+
+ .. doctest::
+
+ >>> from decimal import Decimal as D
+ >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")])
+ Decimal('24.815')
+
+ >>> from fractions import Fraction as F
+ >>> pvariance([F(1, 4), F(5, 4), F(1, 2)])
+ Fraction(13, 72)
+
+ .. note::
+
+ When called with the entire population, this gives the population variance
+ σ². When called on a sample instead, this is the biased sample variance
+ s², also known as variance with N degrees of freedom.
+
+ If you somehow know the true population mean μ, you may use this function
+ to calculate the variance of a sample, giving the known population mean as
+ the second argument. Provided the data points are representative
+ (e.g. independent and identically distributed), the result will be an
+ unbiased estimate of the population variance.
+
+
+.. function:: stdev(data, xbar=None)
+
+ Return the sample standard deviation (the square root of the sample
+ variance). See :func:`variance` for arguments and other details.
+
+ .. doctest::
+
+ >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75])
+ 1.0810874155219827
+
+
+.. function:: variance(data, xbar=None)
+
+ Return the sample variance of *data*, an iterable of at least two real-valued
+ numbers. Variance, or second moment about the mean, is a measure of the
+ variability (spread or dispersion) of data. A large variance indicates that
+ the data is spread out; a small variance indicates it is clustered closely
+ around the mean.
+
+ If the optional second argument *xbar* is given, it should be the mean of
+ *data*. If it is missing or ``None`` (the default), the mean is
+ automatically calculated.
+
+ Use this function when your data is a sample from a population. To calculate
+ the variance from the entire population, see :func:`pvariance`.
+
+ Raises :exc:`StatisticsError` if *data* has fewer than two values.
+
+ Examples:
+
+ .. doctest::
+
+ >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5]
+ >>> variance(data)
+ 1.3720238095238095
+
+ If you have already calculated the mean of your data, you can pass it as the
+ optional second argument *xbar* to avoid recalculation:
+
+ .. doctest::
+
+ >>> m = mean(data)
+ >>> variance(data, m)
+ 1.3720238095238095
+
+ This function does not attempt to verify that you have passed the actual mean
+ as *xbar*. Using arbitrary values for *xbar* can lead to invalid or
+ impossible results.
+
+ Decimal and Fraction values are supported:
+
+ .. doctest::
+
+ >>> from decimal import Decimal as D
+ >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")])
+ Decimal('31.01875')
+
+ >>> from fractions import Fraction as F
+ >>> variance([F(1, 6), F(1, 2), F(5, 3)])
+ Fraction(67, 108)
+
+ .. note::
+
+ This is the sample variance s² with Bessel's correction, also known as
+ variance with N-1 degrees of freedom. Provided that the data points are
+ representative (e.g. independent and identically distributed), the result
+ should be an unbiased estimate of the true population variance.
+
+ If you somehow know the actual population mean μ you should pass it to the
+ :func:`pvariance` function as the *mu* parameter to get the variance of a
+ sample.
+
+Exceptions
+----------
+
+A single exception is defined:
+
+.. exception:: StatisticsError
+
+ Subclass of :exc:`ValueError` for statistics-related exceptions.
+
+..
+ # This modelines must appear within the last ten lines of the file.
+ kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8;
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index ab78cdc..bb421af 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1526,7 +1526,7 @@ expression support in the :mod:`re` module).
at that position.
-.. method:: str.expandtabs([tabsize])
+.. method:: str.expandtabs(tabsize=8)
Return a copy of the string where all tab characters are replaced by one or
more spaces, depending on the current column and the given tab size. Tab
@@ -2462,6 +2462,10 @@ copying.
.. versionchanged:: 3.3
One-dimensional memoryviews with formats 'B', 'b' or 'c' are now hashable.
+ .. versionchanged:: 3.4
+ memoryview is now registered automatically with
+ :class:`collections.abc.Sequence`
+
:class:`memoryview` has several methods:
.. method:: __eq__(exporter)
diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst
index dde32b9..ec2e1be 100644
--- a/Doc/library/struct.rst
+++ b/Doc/library/struct.rst
@@ -66,6 +66,19 @@ The module defines the following exception and functions:
format (``len(buffer[offset:])`` must be at least ``calcsize(fmt)``).
+.. function:: iter_unpack(fmt, buffer)
+
+ Iteratively unpack from the buffer *buffer* according to the format
+ string *fmt*. This function returns an iterator which will read
+ equally-sized chunks from the buffer until all its contents have been
+ consumed. The buffer's size in bytes must be a multiple of the amount
+ of data required by the format, as reflected by :func:`calcsize`.
+
+ Each iteration yields a tuple as specified by the format string.
+
+ .. versionadded:: 3.4
+
+
.. function:: calcsize(fmt)
Return the size of the struct (and hence of the bytes object produced by
@@ -388,6 +401,13 @@ The :mod:`struct` module also defines the following type:
(``len(buffer[offset:])`` must be at least :attr:`self.size`).
+ .. method:: iter_unpack(buffer)
+
+ Identical to the :func:`iter_unpack` function, using the compiled format.
+ (``len(buffer)`` must be a multiple of :attr:`self.size`).
+
+ .. versionadded:: 3.4
+
.. attribute:: format
The format string used to construct this Struct object.
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 347dcc3..cdcbe82 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -116,7 +116,7 @@ use cases, the underlying :class:`Popen` interface can be used directly.
*timeout* was added.
-.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None)
+.. function:: check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None)
Run command with arguments and return its output.
@@ -129,15 +129,21 @@ use cases, the underlying :class:`Popen` interface can be used directly.
in :ref:`frequently-used-arguments` (hence the use of keyword-only notation
in the abbreviated signature). The full function signature is largely the
same as that of the :class:`Popen` constructor - this functions passes all
- supplied arguments other than *timeout* directly through to that interface.
- In addition, *stdout* is not permitted as an argument, as it is used
- internally to collect the output from the subprocess.
+ supplied arguments other than *input* and *timeout* directly through to
+ that interface. In addition, *stdout* is not permitted as an argument, as
+ it is used internally to collect the output from the subprocess.
The *timeout* argument is passed to :meth:`Popen.wait`. If the timeout
expires, the child process will be killed and then waited for again. The
:exc:`TimeoutExpired` exception will be re-raised after the child process
has terminated.
+ The *input* argument is passed to :meth:`Popen.communicate` and thus to the
+ subprocess's stdin. If used it must be a byte sequence, or a string if
+ ``universal_newlines=True``. When used, the internal :class:`Popen` object
+ is automatically created with ``stdin=PIPE``, and the *stdin* argument may
+ not be used as well.
+
Examples::
>>> subprocess.check_output(["echo", "Hello World!"])
@@ -146,6 +152,10 @@ use cases, the underlying :class:`Popen` interface can be used directly.
>>> subprocess.check_output(["echo", "Hello World!"], universal_newlines=True)
'Hello World!\n'
+ >>> subprocess.check_output(["sed", "-e", "s/foo/bar/"],
+ ... input=b"when in the course of fooman events\n")
+ b'when in the course of barman events\n'
+
>>> subprocess.check_output("exit 1", shell=True)
Traceback (most recent call last):
...
@@ -167,10 +177,6 @@ use cases, the underlying :class:`Popen` interface can be used directly.
... shell=True)
'ls: non_existent_file: No such file or directory\n'
- .. versionadded:: 3.1
-
- ..
-
.. warning::
Invoking the system shell with ``shell=True`` can be a security hazard
@@ -183,9 +189,13 @@ use cases, the underlying :class:`Popen` interface can be used directly.
read in the current process, the child process may block if it
generates enough output to the pipe to fill up the OS pipe buffer.
+ .. versionadded:: 3.1
+
.. versionchanged:: 3.3
*timeout* was added.
+ .. versionchanged:: 3.4
+ *input* was added.
.. data:: DEVNULL
@@ -619,6 +629,12 @@ Instances of the :class:`Popen` class have the following methods:
:exc:`TimeoutExpired` exception. It is safe to catch this exception and
retry the wait.
+ .. note::
+
+ The function is implemented using a busy loop (non-blocking call and
+ short sleeps). Use the :mod:`asyncio` module for an asynchronous wait:
+ see :class:`asyncio.create_subprocess_exec`.
+
.. warning::
This will deadlock when using ``stdout=PIPE`` and/or
@@ -629,6 +645,11 @@ Instances of the :class:`Popen` class have the following methods:
.. versionchanged:: 3.3
*timeout* was added.
+ .. deprecated:: 3.4
+
+ Do not use the undocumented *endtime* parameter. It is was
+ unintentionally exposed in 3.3 but was intended to be private
+ for internal use. Use *timeout* instead.
.. method:: Popen.communicate(input=None, timeout=None)
@@ -841,8 +862,6 @@ The :mod:`subprocess` module exposes the following constants.
The new process has a new console, instead of inheriting its parent's
console (the default).
- This flag is always set when :class:`Popen` is created with ``shell=True``.
-
.. data:: CREATE_NEW_PROCESS_GROUP
A :class:`Popen` ``creationflags`` parameter to specify that a new process
@@ -1057,9 +1076,9 @@ handling consistency are valid for these functions.
Return ``(status, output)`` of executing *cmd* in a shell.
- Execute the string *cmd* in a shell with :class:`Popen` and return a 2-tuple
- ``(status, output)`` via :func:`Popen.communicate`. Universal newlines mode
- is used; see the notes on :ref:`frequently-used-arguments` for more details.
+ Execute the string *cmd* in a shell with :meth:`Popen.check_output` and
+ return a 2-tuple ``(status, output)``. Universal newlines mode is used;
+ see the notes on :ref:`frequently-used-arguments` for more details.
A trailing newline is stripped from the output.
The exit status for the command can be interpreted
diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst
index 4bdb99b..a94ae08 100644
--- a/Doc/library/sunau.rst
+++ b/Doc/library/sunau.rst
@@ -150,8 +150,9 @@ AU_read objects, as returned by :func:`.open` above, have the following methods:
.. method:: AU_read.getparams()
- Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype,
- compname)``, equivalent to output of the :meth:`get\*` methods.
+ Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth,
+ framerate, nframes, comptype, compname)``, equivalent to output of the
+ :meth:`get\*` methods.
.. method:: AU_read.readframes(n)
@@ -211,6 +212,9 @@ AU_write objects, as returned by :func:`.open` above, have the following methods
Set the sample width (in bytes.)
+ .. versionchanged:: 3.4
+ Added support for 24-bit samples.
+
.. method:: AU_write.setframerate(n)
@@ -246,11 +250,17 @@ AU_write objects, as returned by :func:`.open` above, have the following methods
Write audio frames, without correcting *nframes*.
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
+
.. method:: AU_write.writeframes(data)
Write audio frames and make sure *nframes* is correct.
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
+
.. method:: AU_write.close()
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 36c608c..2328a10 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -12,11 +12,12 @@ always available.
.. data:: abiflags
- On POSIX systems where Python is build with the standard ``configure``
+ On POSIX systems where Python was built with the standard ``configure``
script, this contains the ABI flags as specified by :pep:`3149`.
.. versionadded:: 3.2
+
.. data:: argv
The list of command line arguments passed to a Python script. ``argv[0]`` is the
@@ -382,6 +383,21 @@ always available.
.. versionadded:: 3.1
+.. function:: getallocatedblocks()
+
+ Return the number of memory blocks currently allocated by the interpreter,
+ regardless of their size. This function is mainly useful for tracking
+ and debugging memory leaks. Because of the interpreter's internal
+ caches, the result can vary from call to call; you may have to call
+ :func:`_clear_type_cache()` and :func:`gc.collect()` to get more
+ predictable results.
+
+ If a Python build or implementation cannot reasonably compute this
+ information, :func:`getallocatedblocks()` is allowed to return 0 instead.
+
+ .. versionadded:: 3.4
+
+
.. function:: getcheckinterval()
Return the interpreter's "check interval"; see :func:`setcheckinterval`.
@@ -398,9 +414,10 @@ always available.
.. function:: getdlopenflags()
- Return the current value of the flags that are used for :c:func:`dlopen` calls.
- The flag constants are defined in the :mod:`ctypes` and :mod:`DLFCN` modules.
- Availability: Unix.
+ Return the current value of the flags that are used for
+ :c:func:`dlopen` calls. Symbolic names for the flag values can be
+ found in the :mod:`os` module (``RTLD_xxx`` constants, e.g.
+ :data:`os.RTLD_LAZY`). Availability: Unix.
.. function:: getfilesystemencoding()
@@ -579,9 +596,20 @@ always available.
| :const:`imag` | multiplier used for the imaginary part of a |
| | complex number |
+---------------------+--------------------------------------------------+
+ | :const:`algorithm` | name of the algorithm for hashing of str, bytes, |
+ | | and memoryview |
+ +---------------------+--------------------------------------------------+
+ | :const:`hash_bits` | internal output size of the hash algorithm |
+ +---------------------+--------------------------------------------------+
+ | :const:`seed_bits` | size of the seed key of the hash algorithm |
+ +---------------------+--------------------------------------------------+
+
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ Added *algorithm*, *hash_bits* and *seed_bits*
+
.. data:: hexversion
@@ -665,6 +693,17 @@ always available.
.. versionadded:: 3.1
+.. data:: __interactivehook__
+
+ When this attribute exists, its value is automatically called (with no
+ arguments) when the interpreter is launched in :ref:`interactive mode
+ <tut-interactive>`. This is done after the :envvar:`PYTHONSTARTUP` file is
+ read, so that you can set this hook there. The :mod:`site` module
+ :ref:`sets this <rlcompleter-config>`.
+
+ .. versionadded:: 3.4
+
+
.. function:: intern(string)
Enter *string* in the table of "interned" strings and return the interned string
@@ -812,8 +851,6 @@ always available.
Windows ``'win32'``
Windows/Cygwin ``'cygwin'``
Mac OS X ``'darwin'``
- OS/2 ``'os2'``
- OS/2 EMX ``'os2emx'``
================ ===========================
.. versionchanged:: 3.3
@@ -884,7 +921,7 @@ always available.
the interpreter loads extension modules. Among other things, this will enable a
lazy resolving of symbols when importing a module, if called as
``sys.setdlopenflags(0)``. To share symbols across extension modules, call as
- ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag modules
+ ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values
can be found in the :mod:`os` module (``RTLD_xxx`` constants, e.g.
:data:`os.RTLD_LAZY`).
@@ -1094,7 +1131,6 @@ always available.
| :const:`name` | Name of the thread implementation: |
| | |
| | * ``'nt'``: Windows threads |
- | | * ``'os2'``: OS/2 threads |
| | * ``'pthread'``: POSIX threads |
| | * ``'solaris'``: Solaris threads |
+------------------+---------------------------------------------------------+
diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst
index c47dcce..535ac54 100644
--- a/Doc/library/sysconfig.rst
+++ b/Doc/library/sysconfig.rst
@@ -83,8 +83,6 @@ Python currently supports seven schemes:
located under the user home directory.
- *nt*: scheme for NT platforms like Windows.
- *nt_user*: scheme for NT platforms, when the *user* option is used.
-- *os2*: scheme for OS/2 platforms.
-- *os2_home*: scheme for OS/2 patforms, when the *user* option is used.
Each scheme is itself composed of a series of paths and each path has a unique
identifier. Python currently uses eight paths:
diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst
index 1ead71c..59336c9 100644
--- a/Doc/library/tarfile.rst
+++ b/Doc/library/tarfile.rst
@@ -591,6 +591,67 @@ A :class:`TarInfo` object also provides some convenient query methods:
Return :const:`True` if it is one of character device, block device or FIFO.
+.. _tarfile-commandline:
+
+Command Line Interface
+----------------------
+
+.. versionadded:: 3.4
+
+The :mod:`tarfile` module provides a simple command line interface to interact
+with tar archives.
+
+If you want to create a new tar archive, specify its name after the :option:`-c`
+option and then list the filename(s) that should be included::
+
+ $ python -m tarfile -c monty.tar spam.txt eggs.txt
+
+Passing a directory is also acceptable::
+
+ $ python -m tarfile -c monty.tar life-of-brian_1979/
+
+If you want to extract a tar archive into the current directory, use
+the :option:`-e` option::
+
+ $ python -m tarfile -e monty.tar
+
+You can also extract a tar archive into a different directory by passing the
+directory's name::
+
+ $ python -m tarfile -e monty.tar other-dir/
+
+For a list of the files in a tar archive, use the :option:`-l` option::
+
+ $ python -m tarfile -l monty.tar
+
+
+Command line options
+~~~~~~~~~~~~~~~~~~~~
+
+.. cmdoption:: -l <tarfile>
+ --list <tarfile>
+
+ List files in a tarfile.
+
+.. cmdoption:: -c <tarfile> <source1> <sourceN>
+ --create <tarfile> <source1> <sourceN>
+
+ Create tarfile from source files.
+
+.. cmdoption:: -e <tarfile> [<output_dir>]
+ --extract <tarfile> [<output_dir>]
+
+ Extract tarfile into the current directory if *output_dir* is not specified.
+
+.. cmdoption:: -t <tarfile>
+ --test <tarfile>
+
+ Test whether the tarfile is valid or not.
+
+.. cmdoption:: -v, --verbose
+
+ Verbose output
+
.. _tar-examples:
Examples
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
index c28536d..83026d8 100644
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -443,13 +443,6 @@ The :mod:`test.support` module defines the following functions:
A decorator for running tests that require support for symbolic links.
-.. function:: suppress_crash_popup()
-
- A context manager that disables Windows Error Reporting dialogs using
- `SetErrorMode <http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621%28v=vs.85%29.aspx>`_.
- On other platforms it's a no-op.
-
-
.. decorator:: anticipate_failure(condition)
A decorator to conditionally mark tests with
@@ -594,6 +587,21 @@ The :mod:`test.support` module defines the following classes:
Temporarily unset the environment variable ``envvar``.
+.. class:: SuppressCrashReport()
+
+ A context manager used to try to prevent crash dialog popups on tests that
+ are expected to crash a subprocess.
+
+ On Windows, it disables Windows Error Reporting dialogs using
+ `SetErrorMode <http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx>`_.
+
+ On UNIX, :func:`resource.setrlimit` is used to set
+ :attr:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file
+ creation.
+
+ On both platforms, the old value is restored by :meth:`__exit__`.
+
+
.. class:: WarningsRecorder()
Class used to record warnings for unit tests. See documentation of
diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst
index c625254..edf1fd6 100644
--- a/Doc/library/textwrap.rst
+++ b/Doc/library/textwrap.rst
@@ -10,11 +10,11 @@
--------------
-The :mod:`textwrap` module provides two convenience functions, :func:`wrap` and
-:func:`fill`, as well as :class:`TextWrapper`, the class that does all the work,
-and two utility functions, :func:`dedent` and :func:`indent`. If you're just wrapping or filling one
-or two text strings, the convenience functions should be good enough;
-otherwise, you should use an instance of :class:`TextWrapper` for efficiency.
+The :mod:`textwrap` module provides some convenience functions,
+as well as :class:`TextWrapper`, the class that does all the work.
+If you're just wrapping or filling one or two text strings, the convenience
+functions should be good enough; otherwise, you should use an instance of
+:class:`TextWrapper` for efficiency.
.. function:: wrap(text, width=70, **kwargs)
@@ -39,19 +39,31 @@ otherwise, you should use an instance of :class:`TextWrapper` for efficiency.
In particular, :func:`fill` accepts exactly the same keyword arguments as
:func:`wrap`.
-Both :func:`wrap` and :func:`fill` work by creating a :class:`TextWrapper`
-instance and calling a single method on it. That instance is not reused, so for
-applications that wrap/fill many text strings, it will be more efficient for you
-to create your own :class:`TextWrapper` object.
-Text is preferably wrapped on whitespaces and right after the hyphens in
-hyphenated words; only then will long words be broken if necessary, unless
-:attr:`TextWrapper.break_long_words` is set to false.
+.. function:: shorten(text, width, **kwargs)
+
+ Collapse and truncate the given *text* to fit in the given *width*.
+
+ First the whitespace in *text* is collapsed (all whitespace is replaced by
+ single spaces). If the result fits in the *width*, it is returned.
+ Otherwise, enough words are dropped from the end so that the remaining words
+ plus the :attr:`placeholder` fit within :attr:`width`::
+
+ >>> textwrap.shorten("Hello world!", width=12)
+ 'Hello world!'
+ >>> textwrap.shorten("Hello world!", width=11)
+ 'Hello [...]'
+ >>> textwrap.shorten("Hello world", width=10, placeholder="...")
+ 'Hello...'
+
+ Optional keyword arguments correspond to the instance attributes of
+ :class:`TextWrapper`, documented below. Note that the whitespace is
+ collapsed before the text is passed to the :class:`TextWrapper` :meth:`fill`
+ function, so changing the value of :attr:`.tabsize`, :attr:`.expand_tabs`,
+ :attr:`.drop_whitespace`, and :attr:`.replace_whitespace` will have no effect.
+
+ .. versionadded:: 3.4
-Two additional utility function, :func:`dedent` and :func:`indent`, are
-provided to remove indentation from strings that have unwanted whitespace
-to the left of the text and to add an arbitrary prefix to selected lines
-in a block of text.
.. function:: dedent(text)
@@ -102,6 +114,16 @@ in a block of text.
+ world
+:func:`wrap`, :func:`fill` and :func:`shorten` work by creating a
+:class:`TextWrapper` instance and calling a single method on it. That
+instance is not reused, so for applications that process many text
+strings using :func:`wrap` and/or :func:`fill`, it may be more efficient to
+create your own :class:`TextWrapper` object.
+
+Text is preferably wrapped on whitespaces and right after the hyphens in
+hyphenated words; only then will long words be broken if necessary, unless
+:attr:`TextWrapper.break_long_words` is set to false.
+
.. class:: TextWrapper(**kwargs)
The :class:`TextWrapper` constructor accepts a number of optional keyword
@@ -235,7 +257,23 @@ in a block of text.
was to always allow breaking hyphenated words.
- :class:`TextWrapper` also provides two public methods, analogous to the
+ .. attribute:: max_lines
+
+ (default: ``None``) If not ``None``, then the output will contain at most
+ *max_lines* lines, with *placeholder* appearing at the end of the output.
+
+ .. versionadded:: 3.4
+
+
+ .. attribute:: placeholder
+
+ (default: ``' [...]'``) String that will appear at the end of the output
+ text if it has been truncated.
+
+ .. versionadded:: 3.4
+
+
+ :class:`TextWrapper` also provides some public methods, analogous to the
module-level convenience functions:
.. method:: wrap(text)
@@ -251,4 +289,3 @@ in a block of text.
Wraps the single paragraph in *text*, and returns a single string
containing the wrapped paragraph.
-
diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst
index 7c326d1..4a3b3ea 100644
--- a/Doc/library/threading.rst
+++ b/Doc/library/threading.rst
@@ -57,6 +57,15 @@ This module defines the following functions:
and threads that have not yet been started.
+.. function:: main_thread()
+
+ Return the main :class:`Thread` object. In normal conditions, the
+ main thread is the thread from which the Python interpreter was
+ started.
+
+ .. versionadded:: 3.4
+
+
.. function:: settrace(func)
.. index:: single: trace function
diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst
index a487917..0cc1586 100644
--- a/Doc/library/timeit.rst
+++ b/Doc/library/timeit.rst
@@ -151,7 +151,7 @@ The module defines three convenience functions and a public class:
t = Timer(...) # outside the try/except
try:
t.timeit(...) # or t.repeat(...)
- except:
+ except Exception:
t.print_exc()
The advantage over the standard traceback is that source lines in the
diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst
index eeb1f80..0715a43 100644
--- a/Doc/library/tkinter.rst
+++ b/Doc/library/tkinter.rst
@@ -15,7 +15,7 @@ this should open a window demonstrating a simple Tk interface.
.. seealso::
- `Python Tkinter Resources <http://www.python.org/topics/tkinter/>`_
+ `Python Tkinter Resources <https://wiki.python.org/moin/TkInter>`_
The Python Tkinter Topic Guide provides a great deal of information on using Tk
from Python and links to other sources of information on Tk.
@@ -747,32 +747,6 @@ Entry widget indexes (index, view index, etc.)
displayed. You can use these :mod:`tkinter` functions to access these special
points in text widgets:
-.. function:: AtEnd()
- refers to the last position in the text
-
- .. deprecated:: 3.3
-
-.. function:: AtInsert()
- refers to the point where the text cursor is
-
- .. deprecated:: 3.3
-
-.. function:: AtSelFirst()
- indicates the beginning point of the selected text
-
- .. deprecated:: 3.3
-
-.. function:: AtSelLast()
- denotes the last point of the selected text and finally
-
- .. deprecated:: 3.3
-
-.. function:: At(x[, y])
- refers to the character at pixel location *x*, *y* (with *y* not used in the
- case of a text entry widget, which contains a single line of text).
-
- .. deprecated:: 3.3
-
Text widget indexes
The index notation for Text widgets is very rich and is best described in the Tk
man pages.
diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst
index 32e5733..b68a8f1 100644
--- a/Doc/library/traceback.rst
+++ b/Doc/library/traceback.rst
@@ -129,6 +129,13 @@ The module defines the following functions:
A shorthand for ``format_list(extract_stack(f, limit))``.
+.. function:: clear_frames(tb)
+
+ Clears the local variables of all the stack frames in a traceback *tb*
+ by calling the :meth:`clear` method of each frame object.
+
+ .. versionadded:: 3.4
+
.. _traceback-example:
@@ -146,7 +153,7 @@ module. ::
source = input(">>> ")
try:
exec(source, envdir)
- except:
+ except Exception:
print("Exception in user code:")
print("-"*60)
traceback.print_exc(file=sys.stdout)
diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst
new file mode 100644
index 0000000..3405518
--- /dev/null
+++ b/Doc/library/tracemalloc.rst
@@ -0,0 +1,634 @@
+:mod:`tracemalloc` --- Trace memory allocations
+===============================================
+
+.. module:: tracemalloc
+ :synopsis: Trace memory allocations.
+
+.. versionadded:: 3.4
+
+The tracemalloc module is a debug tool to trace memory blocks allocated by
+Python. It provides the following information:
+
+* Traceback where an object was allocated
+* Statistics on allocated memory blocks per filename and per line number:
+ total size, number and average size of allocated memory blocks
+* Compute the differences between two snapshots to detect memory leaks
+
+To trace most memory blocks allocated by Python, the module should be started
+as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment
+variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line
+option. The :func:`tracemalloc.start` function can be called at runtime to
+start tracing Python memory allocations.
+
+By default, a trace of an allocated memory block only stores the most recent
+frame (1 frame). To store 25 frames at startup: set the
+:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the
+:option:`-X` ``tracemalloc=25`` command line option.
+
+
+Examples
+========
+
+Display the top 10
+------------------
+
+Display the 10 files allocating the most memory::
+
+ import tracemalloc
+
+ tracemalloc.start()
+
+ # ... run your application ...
+
+ snapshot = tracemalloc.take_snapshot()
+ top_stats = snapshot.statistics('lineno')
+
+ print("[ Top 10 ]")
+ for stat in top_stats[:10]:
+ print(stat)
+
+
+Example of output of the Python test suite::
+
+ [ Top 10 ]
+ <frozen importlib._bootstrap>:716: size=4855 KiB, count=39328, average=126 B
+ <frozen importlib._bootstrap>:284: size=521 KiB, count=3199, average=167 B
+ /usr/lib/python3.4/collections/__init__.py:368: size=244 KiB, count=2315, average=108 B
+ /usr/lib/python3.4/unittest/case.py:381: size=185 KiB, count=779, average=243 B
+ /usr/lib/python3.4/unittest/case.py:402: size=154 KiB, count=378, average=416 B
+ /usr/lib/python3.4/abc.py:133: size=88.7 KiB, count=347, average=262 B
+ <frozen importlib._bootstrap>:1446: size=70.4 KiB, count=911, average=79 B
+ <frozen importlib._bootstrap>:1454: size=52.0 KiB, count=25, average=2131 B
+ <string>:5: size=49.7 KiB, count=148, average=344 B
+ /usr/lib/python3.4/sysconfig.py:411: size=48.0 KiB, count=1, average=48.0 KiB
+
+We can see that Python loaded ``4.8 MiB`` data (bytecode and constants) from
+modules and that the :mod:`collections` module allocated ``244 KiB`` to build
+:class:`~collections.namedtuple` types.
+
+See :meth:`Snapshot.statistics` for more options.
+
+
+Compute differences
+-------------------
+
+Take two snapshots and display the differences::
+
+ import tracemalloc
+ tracemalloc.start()
+ # ... start your application ...
+
+ snapshot1 = tracemalloc.take_snapshot()
+ # ... call the function leaking memory ...
+ snapshot2 = tracemalloc.take_snapshot()
+
+ top_stats = snapshot2.compare_to(snapshot1, 'lineno')
+
+ print("[ Top 10 differences ]")
+ for stat in top_stats[:10]:
+ print(stat)
+
+Example of output before/after running some tests of the Python test suite::
+
+ [ Top 10 differences ]
+ <frozen importlib._bootstrap>:716: size=8173 KiB (+4428 KiB), count=71332 (+39369), average=117 B
+ /usr/lib/python3.4/linecache.py:127: size=940 KiB (+940 KiB), count=8106 (+8106), average=119 B
+ /usr/lib/python3.4/unittest/case.py:571: size=298 KiB (+298 KiB), count=589 (+589), average=519 B
+ <frozen importlib._bootstrap>:284: size=1005 KiB (+166 KiB), count=7423 (+1526), average=139 B
+ /usr/lib/python3.4/mimetypes.py:217: size=112 KiB (+112 KiB), count=1334 (+1334), average=86 B
+ /usr/lib/python3.4/http/server.py:848: size=96.0 KiB (+96.0 KiB), count=1 (+1), average=96.0 KiB
+ /usr/lib/python3.4/inspect.py:1465: size=83.5 KiB (+83.5 KiB), count=109 (+109), average=784 B
+ /usr/lib/python3.4/unittest/mock.py:491: size=77.7 KiB (+77.7 KiB), count=143 (+143), average=557 B
+ /usr/lib/python3.4/urllib/parse.py:476: size=71.8 KiB (+71.8 KiB), count=969 (+969), average=76 B
+ /usr/lib/python3.4/contextlib.py:38: size=67.2 KiB (+67.2 KiB), count=126 (+126), average=546 B
+
+We can see that Python has loaded ``8.2 MiB`` of module data (bytecode and
+constants), and that this is ``4.4 MiB`` more than had been loaded before the
+tests, when the previous snapshot was taken. Similarly, the :mod:`linecache`
+module has cached ``940 KiB`` of Python source code to format tracebacks, all
+of it since the previous snapshot.
+
+If the system has little free memory, snapshots can be written on disk using
+the :meth:`Snapshot.dump` method to analyze the snapshot offline. Then use the
+:meth:`Snapshot.load` method reload the snapshot.
+
+
+Get the traceback of a memory block
+-----------------------------------
+
+Code to display the traceback of the biggest memory block::
+
+ import tracemalloc
+
+ # Store 25 frames
+ tracemalloc.start(25)
+
+ # ... run your application ...
+
+ snapshot = tracemalloc.take_snapshot()
+ top_stats = snapshot.statistics('traceback')
+
+ # pick the biggest memory block
+ stat = top_stats[0]
+ print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024))
+ for line in stat.traceback.format():
+ print(line)
+
+Example of output of the Python test suite (traceback limited to 25 frames)::
+
+ 903 memory blocks: 870.1 KiB
+ File "<frozen importlib._bootstrap>", line 716
+ File "<frozen importlib._bootstrap>", line 1036
+ File "<frozen importlib._bootstrap>", line 934
+ File "<frozen importlib._bootstrap>", line 1068
+ File "<frozen importlib._bootstrap>", line 619
+ File "<frozen importlib._bootstrap>", line 1581
+ File "<frozen importlib._bootstrap>", line 1614
+ File "/usr/lib/python3.4/doctest.py", line 101
+ import pdb
+ File "<frozen importlib._bootstrap>", line 284
+ File "<frozen importlib._bootstrap>", line 938
+ File "<frozen importlib._bootstrap>", line 1068
+ File "<frozen importlib._bootstrap>", line 619
+ File "<frozen importlib._bootstrap>", line 1581
+ File "<frozen importlib._bootstrap>", line 1614
+ File "/usr/lib/python3.4/test/support/__init__.py", line 1728
+ import doctest
+ File "/usr/lib/python3.4/test/test_pickletools.py", line 21
+ support.run_doctest(pickletools)
+ File "/usr/lib/python3.4/test/regrtest.py", line 1276
+ test_runner()
+ File "/usr/lib/python3.4/test/regrtest.py", line 976
+ display_failure=not verbose)
+ File "/usr/lib/python3.4/test/regrtest.py", line 761
+ match_tests=ns.match_tests)
+ File "/usr/lib/python3.4/test/regrtest.py", line 1563
+ main()
+ File "/usr/lib/python3.4/test/__main__.py", line 3
+ regrtest.main_in_temp_cwd()
+ File "/usr/lib/python3.4/runpy.py", line 73
+ exec(code, run_globals)
+ File "/usr/lib/python3.4/runpy.py", line 160
+ "__main__", fname, loader, pkg_name)
+
+We can see that the most memory was allocated in the :mod:`importlib` module to
+load data (bytecode and constants) from modules: ``870 KiB``. The traceback is
+where the :mod:`importlib` loaded data most recently: on the ``import pdb``
+line of the :mod:`doctest` module. The traceback may change if a new module is
+loaded.
+
+
+Pretty top
+----------
+
+Code to display the 10 lines allocating the most memory with a pretty output,
+ignoring ``<frozen importlib._bootstrap>`` and ``<unknown>`` files::
+
+ import linecache
+ import os
+ import tracemalloc
+
+ def display_top(snapshot, group_by='lineno', limit=10):
+ snapshot = snapshot.filter_traces((
+ tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
+ tracemalloc.Filter(False, "<unknown>"),
+ ))
+ top_stats = snapshot.statistics(group_by)
+
+ print("Top %s lines" % limit)
+ for index, stat in enumerate(top_stats[:limit], 1):
+ frame = stat.traceback[0]
+ # replace "/path/to/module/file.py" with "module/file.py"
+ filename = os.sep.join(frame.filename.split(os.sep)[-2:])
+ print("#%s: %s:%s: %.1f KiB"
+ % (index, filename, frame.lineno, stat.size / 1024))
+ line = linecache.getline(frame.filename, frame.lineno).strip()
+ if line:
+ print(' %s' % line)
+
+ other = top_stats[limit:]
+ if other:
+ size = sum(stat.size for stat in other)
+ print("%s other: %.1f KiB" % (len(other), size / 1024))
+ total = sum(stat.size for stat in top_stats)
+ print("Total allocated size: %.1f KiB" % (total / 1024))
+
+ tracemalloc.start()
+
+ # ... run your application ...
+
+ snapshot = tracemalloc.take_snapshot()
+ display_top(snapshot)
+
+Example of output of the Python test suite::
+
+ Top 10 lines
+ #1: Lib/base64.py:414: 419.8 KiB
+ _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
+ #2: Lib/base64.py:306: 419.8 KiB
+ _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
+ #3: collections/__init__.py:368: 293.6 KiB
+ exec(class_definition, namespace)
+ #4: Lib/abc.py:133: 115.2 KiB
+ cls = super().__new__(mcls, name, bases, namespace)
+ #5: unittest/case.py:574: 103.1 KiB
+ testMethod()
+ #6: Lib/linecache.py:127: 95.4 KiB
+ lines = fp.readlines()
+ #7: urllib/parse.py:476: 71.8 KiB
+ for a in _hexdig for b in _hexdig}
+ #8: <string>:5: 62.0 KiB
+ #9: Lib/_weakrefset.py:37: 60.0 KiB
+ self.data = set()
+ #10: Lib/base64.py:142: 59.8 KiB
+ _b32tab2 = [a + b for a in _b32tab for b in _b32tab]
+ 6220 other: 3602.8 KiB
+ Total allocated size: 5303.1 KiB
+
+See :meth:`Snapshot.statistics` for more options.
+
+
+API
+===
+
+Functions
+---------
+
+.. function:: clear_traces()
+
+ Clear traces of memory blocks allocated by Python.
+
+ See also :func:`stop`.
+
+
+.. function:: get_object_traceback(obj)
+
+ Get the traceback where the Python object *obj* was allocated.
+ Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc`
+ module is not tracing memory allocations or did not trace the allocation of
+ the object.
+
+ See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions.
+
+
+.. function:: get_traceback_limit()
+
+ Get the maximum number of frames stored in the traceback of a trace.
+
+ The :mod:`tracemalloc` module must be tracing memory allocations to
+ get the limit, otherwise an exception is raised.
+
+ The limit is set by the :func:`start` function.
+
+
+.. function:: get_traced_memory()
+
+ Get the current size and peak size of memory blocks traced by the
+ :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``.
+
+
+.. function:: get_tracemalloc_memory()
+
+ Get the memory usage in bytes of the :mod:`tracemalloc` module used to store
+ traces of memory blocks.
+ Return an :class:`int`.
+
+
+.. function:: is_tracing()
+
+ ``True`` if the :mod:`tracemalloc` module is tracing Python memory
+ allocations, ``False`` otherwise.
+
+ See also :func:`start` and :func:`stop` functions.
+
+
+.. function:: start(nframe: int=1)
+
+ Start tracing Python memory allocations: install hooks on Python memory
+ allocators. Collected tracebacks of traces will be limited to *nframe*
+ frames. By default, a trace of a memory block only stores the most recent
+ frame: the limit is ``1``. *nframe* must be greater or equal to ``1``.
+
+ Storing more than ``1`` frame is only useful to compute statistics grouped
+ by ``'traceback'`` or to compute cumulative statistics: see the
+ :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods.
+
+ Storing more frames increases the memory and CPU overhead of the
+ :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function
+ to measure how much memory is used by the :mod:`tracemalloc` module.
+
+ The :envvar:`PYTHONTRACEMALLOC` environment variable
+ (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME``
+ command line option can be used to start tracing at startup.
+
+ See also :func:`stop`, :func:`is_tracing` and :func:`get_traceback_limit`
+ functions.
+
+
+.. function:: stop()
+
+ Stop tracing Python memory allocations: uninstall hooks on Python memory
+ allocators. Also clears all previously collected traces of memory blocks
+ allocated by Python.
+
+ Call :func:`take_snapshot` function to take a snapshot of traces before
+ clearing them.
+
+ See also :func:`start`, :func:`is_tracing` and :func:`clear_traces`
+ functions.
+
+
+.. function:: take_snapshot()
+
+ Take a snapshot of traces of memory blocks allocated by Python. Return a new
+ :class:`Snapshot` instance.
+
+ The snapshot does not include memory blocks allocated before the
+ :mod:`tracemalloc` module started to trace memory allocations.
+
+ Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use
+ the *nframe* parameter of the :func:`start` function to store more frames.
+
+ The :mod:`tracemalloc` module must be tracing memory allocations to take a
+ snapshot, see the the :func:`start` function.
+
+ See also the :func:`get_object_traceback` function.
+
+
+Filter
+------
+
+.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False)
+
+ Filter on traces of memory blocks.
+
+ See the :func:`fnmatch.fnmatch` function for the syntax of
+ *filename_pattern*. The ``'.pyc'`` and ``'.pyo'`` file extensions are
+ replaced with ``'.py'``.
+
+ Examples:
+
+ * ``Filter(True, subprocess.__file__)`` only includes traces of the
+ :mod:`subprocess` module
+ * ``Filter(False, tracemalloc.__file__)`` excludes traces of the
+ :mod:`tracemalloc` module
+ * ``Filter(False, "<unknown>")`` excludes empty tracebacks
+
+ .. attribute:: inclusive
+
+ If *inclusive* is ``True`` (include), only trace memory blocks allocated
+ in a file with a name matching :attr:`filename_pattern` at line number
+ :attr:`lineno`.
+
+ If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in
+ a file with a name matching :attr:`filename_pattern` at line number
+ :attr:`lineno`.
+
+ .. attribute:: lineno
+
+ Line number (``int``) of the filter. If *lineno* is ``None``, the filter
+ matches any line number.
+
+ .. attribute:: filename_pattern
+
+ Filename pattern of the filter (``str``).
+
+ .. attribute:: all_frames
+
+ If *all_frames* is ``True``, all frames of the traceback are checked. If
+ *all_frames* is ``False``, only the most recent frame is checked.
+
+ This attribute has no effect if the traceback limit is ``1``. See the
+ :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit`
+ attribute.
+
+
+Frame
+-----
+
+.. class:: Frame
+
+ Frame of a traceback.
+
+ The :class:`Traceback` class is a sequence of :class:`Frame` instances.
+
+ .. attribute:: filename
+
+ Filename (``str``).
+
+ .. attribute:: lineno
+
+ Line number (``int``).
+
+
+Snapshot
+--------
+
+.. class:: Snapshot
+
+ Snapshot of traces of memory blocks allocated by Python.
+
+ The :func:`take_snapshot` function creates a snapshot instance.
+
+ .. method:: compare_to(old_snapshot: Snapshot, group_by: str, cumulative: bool=False)
+
+ Compute the differences with an old snapshot. Get statistics as a sorted
+ list of :class:`StatisticDiff` instances grouped by *group_by*.
+
+ See the :meth:`statistics` method for *group_by* and *cumulative*
+ parameters.
+
+ The result is sorted from the biggest to the smallest by: absolute value
+ of :attr:`StatisticDiff.size_diff`, :attr:`StatisticDiff.size`, absolute
+ value of :attr:`StatisticDiff.count_diff`, :attr:`Statistic.count` and
+ then by :attr:`StatisticDiff.traceback`.
+
+
+ .. method:: dump(filename)
+
+ Write the snapshot into a file.
+
+ Use :meth:`load` to reload the snapshot.
+
+
+ .. method:: filter_traces(filters)
+
+ Create a new :class:`Snapshot` instance with a filtered :attr:`traces`
+ sequence, *filters* is a list of :class:`Filter` instances. If *filters*
+ is an empty list, return a new :class:`Snapshot` instance with a copy of
+ the traces.
+
+ All inclusive filters are applied at once, a trace is ignored if no
+ inclusive filters match it. A trace is ignored if at least one exclusive
+ filter matchs it.
+
+
+ .. classmethod:: load(filename)
+
+ Load a snapshot from a file.
+
+ See also :meth:`dump`.
+
+
+ .. method:: statistics(group_by: str, cumulative: bool=False)
+
+ Get statistics as a sorted list of :class:`Statistic` instances grouped
+ by *group_by*:
+
+ ===================== ========================
+ group_by description
+ ===================== ========================
+ ``'filename'`` filename
+ ``'lineno'`` filename and line number
+ ``'traceback'`` traceback
+ ===================== ========================
+
+ If *cumulative* is ``True``, cumulate size and count of memory blocks of
+ all frames of the traceback of a trace, not only the most recent frame.
+ The cumulative mode can only be used with *group_by* equals to
+ ``'filename'`` and ``'lineno'``.
+
+ The result is sorted from the biggest to the smallest by:
+ :attr:`Statistic.size`, :attr:`Statistic.count` and then by
+ :attr:`Statistic.traceback`.
+
+
+ .. attribute:: traceback_limit
+
+ Maximum number of frames stored in the traceback of :attr:`traces`:
+ result of the :func:`get_traceback_limit` when the snapshot was taken.
+
+ .. attribute:: traces
+
+ Traces of all memory blocks allocated by Python: sequence of
+ :class:`Trace` instances.
+
+ The sequence has an undefined order. Use the :meth:`Snapshot.statistics`
+ method to get a sorted list of statistics.
+
+
+Statistic
+---------
+
+.. class:: Statistic
+
+ Statistic on memory allocations.
+
+ :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances.
+
+ See also the :class:`StatisticDiff` class.
+
+ .. attribute:: count
+
+ Number of memory blocks (``int``).
+
+ .. attribute:: size
+
+ Total size of memory blocks in bytes (``int``).
+
+ .. attribute:: traceback
+
+ Traceback where the memory block was allocated, :class:`Traceback`
+ instance.
+
+
+StatisticDiff
+-------------
+
+.. class:: StatisticDiff
+
+ Statistic difference on memory allocations between an old and a new
+ :class:`Snapshot` instance.
+
+ :func:`Snapshot.compare_to` returns a list of :class:`StatisticDiff`
+ instances. See also the :class:`Statistic` class.
+
+ .. attribute:: count
+
+ Number of memory blocks in the new snapshot (``int``): ``0`` if
+ the memory blocks have been released in the new snapshot.
+
+ .. attribute:: count_diff
+
+ Difference of number of memory blocks between the old and the new
+ snapshots (``int``): ``0`` if the memory blocks have been allocated in
+ the new snapshot.
+
+ .. attribute:: size
+
+ Total size of memory blocks in bytes in the new snapshot (``int``):
+ ``0`` if the memory blocks have been released in the new snapshot.
+
+ .. attribute:: size_diff
+
+ Difference of total size of memory blocks in bytes between the old and
+ the new snapshots (``int``): ``0`` if the memory blocks have been
+ allocated in the new snapshot.
+
+ .. attribute:: traceback
+
+ Traceback where the memory blocks were allocated, :class:`Traceback`
+ instance.
+
+
+Trace
+-----
+
+.. class:: Trace
+
+ Trace of a memory block.
+
+ The :attr:`Snapshot.traces` attribute is a sequence of :class:`Trace`
+ instances.
+
+ .. attribute:: size
+
+ Size of the memory block in bytes (``int``).
+
+ .. attribute:: traceback
+
+ Traceback where the memory block was allocated, :class:`Traceback`
+ instance.
+
+
+Traceback
+---------
+
+.. class:: Traceback
+
+ Sequence of :class:`Frame` instances sorted from the most recent frame to
+ the oldest frame.
+
+ A traceback contains at least ``1`` frame. If the ``tracemalloc`` module
+ failed to get a frame, the filename ``"<unknown>"`` at line number ``0`` is
+ used.
+
+ When a snapshot is taken, tracebacks of traces are limited to
+ :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function.
+
+ The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback`
+ instance.
+
+ .. method:: format(limit=None)
+
+ Format the traceback as a list of lines with newlines. Use the
+ :mod:`linecache` module to retrieve lines from the source code. If
+ *limit* is set, only format the *limit* most recent frames.
+
+ Similar to the :func:`traceback.format_tb` function, except that
+ :meth:`format` does not include newlines.
+
+ Example::
+
+ print("Traceback (most recent call first):")
+ for line in traceback:
+ print(line)
+
+ Output::
+
+ Traceback (most recent call first):
+ File "test.py", line 9
+ obj = Object()
+ File "test.py", line 12
+ tb = tracemalloc.get_object_traceback(f())
+
diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia
new file mode 100644
index 0000000..eccf4fa
--- /dev/null
+++ b/Doc/library/tulip_coro.dia
Binary files differ
diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png
new file mode 100644
index 0000000..65b6951
--- /dev/null
+++ b/Doc/library/tulip_coro.png
Binary files differ
diff --git a/Doc/library/types.rst b/Doc/library/types.rst
index 695480f..abdb939 100644
--- a/Doc/library/types.rst
+++ b/Doc/library/types.rst
@@ -15,6 +15,9 @@ It also defines names for some object types that are used by the standard
Python interpreter, but not exposed as builtins like :class:`int` or
:class:`str` are.
+Finally, it provides some additional type-related utility classes and functions
+that are not fundamental enough to be builtins.
+
Dynamic Type Creation
---------------------
@@ -107,9 +110,35 @@ Standard names are defined for the following types:
C".)
-.. data:: ModuleType
+.. class:: ModuleType(name, doc=None)
+
+ The type of :term:`modules <module>`. Constructor takes the name of the
+ module to be created and optionally its :term:`docstring`.
+
+ .. attribute:: __doc__
+
+ The :term:`docstring` of the module. Defaults to ``None``.
+
+ .. attribute:: __loader__
+
+ The :term:`loader` which loaded the module. Defaults to ``None``.
+
+ .. versionchanged:: 3.4
+ Defaults to ``None``. Previously the attribute was optional.
+
+ .. attribute:: __name__
+
+ The name of the module.
- The type of modules.
+ .. attribute:: __package__
+
+ Which :term:`package` a module belongs to. If the module is top-level
+ (i.e. not a part of any specific package) then the attribute should be set
+ to ``''``, else it should be set to the name of the package (which can be
+ :attr:`__name__` if the module is a package itself). Defaults to ``None``.
+
+ .. versionchanged:: 3.4
+ Defaults to ``None``. Previously the attribute was optional.
.. data:: TracebackType
@@ -194,6 +223,9 @@ Standard names are defined for the following types:
Return a new view of the underlying mapping's values.
+Additional Utility Classes and Functions
+----------------------------------------
+
.. class:: SimpleNamespace
A simple :class:`object` subclass that provides attribute access to its
@@ -212,9 +244,26 @@ Standard names are defined for the following types:
keys = sorted(self.__dict__)
items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys)
return "{}({})".format(type(self).__name__, ", ".join(items))
+ def __eq__(self, other):
+ return self.__dict__ == other.__dict__
``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``.
However, for a structured record type use :func:`~collections.namedtuple`
instead.
.. versionadded:: 3.3
+
+
+.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None)
+
+ Route attribute access on a class to __getattr__.
+
+ This is a descriptor, used to define attributes that act differently when
+ accessed through an instance and through a class. Instance access remains
+ normal, but access to an attribute through a class will be routed to the
+ class's __getattr__ method; this is done by raising AttributeError.
+
+ This allows one to have properties active on an instance, and have virtual
+ attributes on the class with the same name (see Enum for an example).
+
+ .. versionadded:: 3.4
diff --git a/Doc/library/undoc.rst b/Doc/library/undoc.rst
index 80386d2..20830e2 100644
--- a/Doc/library/undoc.rst
+++ b/Doc/library/undoc.rst
@@ -20,7 +20,7 @@ These modules are used to implement the :mod:`os.path` module, and are not
documented beyond this mention. There's little need to document these.
:mod:`ntpath`
- --- Implementation of :mod:`os.path` on Win32, Win64, WinCE, and OS/2 platforms.
+ --- Implementation of :mod:`os.path` on Win32, Win64, and WinCE platforms.
:mod:`posixpath`
--- Implementation of :mod:`os.path` on POSIX.
diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst
index ad70dd9..3b3d3a0 100644
--- a/Doc/library/unicodedata.rst
+++ b/Doc/library/unicodedata.rst
@@ -15,8 +15,8 @@
This module provides access to the Unicode Character Database (UCD) which
defines character properties for all Unicode characters. The data contained in
-this database is compiled from the `UCD version 6.1.0
-<http://www.unicode.org/Public/6.1.0/ucd>`_.
+this database is compiled from the `UCD version 6.3.0
+<http://www.unicode.org/Public/6.3.0/ucd>`_.
The module uses the same names and symbols as defined by Unicode
Standard Annex #44, `"Unicode Character Database"
@@ -166,6 +166,6 @@ Examples:
.. rubric:: Footnotes
-.. [#] http://www.unicode.org/Public/6.1.0/ucd/NameAliases.txt
+.. [#] http://www.unicode.org/Public/6.3.0/ucd/NameAliases.txt
-.. [#] http://www.unicode.org/Public/6.1.0/ucd/NamedSequences.txt
+.. [#] http://www.unicode.org/Public/6.3.0/ucd/NamedSequences.txt
diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst
index 554ef11..1c43bfe 100644
--- a/Doc/library/unittest.mock-examples.rst
+++ b/Doc/library/unittest.mock-examples.rst
@@ -277,6 +277,20 @@ instantiate the class in those tests.
...
AttributeError: object has no attribute 'old_method'
+Using a specification also enables a smarter matching of calls made to the
+mock, regardless of whether some parameters were passed as positional or
+named arguments::
+
+ >>> def f(a, b, c): pass
+ ...
+ >>> mock = Mock(spec=f)
+ >>> mock(1, 2, 3)
+ <Mock name='mock()' id='140161580456576'>
+ >>> mock.assert_called_with(a=1, b=2, c=3)
+
+If you want this smarter matching to also work with method calls on the mock,
+you can use :ref:`auto-speccing <auto-speccing>`.
+
If you want a stronger form of specification that prevents the setting
of arbitrary attributes as well as the getting of them then you can use
`spec_set` instead of `spec`.
@@ -920,8 +934,8 @@ After the `MagicMock` has been used we can use attributes like
the magic methods you specifically want:
>>> mock = Mock()
- >>> mock.__setitem__ = Mock(side_effect=getitem)
- >>> mock.__getitem__ = Mock(side_effect=setitem)
+ >>> mock.__getitem__ = Mock(side_effect=getitem)
+ >>> mock.__setitem__ = Mock(side_effect=setitem)
A *third* option is to use `MagicMock` but passing in `dict` as the `spec`
(or `spec_set`) argument so that the `MagicMock` created only has
diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index f805df3..cb72a68 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -264,7 +264,6 @@ the `new_callable` argument to `patch`.
<Mock name='mock.method()' id='...'>
>>> mock.method.assert_called_with(1, 2, 3, test='wow')
-
.. method:: assert_called_once_with(*args, **kwargs)
Assert that the mock was called exactly once and with the specified
@@ -685,6 +684,27 @@ have to create a dictionary and unpack it using `**`:
...
KeyError
+A callable mock which was created with a *spec* (or a *spec_set*) will
+introspect the specification object's signature when matching calls to
+the mock. Therefore, it can match the actual call's arguments regardless
+of whether they were passed positionally or by name::
+
+ >>> def f(a, b, c): pass
+ ...
+ >>> mock = Mock(spec=f)
+ >>> mock(1, 2, c=3)
+ <Mock name='mock()' id='140161580456576'>
+ >>> mock.assert_called_with(1, 2, 3)
+ >>> mock.assert_called_with(a=1, b=2, c=3)
+
+This applies to :meth:`~Mock.assert_called_with`,
+:meth:`~Mock.assert_called_once_with`, :meth:`~Mock.assert_has_calls` and
+:meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also
+apply to method calls on the mock object.
+
+ .. versionchanged:: 3.4
+ Added signature introspection on specced and autospecced mock objects.
+
.. class:: PropertyMock(*args, **kwargs)
@@ -1969,8 +1989,13 @@ mock_open
default) then a `MagicMock` will be created for you, with the API limited
to methods or attributes available on standard file handles.
- `read_data` is a string for the `~io.IOBase.read` method of the file handle
- to return. This is an empty string by default.
+ `read_data` is a string for the :meth:`~io.IOBase.read`,
+ :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods
+ of the file handle to return. Calls to those methods will take data from
+ `read_data` until it is depleted. The mock of these methods is pretty
+ simplistic. If you need more control over the data that you are feeding to
+ the tested code you will need to customize this mock for yourself.
+ `read_data` is an empty string by default.
Using `open` as a context manager is a great way to ensure your file handles
are closed properly and is becoming common::
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
index fdc9409..690c98b 100644
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -239,9 +239,10 @@ Test Discovery
Unittest supports simple test discovery. In order to be compatible with test
discovery, all of the test files must be :ref:`modules <tut-modules>` or
-:ref:`packages <tut-packages>` importable from the top-level directory of
-the project (this means that their filenames must be valid
-:ref:`identifiers <identifiers>`).
+:ref:`packages <tut-packages>` (including :term:`namespace packages
+<namespace package>`) importable from the top-level directory of
+the project (this means that their filenames must be valid :ref:`identifiers
+<identifiers>`).
Test discovery is implemented in :meth:`TestLoader.discover`, but can also be
used from the command line. The basic command-line usage is::
@@ -306,6 +307,9 @@ as the start directory.
Test modules and packages can customize test loading and discovery by through
the `load_tests protocol`_.
+.. versionchanged:: 3.4
+ Test discovery supports :term:`namespace packages <namespace package>`.
+
.. _organizing-tests:
@@ -559,8 +563,71 @@ The following decorators implement test skipping and expected failures:
Usually you can use :meth:`TestCase.skipTest` or one of the skipping
decorators instead of raising this directly.
-Skipped tests will not have :meth:`setUp` or :meth:`tearDown` run around them.
-Skipped classes will not have :meth:`setUpClass` or :meth:`tearDownClass` run.
+Skipped tests will not have :meth:`~TestCase.setUp` or :meth:`~TestCase.tearDown` run around them.
+Skipped classes will not have :meth:`~TestCase.setUpClass` or :meth:`~TestCase.tearDownClass` run.
+Skipped modules will not have :func:`setUpModule` or :func:`tearDownModule` run.
+
+
+.. _subtests:
+
+Distinguishing test iterations using subtests
+---------------------------------------------
+
+.. versionadded:: 3.4
+
+When some of your tests differ only by a some very small differences, for
+instance some parameters, unittest allows you to distinguish them inside
+the body of a test method using the :meth:`~TestCase.subTest` context manager.
+
+For example, the following test::
+
+ class NumbersTest(unittest.TestCase):
+
+ def test_even(self):
+ """
+ Test that numbers between 0 and 5 are all even.
+ """
+ for i in range(0, 6):
+ with self.subTest(i=i):
+ self.assertEqual(i % 2, 0)
+
+will produce the following output::
+
+ ======================================================================
+ FAIL: test_even (__main__.NumbersTest) (i=1)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ File "subtests.py", line 32, in test_even
+ self.assertEqual(i % 2, 0)
+ AssertionError: 1 != 0
+
+ ======================================================================
+ FAIL: test_even (__main__.NumbersTest) (i=3)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ File "subtests.py", line 32, in test_even
+ self.assertEqual(i % 2, 0)
+ AssertionError: 1 != 0
+
+ ======================================================================
+ FAIL: test_even (__main__.NumbersTest) (i=5)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ File "subtests.py", line 32, in test_even
+ self.assertEqual(i % 2, 0)
+ AssertionError: 1 != 0
+
+Without using a subtest, execution would stop after the first failure,
+and the error would be less easy to diagnose because the value of ``i``
+wouldn't be displayed::
+
+ ======================================================================
+ FAIL: test_even (__main__.NumbersTest)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ File "subtests.py", line 32, in test_even
+ self.assertEqual(i % 2, 0)
+ AssertionError: 1 != 0
.. _unittest-contents:
@@ -676,6 +743,21 @@ Test cases
.. versionadded:: 3.1
+ .. method:: subTest(msg=None, **params)
+
+ Return a context manager which executes the enclosed code block as a
+ subtest. *msg* and *params* are optional, arbitrary values which are
+ displayed whenever a subtest fails, allowing you to identify them
+ clearly.
+
+ A test case can contain any number of subtest declarations, and
+ they can be arbitrarily nested.
+
+ See :ref:`subtests` for more information.
+
+ .. versionadded:: 3.4
+
+
.. method:: debug()
Run the test without collecting the result. This allows exceptions raised
@@ -806,8 +888,8 @@ Test cases
- It is also possible to check that exceptions and warnings are raised using
- the following methods:
+ It is also possible to check the production of exceptions, warnings and
+ log messages using the following methods:
+---------------------------------------------------------+--------------------------------------+------------+
| Method | Checks that | New in |
@@ -824,6 +906,9 @@ Test cases
| :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 |
| <TestCase.assertWarnsRegex>` | and the message matches regex *r* | |
+---------------------------------------------------------+--------------------------------------+------------+
+ | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 |
+ | <TestCase.assertLogs>` | with minimum *level* | |
+ +---------------------------------------------------------+--------------------------------------+------------+
.. method:: assertRaises(exception, callable, *args, **kwds)
assertRaises(exception, msg=None)
@@ -954,6 +1039,47 @@ Test cases
.. versionchanged:: 3.3
Added the *msg* keyword argument when used as a context manager.
+ .. method:: assertLogs(logger=None, level=None)
+
+ A context manager to test that at least one message is logged on
+ the *logger* or one of its children, with at least the given
+ *level*.
+
+ If given, *logger* should be a :class:`logging.Logger` object or a
+ :class:`str` giving the name of a logger. The default is the root
+ logger, which will catch all messages.
+
+ If given, *level* should be either a numeric logging level or
+ its string equivalent (for example either ``"ERROR"`` or
+ :attr:`logging.ERROR`). The default is :attr:`logging.INFO`.
+
+ The test passes if at least one message emitted inside the ``with``
+ block matches the *logger* and *level* conditions, otherwise it fails.
+
+ The object returned by the context manager is a recording helper
+ which keeps tracks of the matching log messages. It has two
+ attributes:
+
+ .. attribute:: records
+
+ A list of :class:`logging.LogRecord` objects of the matching
+ log messages.
+
+ .. attribute:: output
+
+ A list of :class:`str` objects with the formatted output of
+ matching messages.
+
+ Example::
+
+ with self.assertLogs('foo', level='INFO') as cm:
+ logging.getLogger('foo').info('first message')
+ logging.getLogger('foo.bar').error('second message')
+ self.assertEqual(cm.output, ['INFO:foo:first message',
+ 'ERROR:foo.bar:second message'])
+
+ .. versionadded:: 3.4
+
There are also other methods used to perform more specific checks, such as:
@@ -1393,15 +1519,24 @@ Grouping tests
Tests grouped by a :class:`TestSuite` are always accessed by iteration.
Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note
- that this method maybe called several times on a single suite
- (for example when counting tests or comparing for equality)
- so the tests returned must be the same for repeated iterations.
+ that this method may be called several times on a single suite (for
+ example when counting tests or comparing for equality) so the tests
+ returned by repeated iterations before :meth:`TestSuite.run` must be the
+ same for each call iteration. After :meth:`TestSuite.run`, callers should
+ not rely on the tests returned by this method unless the caller uses a
+ subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve
+ test references.
.. versionchanged:: 3.2
In earlier versions the :class:`TestSuite` accessed tests directly rather
than through iteration, so overriding :meth:`__iter__` wasn't sufficient
for providing tests.
+ .. versionchanged:: 3.4
+ In earlier versions the :class:`TestSuite` held references to each
+ :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore
+ that behavior by overriding :meth:`TestSuite._removeTestAtIndex`.
+
In the typical usage of a :class:`TestSuite` object, the :meth:`run` method
is invoked by a :class:`TestRunner` rather than by the end-user test harness.
@@ -1500,7 +1635,9 @@ Loading and running tests
directory must be specified separately.
If importing a module fails, for example due to a syntax error, then this
- will be recorded as a single error and discovery will continue.
+ will be recorded as a single error and discovery will continue. If the
+ import failure is due to :exc:`SkipTest` being raised, it will be recorded
+ as a skip instead of an error.
If a test package name (directory with :file:`__init__.py`) matches the
pattern then the package will be checked for a ``load_tests``
@@ -1519,6 +1656,14 @@ Loading and running tests
.. versionadded:: 3.2
+ .. versionchanged:: 3.4
+ Modules that raise :exc:`SkipTest` on import are recorded as skips,
+ not errors.
+ Discovery works for :term:`namespace packages <namespace package>`.
+ Paths are sorted before being imported so that execution order is
+ the same even if the underlying file system's ordering is not
+ dependent on file name.
+
The following attributes of a :class:`TestLoader` can be configured either by
subclassing or assignment on an instance:
@@ -1630,6 +1775,10 @@ Loading and running tests
Return ``True`` if all tests run so far have passed, otherwise returns
``False``.
+ .. versionchanged:: 3.4
+ Returns ``False`` if there were any :attr:`unexpectedSuccesses`
+ from tests marked with the :func:`expectedFailure` decorator.
+
.. method:: stop()
@@ -1728,6 +1877,22 @@ Loading and running tests
:attr:`unexpectedSuccesses` attribute.
+ .. method:: addSubTest(test, subtest, outcome)
+
+ Called when a subtest finishes. *test* is the test case
+ corresponding to the test method. *subtest* is a custom
+ :class:`TestCase` instance describing the subtest.
+
+ If *outcome* is :const:`None`, the subtest succeeded. Otherwise,
+ it failed with an exception where *outcome* is a tuple of the form
+ returned by :func:`sys.exc_info`: ``(type, value, traceback)``.
+
+ The default implementation does nothing when the outcome is a
+ success, and records subtest failures as normal failures.
+
+ .. versionadded:: 3.4
+
+
.. class:: TextTestResult(stream, descriptions, verbosity)
A concrete implementation of :class:`TestResult` used by the
@@ -1783,6 +1948,14 @@ Loading and running tests
stream, descriptions, verbosity
+ .. method:: run(test)
+
+ This method is the main public interface to the `TextTestRunner`. This
+ method takes a :class:`TestSuite` or :class:`TestCase` instance. A
+ :class:`TestResult` is created by calling
+ :func:`_makeResult` and the test(s) are run and the
+ results printed to stdout.
+
.. function:: main(module='__main__', defaultTest=None, argv=None, testRunner=None, \
testLoader=unittest.defaultTestLoader, exit=True, verbosity=1, \
@@ -1802,9 +1975,10 @@ Loading and running tests
if __name__ == '__main__':
unittest.main(verbosity=2)
- The *defaultTest* argument is the name of the test to run if no test names
- are specified via *argv*. If not specified or ``None`` and no test names are
- provided via *argv*, all tests found in *module* are run.
+ The *defaultTest* argument is either the name of a single test or an
+ iterable of test names to run if no test names are specified via *argv*. If
+ not specified or ``None`` and no test names are provided via *argv*, all
+ tests found in *module* are run.
The *argv* argument can be a list of options passed to the program, with the
first element being the program name. If not specified or ``None``,
@@ -1842,6 +2016,10 @@ Loading and running tests
The *verbosity*, *failfast*, *catchbreak*, *buffer*
and *warnings* parameters were added.
+ .. versionchanged:: 3.4
+ The *defaultTest* parameter was changed to also accept an iterable of
+ test names.
+
load_tests Protocol
###################
diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst
index 25c13bd..f7f0c97 100644
--- a/Doc/library/urllib.error.rst
+++ b/Doc/library/urllib.error.rst
@@ -46,6 +46,13 @@ The following exceptions are raised by :mod:`urllib.error` as appropriate:
This is usually a string explaining the reason for this error.
+ .. attribute:: headers
+
+ The HTTP response headers for the HTTP request that caused the
+ :exc:`HTTPError`.
+
+ .. versionadded:: 3.4
+
.. exception:: ContentTooShortError(msg, content)
This exception is raised when the :func:`~urllib.request.urlretrieve`
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 6493a29..019f59c 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -218,12 +218,17 @@ The following classes are provided:
fetching of the image, this should be true.
*method* should be a string that indicates the HTTP request method that
- will be used (e.g. ``'HEAD'``). Its value is stored in the
+ will be used (e.g. ``'HEAD'``). If provided, its value is stored in the
:attr:`~Request.method` attribute and is used by :meth:`get_method()`.
+ Subclasses may indicate a default method by setting the
+ :attr:`~Request.method` attribute in the class itself.
.. versionchanged:: 3.3
:attr:`Request.method` argument is added to the Request class.
+ .. versionchanged:: 3.4
+ Default :attr:`Request.method` may be indicated at the class level.
+
.. class:: OpenerDirector()
@@ -356,6 +361,11 @@ The following classes are provided:
Open local files.
+.. class:: DataHandler()
+
+ Open data URLs.
+
+ .. versionadded:: 3.4
.. class:: FTPHandler()
@@ -391,6 +401,12 @@ request.
The original URL passed to the constructor.
+ .. versionchanged:: 3.4
+
+ Request.full_url is a property with setter, getter and a deleter. Getting
+ :attr:`~Request.full_url` returns the original request URL with the
+ fragment, if it was present.
+
.. attribute:: Request.type
The URI scheme.
@@ -413,6 +429,10 @@ request.
The entity body for the request, or None if not specified.
+ .. versionchanged:: 3.4
+ Changing value of :attr:`Request.data` now deletes "Content-Length"
+ header if it was previously set or calculated.
+
.. attribute:: Request.unverifiable
boolean, indicates whether the request is unverifiable as defined
@@ -420,13 +440,20 @@ request.
.. attribute:: Request.method
- The HTTP request method to use. This value is used by
- :meth:`~Request.get_method` to override the computed HTTP request
- method that would otherwise be returned. This attribute is initialized with
- the value of the *method* argument passed to the constructor.
+ The HTTP request method to use. By default its value is :const:`None`,
+ which means that :meth:`~Request.get_method` will do its normal computation
+ of the method to be used. Its value can be set (thus overriding the default
+ computation in :meth:`~Request.get_method`) either by providing a default
+ value by setting it at the class level in a :class:`Request` subclass, or by
+ passing a value in to the :class:`Request` constructor via the *method*
+ argument.
.. versionadded:: 3.3
+ .. versionchanged:: 3.4
+ A default value can now be set in subclasses; previously it could only
+ be set via the constructor argument.
+
.. method:: Request.get_method()
@@ -461,65 +488,29 @@ request.
unredirected).
-.. method:: Request.get_full_url()
-
- Return the URL given in the constructor.
-
-
-.. method:: Request.set_proxy(host, type)
-
- Prepare the request by connecting to a proxy server. The *host* and *type* will
- replace those of the instance, and the instance's selector will be the original
- URL given in the constructor.
-
-
-.. method:: Request.add_data(data)
-
- Set the :class:`Request` data to *data*. This is ignored by all handlers except
- HTTP handlers --- and there it should be a byte string, and will change the
- request to be ``POST`` rather than ``GET``. Deprecated in 3.3, use
- :attr:`Request.data`.
-
- .. deprecated-removed:: 3.3 3.4
-
-
-.. method:: Request.has_data()
-
- Return whether the instance has a non-\ ``None`` data. Deprecated in 3.3,
- use :attr:`Request.data`.
-
- .. deprecated-removed:: 3.3 3.4
-
+.. method:: Request.remove_header(header)
-.. method:: Request.get_data()
+ Remove named header from the request instance (both from regular and
+ unredirected headers).
- Return the instance's data. Deprecated in 3.3, use :attr:`Request.data`.
+ .. versionadded:: 3.4
- .. deprecated-removed:: 3.3 3.4
-
-
-.. method:: Request.get_type()
-
- Return the type of the URL --- also known as the scheme. Deprecated in 3.3,
- use :attr:`Request.type`.
-
- .. deprecated-removed:: 3.3 3.4
+.. method:: Request.get_full_url()
-.. method:: Request.get_host()
+ Return the URL given in the constructor.
- Return the host to which a connection will be made. Deprecated in 3.3, use
- :attr:`Request.host`.
+ .. versionchanged:: 3.4
- .. deprecated-removed:: 3.3 3.4
+ Returns :attr:`Request.full_url`
-.. method:: Request.get_selector()
+.. method:: Request.set_proxy(host, type)
- Return the selector --- the part of the URL that is sent to the server.
- Deprecated in 3.3, use :attr:`Request.selector`.
+ Prepare the request by connecting to a proxy server. The *host* and *type* will
+ replace those of the instance, and the instance's selector will be the original
+ URL given in the constructor.
- .. deprecated-removed:: 3.3 3.4
.. method:: Request.get_header(header_name, default=None)
@@ -531,25 +522,10 @@ request.
Return a list of tuples (header_name, header_value) of the Request headers.
-
-.. method:: Request.set_proxy(host, type)
-
-.. method:: Request.get_origin_req_host()
-
- Return the request-host of the origin transaction, as defined by
- :rfc:`2965`. See the documentation for the :class:`Request` constructor.
- Deprecated in 3.3, use :attr:`Request.origin_req_host`.
-
- .. deprecated-removed:: 3.3 3.4
-
-
-.. method:: Request.is_unverifiable()
-
- Return whether the request is unverifiable, as defined by RFC 2965. See the
- documentation for the :class:`Request` constructor. Deprecated in 3.3, use
- :attr:`Request.unverifiable`.
-
- .. deprecated-removed:: 3.3 3.4
+.. versionchanged:: 3.4
+ The request methods add_data, has_data, get_data, get_type, get_host,
+ get_selector, get_origin_req_host and is_unverifiable that were deprecated
+ since 3.3 have been removed.
.. _opener-director-objects:
@@ -983,6 +959,21 @@ FileHandler Objects
hostname is given, an :exc:`~urllib.error.URLError` is raised.
+.. _data-handler-objects:
+
+DataHandler Objects
+-------------------
+
+.. method:: DataHandler.data_open(req)
+
+ Read a data URL. This kind of URL contains the content encoded in the URL
+ itself. The data URL syntax is specified in :rfc:`2397`. This implementation
+ ignores white spaces in base64 encoded data URLs so the URL may be wrapped
+ in whatever source file it comes from. But even though some browsers don't
+ mind about a missing padding at the end of a base64 encoded data URL, this
+ implementation will raise an :exc:`ValueError` in that case.
+
+
.. _ftp-handler-objects:
FTPHandler Objects
@@ -1398,7 +1389,9 @@ some point in the future.
pair: FTP; protocol
* Currently, only the following protocols are supported: HTTP (versions 0.9 and
- 1.0), FTP, and local files.
+ 1.0), FTP, local files, and data URLs.
+
+ .. versionchanged:: 3.4 Added support for data URLs.
* The caching feature of :func:`urlretrieve` has been disabled until someone
finds the time to hack proper processing of Expiration time headers.
diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst
index 96ca2f5..e93f48e 100644
--- a/Doc/library/venv.rst
+++ b/Doc/library/venv.rst
@@ -76,6 +76,8 @@ Creating virtual environments
without there needing to be any reference to its venv in ``PATH``.
+.. _venv-api:
+
API
---
@@ -85,7 +87,8 @@ The high-level method described above makes use of a simple API which provides
mechanisms for third-party virtual environment creators to customize environment
creation according to their needs, the :class:`EnvBuilder` class.
-.. class:: EnvBuilder(system_site_packages=False, clear=False, symlinks=False, upgrade=False)
+.. class:: EnvBuilder(system_site_packages=False, clear=False, \
+ symlinks=False, upgrade=False, with_pip=False)
The :class:`EnvBuilder` class accepts the following keyword arguments on
instantiation:
@@ -93,8 +96,8 @@ creation according to their needs, the :class:`EnvBuilder` class.
* ``system_site_packages`` -- a Boolean value indicating that the system Python
site-packages should be available to the environment (defaults to ``False``).
- * ``clear`` -- a Boolean value which, if true, will delete any existing target
- directory instead of raising an exception (defaults to ``False``).
+ * ``clear`` -- a Boolean value which, if true, will delete the contents of
+ any existing target directory, before creating the environment.
* ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the
Python binary (and any necessary DLLs or other binaries,
@@ -105,6 +108,12 @@ creation according to their needs, the :class:`EnvBuilder` class.
environment with the running Python - for use when that Python has been
upgraded in-place (defaults to ``False``).
+ * ``with_pip`` -- a Boolean value which, if true, ensures pip is
+ installed in the virtual environment. This uses :mod:`ensurepip` with
+ the ``--default-pip`` option.
+
+ .. versionchanged:: 3.4
+ Added the ``with_pip`` parameter
Creators of third-party virtual environment tools will be free to use the
@@ -188,6 +197,9 @@ creation according to their needs, the :class:`EnvBuilder` class.
* ``__VENV_NAME__`` is replaced with the environment name (final path
segment of environment directory).
+ * ``__VENV_PROMPT__`` is replaced with the prompt (the environment
+ name surrounded by parentheses and with a following space)
+
* ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory
(either ``bin`` or ``Scripts``).
@@ -199,11 +211,15 @@ creation according to their needs, the :class:`EnvBuilder` class.
There is also a module-level convenience function:
-.. function:: create(env_dir, system_site_packages=False, clear=False, symlinks=False)
+.. function:: create(env_dir, system_site_packages=False, clear=False, \
+ symlinks=False, with_pip=False)
Create an :class:`EnvBuilder` with the given keyword arguments, and call its
:meth:`~EnvBuilder.create` method with the *env_dir* argument.
+ .. versionchanged:: 3.4
+ Added the ``with_pip`` parameter
+
An example of extending ``EnvBuilder``
--------------------------------------
diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst
index afafb45..ab64978 100644
--- a/Doc/library/wave.rst
+++ b/Doc/library/wave.rst
@@ -19,26 +19,32 @@ The :mod:`wave` module defines the following function and exception:
.. function:: open(file, mode=None)
If *file* is a string, open the file by that name, otherwise treat it as a
- seekable file-like object. *mode* can be any of
+ file-like object. *mode* can be:
- ``'r'``, ``'rb'``
+ ``'rb'``
Read only mode.
- ``'w'``, ``'wb'``
+ ``'wb'``
Write only mode.
Note that it does not allow read/write WAV files.
- A *mode* of ``'r'`` or ``'rb'`` returns a :class:`Wave_read` object, while a
- *mode* of ``'w'`` or ``'wb'`` returns a :class:`Wave_write` object. If
- *mode* is omitted and a file-like object is passed as *file*, ``file.mode``
- is used as the default value for *mode* (the ``'b'`` flag is still added if
- necessary).
+ A *mode* of ``'rb'`` returns a :class:`Wave_read` object, while a *mode* of
+ ``'wb'`` returns a :class:`Wave_write` object. If *mode* is omitted and a
+ file-like object is passed as *file*, ``file.mode`` is used as the default
+ value for *mode*.
If you pass in a file-like object, the wave object will not close it when its
:meth:`close` method is called; it is the caller's responsibility to close
the file object.
+ The :func:`.open` function may be used in a :keyword:`with` statement. When
+ the :keyword:`with` block completes, the :meth:`Wave_read.close()
+ <wave.Wave_read.close>` or :meth:`Wave_write.close()
+ <wave.Wave_write.close()>` method is called.
+
+ .. versionchanged:: 3.4
+ Added support for unseekable files.
.. function:: openfp(file, mode)
@@ -98,8 +104,9 @@ Wave_read objects, as returned by :func:`.open`, have the following methods:
.. method:: Wave_read.getparams()
- Returns a tuple ``(nchannels, sampwidth, framerate, nframes, comptype,
- compname)``, equivalent to output of the :meth:`get\*` methods.
+ Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth,
+ framerate, nframes, comptype, compname)``, equivalent to output of the
+ :meth:`get\*` methods.
.. method:: Wave_read.readframes(n)
@@ -143,13 +150,30 @@ them, and is otherwise implementation dependent.
Wave_write Objects
------------------
+For seekable output streams, the ``wave`` header will automatically be updated
+to reflect the number of frames actually written. For unseekable streams, the
+*nframes* value must be accurate when the first frame data is written. An
+accurate *nframes* value can be achieved either by calling
+:meth:`~Wave_write.setnframes` or :meth:`~Wave_write.setparams` with the number
+of frames that will be written before :meth:`~Wave_write.close` is called and
+then using :meth:`~Wave_write.writeframesraw` to write the frame data, or by
+calling :meth:`~Wave_write.writeframes` with all of the frame data to be
+written. In the latter case :meth:`~Wave_write.writeframes` will calculate
+the number of frames in the data and set *nframes* accordingly before writing
+the frame data.
+
Wave_write objects, as returned by :func:`.open`, have the following methods:
+.. versionchanged:: 3.4
+ Added support for unseekable files.
+
.. method:: Wave_write.close()
Make sure *nframes* is correct, and close the file if it was opened by
- :mod:`wave`. This method is called upon object collection.
+ :mod:`wave`. This method is called upon object collection. It will raise
+ an exception if the output stream is not seekable and *nframes* does not
+ match the number of frames actually written.
.. method:: Wave_write.setnchannels(n)
@@ -173,8 +197,9 @@ Wave_write objects, as returned by :func:`.open`, have the following methods:
.. method:: Wave_write.setnframes(n)
- Set the number of frames to *n*. This will be changed later if more frames are
- written.
+ Set the number of frames to *n*. This will be changed later if the number
+ of frames actually written is different (this update attempt will
+ raise an error if the output stream is not seekable).
.. method:: Wave_write.setcomptype(type, name)
@@ -200,10 +225,19 @@ Wave_write objects, as returned by :func:`.open`, have the following methods:
Write audio frames, without correcting *nframes*.
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
+
.. method:: Wave_write.writeframes(data)
- Write audio frames and make sure *nframes* is correct.
+ Write audio frames and make sure *nframes* is correct. It will raise an
+ error if the output stream is not seekable and the total number of frames
+ that have been written after *data* has been written does not match the
+ previously set value for *nframes*.
+
+ .. versionchanged:: 3.4
+ Any :term:`bytes-like object` is now accepted.
Note that it is invalid to set any parameters after calling :meth:`writeframes`
diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst
index 224f442..9ca60a9 100644
--- a/Doc/library/weakref.rst
+++ b/Doc/library/weakref.rst
@@ -51,10 +51,16 @@ garbage collection. :class:`WeakSet` implements the :class:`set` interface,
but keeps weak references to its elements, just like a
:class:`WeakKeyDictionary` does.
-Most programs should find that using one of these weak container types is all
-they need -- it's not usually necessary to create your own weak references
-directly. The low-level machinery used by the weak dictionary implementations
-is exposed by the :mod:`weakref` module for the benefit of advanced uses.
+:class:`finalize` provides a straight forward way to register a
+cleanup function to be called when an object is garbage collected.
+This is simpler to use than setting up a callback function on a raw
+weak reference, since the module automatically ensures that the finalizer
+remains alive until the object is collected.
+
+Most programs should find that using one of these weak container types
+or :class:`finalize` is all they need -- it's not usually necessary to
+create your own weak references directly. The low-level machinery is
+exposed by the :mod:`weakref` module for the benefit of advanced uses.
Not all objects can be weakly referenced; those objects which can include class
instances, functions written in Python (but not in C), instance methods, sets,
@@ -111,6 +117,15 @@ Extension types can easily be made to support weak references; see
This is a subclassable type rather than a factory function.
+ .. attribute:: __callback__
+
+ This read-only attribute returns the callback currently associated to the
+ weakref. If there is no callback or if the referent of the weakref is
+ no longer alive then this attribute will have value ``None``.
+
+ .. versionchanged:: 3.4
+ Added the :attr:`__callback__` attribute.
+
.. function:: proxy(object[, callback])
@@ -192,6 +207,98 @@ These method have the same issues as the and :meth:`keyrefs` method of
discarded when no strong reference to it exists any more.
+.. class:: WeakMethod(method)
+
+ A custom :class:`ref` subclass which simulates a weak reference to a bound
+ method (i.e., a method defined on a class and looked up on an instance).
+ Since a bound method is ephemeral, a standard weak reference cannot keep
+ hold of it. :class:`WeakMethod` has special code to recreate the bound
+ method until either the object or the original function dies::
+
+ >>> class C:
+ ... def method(self):
+ ... print("method called!")
+ ...
+ >>> c = C()
+ >>> r = weakref.ref(c.method)
+ >>> r()
+ >>> r = weakref.WeakMethod(c.method)
+ >>> r()
+ <bound method C.method of <__main__.C object at 0x7fc859830220>>
+ >>> r()()
+ method called!
+ >>> del c
+ >>> gc.collect()
+ 0
+ >>> r()
+ >>>
+
+ .. versionadded:: 3.4
+
+.. class:: finalize(obj, func, *args, **kwargs)
+
+ Return a callable finalizer object which will be called when *obj*
+ is garbage collected. Unlike an ordinary weak reference, a finalizer
+ will always survive until the reference object is collected, greatly
+ simplifying lifecycle management.
+
+ A finalizer is considered *alive* until it is called (either explicitly
+ or at garbage collection), and after that it is *dead*. Calling a live
+ finalizer returns the result of evaluating ``func(*arg, **kwargs)``,
+ whereas calling a dead finalizer returns :const:`None`.
+
+ Exceptions raised by finalizer callbacks during garbage collection
+ will be shown on the standard error output, but cannot be
+ propagated. They are handled in the same way as exceptions raised
+ from an object's :meth:`__del__` method or a weak reference's
+ callback.
+
+ When the program exits, each remaining live finalizer is called
+ unless its :attr:`atexit` attribute has been set to false. They
+ are called in reverse order of creation.
+
+ A finalizer will never invoke its callback during the later part of
+ the interpreter shutdown when module globals are liable to have
+ been replaced by :const:`None`.
+
+ .. method:: __call__()
+
+ If *self* is alive then mark it as dead and return the result of
+ calling ``func(*args, **kwargs)``. If *self* is dead then return
+ :const:`None`.
+
+ .. method:: detach()
+
+ If *self* is alive then mark it as dead and return the tuple
+ ``(obj, func, args, kwargs)``. If *self* is dead then return
+ :const:`None`.
+
+ .. method:: peek()
+
+ If *self* is alive then return the tuple ``(obj, func, args,
+ kwargs)``. If *self* is dead then return :const:`None`.
+
+ .. attribute:: alive
+
+ Property which is true if the finalizer is alive, false otherwise.
+
+ .. attribute:: atexit
+
+ A writable boolean property which by default is true. When the
+ program exits, it calls all remaining live finalizers for which
+ :attr:`.atexit` is true. They are called in reverse order of
+ creation.
+
+ .. note::
+
+ It is important to ensure that *func*, *args* and *kwargs* do
+ not own any references to *obj*, either directly or indirectly,
+ since otherwise *obj* will never be garbage collected. In
+ particular, *func* should not be a bound method of *obj*.
+
+ .. versionadded:: 3.4
+
+
.. data:: ReferenceType
The type object for weak references objects.
@@ -232,8 +339,9 @@ These method have the same issues as the and :meth:`keyrefs` method of
Weak Reference Objects
----------------------
-Weak reference objects have no attributes or methods, but do allow the referent
-to be obtained, if it still exists, by calling it:
+Weak reference objects have no methods and no attributes besides
+:attr:`ref.__callback__`. A weak reference object allows the referent to be
+obtained, if it still exists, by calling it:
>>> import weakref
>>> class Object:
@@ -326,3 +434,140 @@ objects can still be retrieved by ID if they do.
def id2obj(oid):
return _id2obj_dict[oid]
+
+.. _finalize-examples:
+
+Finalizer Objects
+-----------------
+
+The main benefit of using :class:`finalize` is that it makes it simple
+to register a callback without needing to preserve the returned finalizer
+object. For instance
+
+ >>> import weakref
+ >>> class Object:
+ ... pass
+ ...
+ >>> kenny = Object()
+ >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS
+ <finalize object at ...; for 'Object' at ...>
+ >>> del kenny
+ You killed Kenny!
+
+The finalizer can be called directly as well. However the finalizer
+will invoke the callback at most once.
+
+ >>> def callback(x, y, z):
+ ... print("CALLBACK")
+ ... return x + y + z
+ ...
+ >>> obj = Object()
+ >>> f = weakref.finalize(obj, callback, 1, 2, z=3)
+ >>> assert f.alive
+ >>> assert f() == 6
+ CALLBACK
+ >>> assert not f.alive
+ >>> f() # callback not called because finalizer dead
+ >>> del obj # callback not called because finalizer dead
+
+You can unregister a finalizer using its :meth:`~finalize.detach`
+method. This kills the finalizer and returns the arguments passed to
+the constructor when it was created.
+
+ >>> obj = Object()
+ >>> f = weakref.finalize(obj, callback, 1, 2, z=3)
+ >>> f.detach() #doctest:+ELLIPSIS
+ (<__main__.Object object ...>, <function callback ...>, (1, 2), {'z': 3})
+ >>> newobj, func, args, kwargs = _
+ >>> assert not f.alive
+ >>> assert newobj is obj
+ >>> assert func(*args, **kwargs) == 6
+ CALLBACK
+
+Unless you set the :attr:`~finalize.atexit` attribute to
+:const:`False`, a finalizer will be called when the program exits if it
+is still alive. For instance
+
+ >>> obj = Object()
+ >>> weakref.finalize(obj, print, "obj dead or exiting") #doctest:+ELLIPSIS
+ <finalize object at ...; for 'Object' at ...>
+ >>> exit() #doctest:+SKIP
+ obj dead or exiting
+
+
+Comparing finalizers with :meth:`__del__` methods
+-------------------------------------------------
+
+Suppose we want to create a class whose instances represent temporary
+directories. The directories should be deleted with their contents
+when the first of the following events occurs:
+
+* the object is garbage collected,
+* the object's :meth:`remove` method is called, or
+* the program exits.
+
+We might try to implement the class using a :meth:`__del__` method as
+follows::
+
+ class TempDir:
+ def __init__(self):
+ self.name = tempfile.mkdtemp()
+
+ def remove(self):
+ if self.name is not None:
+ shutil.rmtree(self.name)
+ self.name = None
+
+ @property
+ def removed(self):
+ return self.name is None
+
+ def __del__(self):
+ self.remove()
+
+Starting with Python 3.4, :meth:`__del__` methods no longer prevent
+reference cycles from being garbage collected, and module globals are
+no longer forced to :const:`None` during interpreter shutdown. So this
+code should work without any issues on CPython.
+
+However, handling of :meth:`__del__` methods is notoriously implementation
+specific, since it depends on internal details of the interpreter's garbage
+collector implementation.
+
+A more robust alternative can be to define a finalizer which only references
+the specific functions and objects that it needs, rather than having access
+to the full state of the object::
+
+ class TempDir:
+ def __init__(self):
+ self.name = tempfile.mkdtemp()
+ self._finalizer = weakref.finalize(self, shutil.rmtree, self.name)
+
+ def remove(self):
+ self._finalizer()
+
+ @property
+ def removed(self):
+ return not self._finalizer.alive
+
+Defined like this, our finalizer only receives a reference to the details
+it needs to clean up the directory appropriately. If the object never gets
+garbage collected the finalizer will still be called at exit.
+
+The other advantage of weakref based finalizers is that they can be used to
+register finalizers for classes where the definition is controlled by a
+third party, such as running code when a module is unloaded::
+
+ import weakref, sys
+ def unloading_module():
+ # implicit reference to the module globals from the function body
+ weakref.finalize(sys.modules[__name__], unloading_module)
+
+
+.. note::
+
+ If you create a finalizer object in a daemonic thread just as the
+ the program exits then there is the possibility that the finalizer
+ does not get called at exit. However, in a daemonic thread
+ :func:`atexit.register`, ``try: ... finally: ...`` and ``with: ...``
+ do not guarantee that cleanup occurs either.
diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
index 3f2dcbb..0617371 100644
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -105,6 +105,46 @@ Children are nested, and we can access specific child nodes by index::
>>> root[0][1].text
'2008'
+
+.. _elementtree-pull-parsing:
+
+Pull API for non-blocking parsing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Most parsing functions provided by this module require the whole document
+to be read at once before returning any result. It is possible to use an
+:class:`XMLParser` and feed data into it incrementally, but it is a push API that
+calls methods on a callback target, which is too low-level and inconvenient for
+most needs. Sometimes what the user really wants is to be able to parse XML
+incrementally, without blocking operations, while enjoying the convenience of
+fully constructed :class:`Element` objects.
+
+The most powerful tool for doing this is :class:`XMLPullParser`. It does not
+require a blocking read to obtain the XML data, and is instead fed with data
+incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML
+elements, call :meth:`XMLPullParser.read_events`. Here is an example::
+
+ >>> parser = ET.XMLPullParser(['start', 'end'])
+ >>> parser.feed('<mytag>sometext')
+ >>> list(parser.read_events())
+ [('start', <Element 'mytag' at 0x7fa66db2be58>)]
+ >>> parser.feed(' more text</mytag>')
+ >>> for event, elem in parser.read_events():
+ ... print(event)
+ ... print(elem.tag, 'text=', elem.text)
+ ...
+ end
+
+The obvious use case is applications that operate in a non-blocking fashion
+where the XML data is being received from a socket or read incrementally from
+some storage device. In such cases, blocking reads are unacceptable.
+
+Because it's so flexible, :class:`XMLPullParser` can be inconvenient to use for
+simpler use-cases. If you don't mind your application blocking on reading XML
+data but would still like to have incremental parsing capabilities, take a look
+at :func:`iterparse`. It can be useful when you're reading a large XML document
+and don't want to hold it wholly in memory.
+
Finding interesting elements
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -285,7 +325,7 @@ Supported XPath syntax
+=======================+======================================================+
| ``tag`` | Selects all child elements with the given tag. |
| | For example, ``spam`` selects all child elements |
-| | named ``spam``, ``spam/egg`` selects all |
+| | named ``spam``, and ``spam/egg`` selects all |
| | grandchildren named ``egg`` in all children named |
| | ``spam``. |
+-----------------------+------------------------------------------------------+
@@ -379,25 +419,32 @@ Functions
Parses an XML section into an element tree incrementally, and reports what's
going on to the user. *source* is a filename or :term:`file object`
- containing XML data. *events* is a tuple of events to report back. The
- supported events are the strings ``"start"``, ``"end"``, ``"start-ns"``
- and ``"end-ns"`` (the "ns" events are used to get detailed namespace
+ containing XML data. *events* is a sequence of events to report back. The
+ supported events are the strings ``"start"``, ``"end"``, ``"start-ns"`` and
+ ``"end-ns"`` (the "ns" events are used to get detailed namespace
information). If *events* is omitted, only ``"end"`` events are reported.
*parser* is an optional parser instance. If not given, the standard
- :class:`XMLParser` parser is used. *parser* can only use the default
- :class:`TreeBuilder` as a target. Returns an :term:`iterator` providing
- ``(event, elem)`` pairs.
+ :class:`XMLParser` parser is used. *parser* must be a subclass of
+ :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a
+ target. Returns an :term:`iterator` providing ``(event, elem)`` pairs.
+
+ Note that while :func:`iterparse` builds the tree incrementally, it issues
+ blocking reads on *source* (or the file it names). As such, it's unsuitable
+ for applications where blocking reads can't be made. For fully non-blocking
+ parsing, see :class:`XMLPullParser`.
.. note::
- :func:`iterparse` only guarantees that it has seen the ">"
- character of a starting tag when it emits a "start" event, so the
- attributes are defined, but the contents of the text and tail attributes
- are undefined at that point. The same applies to the element children;
- they may or may not be present.
+ :func:`iterparse` only guarantees that it has seen the ">" character of a
+ starting tag when it emits a "start" event, so the attributes are defined,
+ but the contents of the text and tail attributes are undefined at that
+ point. The same applies to the element children; they may or may not be
+ present.
If you need a fully populated element, look for "end" events instead.
+ .. deprecated:: 3.4
+ The *parser* argument.
.. function:: parse(source, parser=None)
@@ -438,29 +485,39 @@ Functions
arguments. Returns an element instance.
-.. function:: tostring(element, encoding="us-ascii", method="xml")
+.. function:: tostring(element, encoding="us-ascii", method="xml", *, \
+ short_empty_elements=True)
Generates a string representation of an XML element, including all
subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is
the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to
generate a Unicode string (otherwise, a bytestring is generated). *method*
is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``).
+ *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`.
Returns an (optionally) encoded string containing the XML data.
+ .. versionadded:: 3.4
+ The *short_empty_elements* parameter.
-.. function:: tostringlist(element, encoding="us-ascii", method="xml")
+
+.. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \
+ short_empty_elements=True)
Generates a string representation of an XML element, including all
subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is
the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to
generate a Unicode string (otherwise, a bytestring is generated). *method*
is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``).
+ *short_empty_elements* has the same meaning as in :meth:`ElementTree.write`.
Returns a list of (optionally) encoded strings containing the XML data.
It does not guarantee any specific sequence, except that
``b"".join(tostringlist(element)) == tostring(element)``.
.. versionadded:: 3.2
+ .. versionadded:: 3.4
+ The *short_empty_elements* parameter.
+
.. function:: XML(text, parser=None)
@@ -753,7 +810,8 @@ ElementTree Objects
.. method:: write(file, encoding="us-ascii", xml_declaration=None, \
- default_namespace=None, method="xml")
+ default_namespace=None, method="xml", *, \
+ short_empty_elements=True)
Writes the element tree to a file, as XML. *file* is a file name, or a
:term:`file object` opened for writing. *encoding* [1]_ is the output
@@ -764,6 +822,10 @@ ElementTree Objects
*default_namespace* sets the default XML namespace (for "xmlns").
*method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is
``"xml"``).
+ The keyword-only *short_empty_elements* parameter controls the formatting
+ of elements that contain no content. If *True* (the default), they are
+ emitted as a single self-closed tag, otherwise they are emitted as a pair
+ of start/end tags.
The output is either a string (:class:`str`) or binary (:class:`bytes`).
This is controlled by the *encoding* argument. If *encoding* is
@@ -772,6 +834,9 @@ ElementTree Objects
:term:`file object`; make sure you do not try to write a string to a
binary stream and vice versa.
+ .. versionadded:: 3.4
+ The *short_empty_elements* parameter.
+
This is the XML file that is going to be manipulated::
@@ -817,6 +882,7 @@ QName Objects
:class:`QName` instances are opaque.
+
.. _elementtree-treebuilder-objects:
TreeBuilder Objects
@@ -876,13 +942,18 @@ XMLParser Objects
.. class:: XMLParser(html=0, target=None, encoding=None)
- :class:`Element` structure builder for XML source data, based on the expat
- parser. *html* are predefined HTML entities. This flag is not supported by
- the current implementation. *target* is the target object. If omitted, the
- builder uses an instance of the standard :class:`TreeBuilder` class.
- *encoding* [1]_ is optional. If given, the value overrides the encoding
+ This class is the low-level building block of the module. It uses
+ :mod:`xml.parsers.expat` for efficient, event-based parsing of XML. It can
+ be fed XML data incrementall with the :meth:`feed` method, and parsing events
+ are translated to a push API - by invoking callbacks on the *target* object.
+ If *target* is omitted, the standard :class:`TreeBuilder` is used. The
+ *html* argument was historically used for backwards compatibility and is now
+ deprecated. If *encoding* [1]_ is given, the value overrides the encoding
specified in the XML file.
+ .. deprecated:: 3.4
+ The *html* argument. The remaining arguments should be passed via
+ keywword to prepare for the removal of the *html* argument.
.. method:: close()
@@ -902,12 +973,12 @@ XMLParser Objects
Feeds data to the parser. *data* is encoded data.
- :meth:`XMLParser.feed` calls *target*\'s ``start()`` method
- for each opening tag, its ``end()`` method for each closing tag,
- and data is processed by method ``data()``. :meth:`XMLParser.close`
- calls *target*\'s method ``close()``.
- :class:`XMLParser` can be used not only for building a tree structure.
- This is an example of counting the maximum depth of an XML file::
+ :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method
+ for each opening tag, its ``end(tag)`` method for each closing tag, and data
+ is processed by method ``data(data)``. :meth:`XMLParser.close` calls
+ *target*\'s method ``close()``. :class:`XMLParser` can be used not only for
+ building a tree structure. This is an example of counting the maximum depth
+ of an XML file::
>>> from xml.etree.ElementTree import XMLParser
>>> class MaxDepth: # The target object of the parser
@@ -941,6 +1012,60 @@ XMLParser Objects
>>> parser.close()
4
+
+.. _elementtree-xmlpullparser-objects:
+
+XMLPullParser Objects
+^^^^^^^^^^^^^^^^^^^^^
+
+.. class:: XMLPullParser(events=None)
+
+ A pull parser suitable for non-blocking applications. Its input-side API is
+ similar to that of :class:`XMLParser`, but instead of pushing calls to a
+ callback target, :class:`XMLPullParser` collects an internal list of parsing
+ events and lets the user read from it. *events* is a sequence of events to
+ report back. The supported events are the strings ``"start"``, ``"end"``,
+ ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed
+ namespace information). If *events* is omitted, only ``"end"`` events are
+ reported.
+
+ .. method:: feed(data)
+
+ Feed the given bytes data to the parser.
+
+ .. method:: close()
+
+ Signal the parser that the data stream is terminated. Unlike
+ :meth:`XMLParser.close`, this method always returns :const:`None`.
+ Any events not yet retrieved when the parser is closed can still be
+ read with :meth:`read_events`.
+
+ .. method:: read_events()
+
+ Return an iterator over the events which have been encountered in the
+ data fed to the
+ parser. The iterator yields ``(event, elem)`` pairs, where *event* is a
+ string representing the type of event (e.g. ``"end"``) and *elem* is the
+ encountered :class:`Element` object.
+
+ Events provided in a previous call to :meth:`read_events` will not be
+ yielded again. Events are consumed from the internal queue only when
+ they are retrieved from the iterator, so multiple readers iterating in
+ parallel over iterators obtained from :meth:`read_events` will have
+ unpredictable results.
+
+ .. note::
+
+ :class:`XMLPullParser` only guarantees that it has seen the ">"
+ character of a starting tag when it emits a "start" event, so the
+ attributes are defined, but the contents of the text and tail attributes
+ are undefined at that point. The same applies to the element children;
+ they may or may not be present.
+
+ If you need a fully populated element, look for "end" events instead.
+
+ .. versionadded:: 3.4
+
Exceptions
^^^^^^^^^^
diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst
index 7a6482b..1d23a7c 100644
--- a/Doc/library/zipfile.rst
+++ b/Doc/library/zipfile.rst
@@ -130,7 +130,7 @@ ZipFile Objects
---------------
-.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False)
+.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True)
Open a ZIP file, where *file* can be either a path to a file (a string) or a
file-like object. The *mode* parameter should be ``'r'`` to read an existing
@@ -144,15 +144,12 @@ ZipFile Objects
and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`,
:const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized
values will cause :exc:`RuntimeError` to be raised. If :const:`ZIP_DEFLATED`,
- :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponded module
+ :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module
(:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError`
is also raised. The default is :const:`ZIP_STORED`. If *allowZip64* is
- ``True`` zipfile will create ZIP files that use the ZIP64 extensions when
- the zipfile is larger than 2 GiB. If it is false (the default) :mod:`zipfile`
+ ``True`` (the default) zipfile will create ZIP files that use the ZIP64
+ extensions when the zipfile is larger than 2 GiB. If it is false :mod:`zipfile`
will raise an exception when the ZIP file would require ZIP64 extensions.
- ZIP64 extensions are disabled by default because the default :program:`zip`
- and :program:`unzip` commands on Unix (the InfoZIP utilities) don't support
- these extensions.
If the file is created with mode ``'a'`` or ``'w'`` and then
:meth:`closed <close>` without adding any files to the archive, the appropriate
@@ -171,6 +168,9 @@ ZipFile Objects
.. versionchanged:: 3.3
Added support for :mod:`bzip2 <bz2>` and :mod:`lzma` compression.
+ .. versionchanged:: 3.4
+ ZIP64 extensions are enabled by default.
+
.. method:: ZipFile.close()
@@ -234,6 +234,9 @@ ZipFile Objects
or a :class:`ZipInfo` object. You will appreciate this when trying to read a
ZIP file that contains members with duplicate names.
+ .. deprecated-removed:: 3.4 3.6
+ The ``'U'`` or ``'rU'`` mode. Use :class:`io.TextIOWrapper` for reading
+ compressed text files in :term:`universal newlines` mode.
.. method:: ZipFile.extract(member, path=None, pwd=None)
@@ -266,10 +269,8 @@ ZipFile Objects
Never extract archives from untrusted sources without prior inspection.
It is possible that files are created outside of *path*, e.g. members
that have absolute filenames starting with ``"/"`` or filenames with two
- dots ``".."``.
-
- .. versionchanged:: 3.3.1
- The zipfile module attempts to prevent that. See :meth:`extract` note.
+ dots ``".."``. This module attempts to prevent that.
+ See :meth:`extract` note.
.. method:: ZipFile.printdir()
@@ -376,15 +377,18 @@ PyZipFile Objects
The :class:`PyZipFile` constructor takes the same parameters as the
:class:`ZipFile` constructor, and one additional parameter, *optimize*.
-.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=False, \
+.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \
optimize=-1)
.. versionadded:: 3.2
The *optimize* parameter.
+ .. versionchanged:: 3.4
+ ZIP64 extensions are enabled by default.
+
Instances have one method in addition to those of :class:`ZipFile` objects:
- .. method:: PyZipFile.writepy(pathname, basename='')
+ .. method:: PyZipFile.writepy(pathname, basename='', filterfunc=None)
Search for files :file:`\*.py` and add the corresponding file to the
archive.
@@ -397,16 +401,33 @@ The :class:`PyZipFile` constructor takes the same parameters as the
``2``, only files with that optimization level (see :func:`compile`) are
added to the archive, compiling if necessary.
- If the pathname is a file, the filename must end with :file:`.py`, and
+ If *pathname* is a file, the filename must end with :file:`.py`, and
just the (corresponding :file:`\*.py[co]`) file is added at the top level
- (no path information). If the pathname is a file that does not end with
+ (no path information). If *pathname* is a file that does not end with
:file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory,
and the directory is not a package directory, then all the files
:file:`\*.py[co]` are added at the top level. If the directory is a
package directory, then all :file:`\*.py[co]` are added under the package
name as a file path, and if any subdirectories are package directories,
- all of these are added recursively. *basename* is intended for internal
- use only. The :meth:`writepy` method makes archives with file names like
+ all of these are added recursively.
+
+ *basename* is intended for internal use only.
+
+ *filterfunc*, if given, must be a function taking a single string
+ argument. It will be passed each path (including each individual full
+ file path) before it is added to the archive. If *filterfunc* returns a
+ false value, the path will not be added, and if it is a directory its
+ contents will be ignored. For example, if our test files are all either
+ in ``test`` directories or start with the string ``test_``, we can use a
+ *filterfunc* to exclude them::
+
+ >>> zf = PyZipFile('myprog.zip')
+ >>> def notests(s):
+ ... fn = os.path.basename(s)
+ ... return (not (fn == 'test' or fn.startswith('test_')))
+ >>> zf.writepy('myprog', filterfunc=notests)
+
+ The :meth:`writepy` method makes archives with file names like
this::
string.pyc # Top level name
@@ -415,6 +436,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the
test/bogus/__init__.pyc # Subpackage directory
test/bogus/myfile.pyc # Submodule test.bogus.myfile
+ .. versionadded:: 3.4
+ The *filterfunc* parameter.
+
.. _zipinfo-objects:
diff --git a/Doc/license.rst b/Doc/license.rst
index 4440c7b..6190fe0 100644
--- a/Doc/license.rst
+++ b/Doc/license.rst
@@ -84,9 +84,9 @@ Terms and conditions for accessing or otherwise using Python
analyze, test, perform and/or display publicly, prepare derivative works,
distribute, and otherwise use Python |release| alone or in any derivative
version, provided, however, that PSF's License Agreement and PSF's notice of
- copyright, i.e., "Copyright © 2001-2014 Python Software Foundation; All Rights
- Reserved" are retained in Python |release| alone or in any derivative version
- prepared by Licensee.
+ copyright, i.e., "Copyright © 2001-2014 Python Software Foundation; All
+ Rights Reserved" are retained in Python |release| alone or in any derivative
+ version prepared by Licensee.
#. In the event Licensee prepares a derivative work that is based on or
incorporates Python |release| or any part thereof, and wants to make the
@@ -590,6 +590,35 @@ The :mod:`select` and contains the following notice for the kqueue interface::
SUCH DAMAGE.
+SipHash24
+---------
+
+The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of
+Dan Bernstein's SipHash24 algorithm. The contains the following note::
+
+ <MIT License>
+ Copyright (c) 2013 Marek Majkowski <marek@popcount.org>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ </MIT License>
+
+ Original location:
+ https://github.com/majek/csiphash/
+
+ Solution inspired by code from:
+ Samuel Neves (supercop/crypto_auth/siphash24/little)
+ djb (supercop/crypto_auth/siphash24/little2)
+ Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c)
+
+
strtod and dtoa
---------------
@@ -846,6 +875,47 @@ used for the build::
jloup@gzip.org madler@alumni.caltech.edu
+cfuhash
+-------
+
+The implementation of the hash table used by the :mod:`tracemalloc` is based
+on the cfuhash project::
+
+ Copyright (c) 2005 Don Owens
+ All rights reserved.
+
+ This code is released under the BSD license:
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * Neither the name of the author nor the names of its
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
libmpdec
--------
diff --git a/Doc/make.bat b/Doc/make.bat
index d6f7074..0b5b56e 100644
--- a/Doc/make.bat
+++ b/Doc/make.bat
@@ -1,7 +1,6 @@
@@echo off
setlocal
-set SVNROOT=http://svn.python.org/projects
if "%PYTHON%" EQU "" set PYTHON=py -2
if "%HTMLHELP%" EQU "" set HTMLHELP=%ProgramFiles%\HTML Help Workshop\hhc.exe
if "%DISTVERSION%" EQU "" for /f "usebackq" %%v in (`%PYTHON% tools/sphinxext/patchlevel.py`) do set DISTVERSION=%%v
@@ -14,15 +13,11 @@ if "%1" EQU "text" goto build
if "%1" EQU "suspicious" goto build
if "%1" EQU "linkcheck" goto build
if "%1" EQU "changes" goto build
-if "%1" EQU "checkout" goto checkout
-if "%1" EQU "update" goto update
:help
set this=%~n0
echo HELP
echo.
-echo %this% checkout
-echo %this% update
echo %this% html
echo %this% htmlhelp
echo %this% latex
@@ -33,20 +28,6 @@ echo %this% changes
echo.
goto end
-:checkout
-svn co %SVNROOT%/external/Sphinx-1.2/sphinx tools/sphinx
-svn co %SVNROOT%/external/docutils-0.11/docutils tools/docutils
-svn co %SVNROOT%/external/Jinja-2.3.1/jinja2 tools/jinja2
-svn co %SVNROOT%/external/Pygments-1.6/pygments tools/pygments
-goto end
-
-:update
-svn update tools/sphinx
-svn update tools/docutils
-svn update tools/jinja2
-svn update tools/pygments
-goto end
-
:build
if not exist build mkdir build
if not exist build\%1 mkdir build\%1
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index 8afc69e..e35aa9f 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -493,14 +493,15 @@ case the parameter's default value is substituted. If a parameter has a default
value, all following parameters up until the "``*``" must also have a default
value --- this is a syntactic restriction that is not expressed by the grammar.
-**Default parameter values are evaluated when the function definition is
-executed.** This means that the expression is evaluated once, when the function
-is defined, and that the same "pre-computed" value is used for each call. This
-is especially important to understand when a default parameter is a mutable
-object, such as a list or a dictionary: if the function modifies the object
-(e.g. by appending an item to a list), the default value is in effect modified.
-This is generally not what was intended. A way around this is to use ``None``
-as the default, and explicitly test for it in the body of the function, e.g.::
+**Default parameter values are evaluated from left to right when the function
+definition is executed.** This means that the expression is evaluated once, when
+the function is defined, and that the same "pre-computed" value is used for each
+call. This is especially important to understand when a default parameter is a
+mutable object, such as a list or a dictionary: if the function modifies the
+object (e.g. by appending an item to a list), the default value is in effect
+modified. This is generally not what was intended. A way around this is to use
+``None`` as the default, and explicitly test for it in the body of the function,
+e.g.::
def whats_on_the_telly(penguin=None):
if penguin is None:
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 160af30..8204dc3 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -934,6 +934,20 @@ Internal types
frame). A debugger can implement a Jump command (aka Set Next Statement)
by writing to f_lineno.
+ Frame objects support one method:
+
+ .. method:: frame.clear()
+
+ This method clears all references to local variables held by the
+ frame. Also, if the frame belonged to a generator, the generator
+ is finalized. This helps break reference cycles involving frame
+ objects (for example when catching an exception and storing its
+ traceback for later use).
+
+ :exc:`RuntimeError` is raised if the frame is currently executing.
+
+ .. versionadded:: 3.4
+
Traceback objects
.. index::
object: traceback
@@ -1121,12 +1135,10 @@ Basic customization
``sys.last_traceback`` keeps the stack frame alive). The first situation
can only be remedied by explicitly breaking the cycles; the latter two
situations can be resolved by storing ``None`` in ``sys.last_traceback``.
- Circular references which are garbage are detected when the option cycle
- detector is enabled (it's on by default), but can only be cleaned up if
- there are no Python- level :meth:`__del__` methods involved. Refer to the
- documentation for the :mod:`gc` module for more information about how
- :meth:`__del__` methods are handled by the cycle detector, particularly
- the description of the ``garbage`` value.
+ Circular references which are garbage are detected and cleaned up when
+ the cyclic garbage collector is enabled (it's on by default). Refer to the
+ documentation for the :mod:`gc` module for more information about this
+ topic.
.. warning::
@@ -1214,6 +1226,10 @@ Basic customization
The return value must be a string object.
+ .. versionchanged:: 3.4
+ The __format__ method of ``object`` itself raises a :exc:`TypeError`
+ if passed any non-empty string.
+
.. _richcmpfuncs:
.. method:: object.__lt__(self, other)
@@ -1631,6 +1647,8 @@ of these candidate metaclasses. If none of the candidate metaclasses meets
that criterion, then the class definition will fail with ``TypeError``.
+.. _prepare:
+
Preparing the class namespace
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1830,6 +1848,15 @@ through the container; for mappings, :meth:`__iter__` should be the same as
considered to be false in a Boolean context.
+.. method:: object.__length_hint__(self)
+
+ Called to implement :func:`operator.length_hint`. Should return an estimated
+ length for the object (which may be greater or less than the actual length).
+ The length must be an integer ``>=`` 0. This method is purely an
+ optimization and is never required for correctness.
+
+ .. versionadded:: 3.4
+
.. note::
Slicing is done exclusively with the following three methods. A call like ::
@@ -2061,9 +2088,17 @@ left undefined.
.. method:: object.__index__(self)
- Called to implement :func:`operator.index`. Also called whenever Python needs
- an integer object (such as in slicing, or in the built-in :func:`bin`,
- :func:`hex` and :func:`oct` functions). Must return an integer.
+ Called to implement :func:`operator.index`, and whenever Python needs to
+ losslessly convert the numeric object to an integer object (such as in
+ slicing, or in the built-in :func:`bin`, :func:`hex` and :func:`oct`
+ functions). Presence of this method indicates that the numeric object is
+ an integer type. Must return an integer.
+
+ .. note::
+
+ When :meth:`__index__` is defined, :meth:`__int__` should also be defined,
+ and both shuld return the same value, in order to have a coherent integer
+ type class.
.. _context-managers:
diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst
index eb497a1..645802d 100644
--- a/Doc/reference/import.rst
+++ b/Doc/reference/import.rst
@@ -210,6 +210,7 @@ Finders and loaders
.. index::
single: finder
single: loader
+ single: module spec
If the named module is not found in :data:`sys.modules`, then Python's import
protocol is invoked to find and load the module. This protocol consists of
@@ -230,13 +231,17 @@ The import machinery is extensible, so new finders can be added to extend the
range and scope of module searching.
Finders do not actually load modules. If they can find the named module, they
-return a :term:`loader`, which the import machinery then invokes to load the
-module and create the corresponding module object.
+return a :dfn:`module spec`, an encapsulation of the module's import-related
+information, which the import machinery then uses when loading the module.
The following sections describe the protocol for finders and loaders in more
detail, including how you can create and register new ones to extend the
import machinery.
+.. versionchanged:: 3.4
+ In previous versions of Python, finders returned :term:`loaders <loader>`
+ directly, whereas now they return module specs which *contain* loaders.
+ Loaders are still used during import but have fewer responsibilities.
Import hooks
------------
@@ -270,41 +275,43 @@ The meta path
.. index::
single: sys.meta_path
- pair: finder; find_module
- pair: finder; find_loader
+ pair: finder; find_spec
When the named module is not found in :data:`sys.modules`, Python next
searches :data:`sys.meta_path`, which contains a list of meta path finder
objects. These finders are queried in order to see if they know how to handle
the named module. Meta path finders must implement a method called
-:meth:`find_module()` which takes two arguments, a name and an import path.
-The meta path finder can use any strategy it wants to determine whether it can
-handle the named module or not.
+:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments:
+a name, an import path, and (optionally) a target module. The meta path
+finder can use any strategy it wants to determine whether it can handle
+the named module or not.
If the meta path finder knows how to handle the named module, it returns a
-loader object. If it cannot handle the named module, it returns ``None``. If
+spec object. If it cannot handle the named module, it returns ``None``. If
:data:`sys.meta_path` processing reaches the end of its list without returning
-a loader, then an :exc:`ImportError` is raised. Any other exceptions raised
+a spec, then an :exc:`ImportError` is raised. Any other exceptions raised
are simply propagated up, aborting the import process.
-The :meth:`find_module()` method of meta path finders is called with two
-arguments. The first is the fully qualified name of the module being
-imported, for example ``foo.bar.baz``. The second argument is the path
-entries to use for the module search. For top-level modules, the second
-argument is ``None``, but for submodules or subpackages, the second
-argument is the value of the parent package's ``__path__`` attribute. If
-the appropriate ``__path__`` attribute cannot be accessed, an
-:exc:`ImportError` is raised.
+The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path
+finders is called with two or three arguments. The first is the fully
+qualified name of the module being imported, for example ``foo.bar.baz``.
+The second argument is the path entries to use for the module search. For
+top-level modules, the second argument is ``None``, but for submodules or
+subpackages, the second argument is the value of the parent package's
+``__path__`` attribute. If the appropriate ``__path__`` attribute cannot
+be accessed, an :exc:`ImportError` is raised. The third argument is an
+existing module object that will be the target of loading later. The
+import system passes in a target module only during reload.
The meta path may be traversed multiple times for a single import request.
For example, assuming none of the modules involved has already been cached,
importing ``foo.bar.baz`` will first perform a top level import, calling
-``mpf.find_module("foo", None)`` on each meta path finder (``mpf``). After
+``mpf.find_spec("foo", None, None)`` on each meta path finder (``mpf``). After
``foo`` has been imported, ``foo.bar`` will be imported by traversing the
meta path a second time, calling
-``mpf.find_module("foo.bar", foo.__path__)``. Once ``foo.bar`` has been
+``mpf.find_spec("foo.bar", foo.__path__, None)``. Once ``foo.bar`` has been
imported, the final traversal will call
-``mpf.find_module("foo.bar.baz", foo.bar.__path__)``.
+``mpf.find_spec("foo.bar.baz", foo.bar.__path__, None)``.
Some meta path finders only support top level imports. These importers will
always return ``None`` when anything other than ``None`` is passed as the
@@ -315,129 +322,239 @@ knows how to import built-in modules, one that knows how to import frozen
modules, and one that knows how to import modules from an :term:`import path`
(i.e. the :term:`path based finder`).
+.. versionchanged:: 3.4
+ The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path
+ finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which
+ is now deprecated. While it will continue to work without change, the
+ import machinery will try it only if the finder does not implement
+ ``find_spec()``.
-Loaders
-=======
-
-If and when a module loader is found its
-:meth:`~importlib.abc.Loader.load_module` method is called, with a single
-argument, the fully qualified name of the module being imported. This method
-has several responsibilities, and should return the module object it has
-loaded [#fnlo]_. If it cannot load the module, it should raise an
-:exc:`ImportError`, although any other exception raised during
-:meth:`load_module()` will be propagated.
-In many cases, the finder and loader can be the same object; in such cases the
-:meth:`finder.find_module()` would just return ``self``.
+Loading
+=======
-Loaders must satisfy the following requirements:
+If and when a module spec is found, the import machinery will use it (and
+the loader it contains) when loading the module. Here is an approximation
+of what happens during the loading portion of import::
+
+ module = None
+ if spec.loader is not None and hasattr(spec.loader, 'create_module'):
+ module = spec.loader.create_module(spec)
+ if module is None:
+ module = ModuleType(spec.name)
+ # The import-related module attributes get set here:
+ _init_module_attrs(spec, module)
+
+ if spec.loader is None:
+ if spec.submodule_search_locations is not None:
+ # namespace package
+ sys.modules[spec.name] = module
+ else:
+ # unsupported
+ raise ImportError
+ elif not hasattr(spec.loader, 'exec_module'):
+ module = spec.loader.load_module(spec.name)
+ # Set __loader__ and __package__ if missing.
+ else:
+ sys.modules[spec.name] = module
+ try:
+ spec.loader.exec_module(module)
+ except BaseException:
+ try:
+ del sys.modules[spec.name]
+ except KeyError:
+ pass
+ raise
+ return sys.modules[spec.name]
+
+Note the following details:
* If there is an existing module object with the given name in
- :data:`sys.modules`, the loader must use that existing module. (Otherwise,
- :func:`imp.reload` will not work correctly.) If the named module does
- not exist in :data:`sys.modules`, the loader must create a new module
- object and add it to :data:`sys.modules`.
+ :data:`sys.modules`, import will have already returned it.
- Note that the module *must* exist in :data:`sys.modules` before the loader
+ * The module will exist in :data:`sys.modules` before the loader
executes the module code. This is crucial because the module code may
(directly or indirectly) import itself; adding it to :data:`sys.modules`
beforehand prevents unbounded recursion in the worst case and multiple
loading in the best.
- If loading fails, the loader must remove any modules it has inserted into
- :data:`sys.modules`, but it must remove **only** the failing module, and
- only if the loader itself has loaded it explicitly. Any module already in
- the :data:`sys.modules` cache, and any module that was successfully loaded
- as a side-effect, must remain in the cache.
-
- * The loader may set the ``__file__`` attribute of the module. If set, this
- attribute's value must be a string. The loader may opt to leave
- ``__file__`` unset if it has no semantic meaning (e.g. a module loaded from
- a database). If ``__file__`` is set, it may also be appropriate to set the
- ``__cached__`` attribute which is the path to any compiled version of the
- code (e.g. byte-compiled file). The file does not need to exist to set this
- attribute; the path can simply point to whether the compiled file would
- exist (see :pep:`3147`).
-
- * The loader may set the ``__name__`` attribute of the module. While not
- required, setting this attribute is highly recommended so that the
- :meth:`repr()` of the module is more informative.
-
- * If the module is a package (either regular or namespace), the loader must
- set the module object's ``__path__`` attribute. The value must be
- iterable, but may be empty if ``__path__`` has no further significance
- to the loader. If ``__path__`` is not empty, it must produce strings
- when iterated over. More details on the semantics of ``__path__`` are
- given :ref:`below <package-path-rules>`.
-
- * The ``__loader__`` attribute must be set to the loader object that loaded
- the module. This is mostly for introspection and reloading, but can be
- used for additional loader-specific functionality, for example getting
- data associated with a loader.
-
- * The module's ``__package__`` attribute should be set. Its value must be a
- string, but it can be the same value as its ``__name__``. If the attribute
- is set to ``None`` or is missing, the import system will fill it in with a
- more appropriate value. When the module is a package, its ``__package__``
- value should be set to its ``__name__``. When the module is not a package,
- ``__package__`` should be set to the empty string for top-level modules, or
- for submodules, to the parent package's name. See :pep:`366` for further
- details.
+ * If loading fails, the failing module -- and only the failing module --
+ gets removed from :data:`sys.modules`. Any module already in the
+ :data:`sys.modules` cache, and any module that was successfully loaded
+ as a side-effect, must remain in the cache. This contrasts with
+ reloading where even the failing module is left in :data:`sys.modules`.
- This attribute is used instead of ``__name__`` to calculate explicit
- relative imports for main modules, as defined in :pep:`366`.
+ * After the module is created but before execution, the import machinery
+ sets the import-related module attributes ("_init_module_attrs" in
+ the pseudo-code example above), as summarized in a
+ :ref:`later section <import-mod-attrs>`.
+
+ * Module execution is the key moment of loading in which the module's
+ namespace gets populated. Execution is entirely delegated to the
+ loader, which gets to decide what gets populated and how.
+
+ * The module created during loading and passed to exec_module() may
+ not be the one returned at the end of import [#fnlo]_.
+
+.. versionchanged:: 3.4
+ The import system has taken over the boilerplate responsibilities of
+ loaders. These were previously performed by the
+ :meth:`importlib.abc.Loader.load_module` method.
+
+Loaders
+-------
+
+Module loaders provide the critical function of loading: module execution.
+The import machinery calls the :meth:`importlib.abc.Loader.exec_module`
+method with a single argument, the module object to execute. Any value
+returned from :meth:`~importlib.abc.Loader.exec_module` is ignored.
+
+Loaders must satisfy the following requirements:
* If the module is a Python module (as opposed to a built-in module or a
dynamically loaded extension), the loader should execute the module's code
in the module's global name space (``module.__dict__``).
+ * If the loader cannot execute the module, it should raise an
+ :exc:`ImportError`, although any other exception raised during
+ :meth:`~importlib.abc.Loader.exec_module` will be propagated.
-Module reprs
-------------
+In many cases, the finder and loader can be the same object; in such cases the
+:meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a
+spec with the loader set to ``self``.
-By default, all modules have a usable repr, however depending on the
-attributes set above, and hooks in the loader, you can more explicitly control
-the repr of module objects.
-
-Loaders may implement a :meth:`module_repr()` method which takes a single
-argument, the module object. When ``repr(module)`` is called for a module
-with a loader supporting this protocol, whatever is returned from
-``module.__loader__.module_repr(module)`` is returned as the module's repr
-without further processing. This return value must be a string.
-
-If the module has no ``__loader__`` attribute, or the loader has no
-:meth:`module_repr()` method, then the module object implementation itself
-will craft a default repr using whatever information is available. It will
-try to use the ``module.__name__``, ``module.__file__``, and
-``module.__loader__`` as input into the repr, with defaults for whatever
-information is missing.
+Module loaders may opt in to creating the module object during loading
+by implementing a :meth:`~importlib.abc.Loader.create_module` method.
+It takes one argument, the module spec, and returns the new module object
+to use during loading. ``create_module()`` does not need to set any attributes
+on the module object. If the loader does not define ``create_module()``, the
+import machinery will create the new module itself.
-Here are the exact rules used:
+.. versionadded:: 3.4
+ The create_module() method of loaders.
- * If the module has a ``__loader__`` and that loader has a
- :meth:`module_repr()` method, call it with a single argument, which is the
- module object. The value returned is used as the module's repr.
+.. versionchanged:: 3.4
+ The :meth:`~importlib.abc.Loader.load_module` method was replaced by
+ :meth:`~importlib.abc.Loader.exec_module` and the import
+ machinery assumed all the boilerplate responsibilities of loading.
- * If an exception occurs in :meth:`module_repr()`, the exception is caught
- and discarded, and the calculation of the module's repr continues as if
- :meth:`module_repr()` did not exist.
+ For compatibility with existing loaders, the import machinery will use
+ the ``load_module()`` method of loaders if it exists and the loader does
+ not also implement ``exec_module()``. However, ``load_module()`` has been
+ deprecated and loaders should implement ``exec_module()`` instead.
- * If the module has a ``__file__`` attribute, this is used as part of the
- module's repr.
+ The ``load_module()`` method must implement all the boilerplate loading
+ functionality described above in addition to executing the module. All
+ the same constraints apply, with some additional clarification:
- * If the module has no ``__file__`` but does have a ``__loader__``, then the
- loader's repr is used as part of the module's repr.
+ * If there is an existing module object with the given name in
+ :data:`sys.modules`, the loader must use that existing module.
+ (Otherwise, :func:`importlib.reload` will not work correctly.) If the
+ named module does not exist in :data:`sys.modules`, the loader
+ must create a new module object and add it to :data:`sys.modules`.
- * Otherwise, just use the module's ``__name__`` in the repr.
+ * The module *must* exist in :data:`sys.modules` before the loader
+ executes the module code, to prevent unbounded recursion or multiple
+ loading.
+
+ * If loading fails, the loader must remove any modules it has inserted
+ into :data:`sys.modules`, but it must remove **only** the failing
+ module, and only if the loader itself has loaded it explicitly.
+
+Module spec
+-----------
+
+The import machinery uses a variety of information about each module
+during import, especially before loading. Most of the information is
+common to all modules. The purpose of a module's spec is to encapsulate
+this import-related information on a per-module basis.
+
+Using a spec during import allows state to be transferred between import
+system components, e.g. between the finder that creates the module spec
+and the loader that executes it. Most importantly, it allows the
+import machinery to perform the boilerplate operations of loading,
+whereas without a module spec the loader had that responsibility.
+
+See :class:`~importlib.machinery.ModuleSpec` for more specifics on what
+information a module's spec may hold.
+
+.. versionadded:: 3.4
+
+.. _import-mod-attrs:
+
+Import-related module attributes
+--------------------------------
+
+The import machinery fills in these attributes on each module object
+during loading, based on the module's spec, before the loader executes
+the module.
+
+.. attribute:: __name__
+
+ The ``__name__`` attribute must be set to the fully-qualified name of
+ the module. This name is used to uniquely identify the module in
+ the import system.
+
+.. attribute:: __loader__
+
+ The ``__loader__`` attribute must be set to the loader object that
+ the import machinery used when loading the module. This is mostly
+ for introspection, but can be used for additional loader-specific
+ functionality, for example getting data associated with a loader.
+
+.. attribute:: __package__
+
+ The module's ``__package__`` attribute must be set. Its value must
+ be a string, but it can be the same value as its ``__name__``. When
+ the module is a package, its ``__package__`` value should be set to
+ its ``__name__``. When the module is not a package, ``__package__``
+ should be set to the empty string for top-level modules, or for
+ submodules, to the parent package's name. See :pep:`366` for further
+ details.
-This example, from :pep:`420` shows how a loader can craft its own module
-repr::
+ This attribute is used instead of ``__name__`` to calculate explicit
+ relative imports for main modules, as defined in :pep:`366`.
+
+.. attribute:: __spec__
+
+ The ``__spec__`` attribute must be set to the module spec that was
+ used when importing the module. This is used primarily for
+ introspection and during reloading. Setting ``__spec__``
+ appropriately applies equally to :ref:`modules initialized during
+ interpreter startup <programs>`. The one exception is ``__main__``,
+ where ``__spec__`` is :ref:`set to None in some cases <main_spec>`.
+
+ .. versionadded:: 3.4
+
+.. attribute:: __path__
- class NamespaceLoader:
- @classmethod
- def module_repr(cls, module):
- return "<module '{}' (namespace)>".format(module.__name__)
+ If the module is a package (either regular or namespace), the module
+ object's ``__path__`` attribute must be set. The value must be
+ iterable, but may be empty if ``__path__`` has no further significance.
+ If ``__path__`` is not empty, it must produce strings when iterated
+ over. More details on the semantics of ``__path__`` are given
+ :ref:`below <package-path-rules>`.
+ Non-package modules should not have a ``__path__`` attribute.
+
+.. attribute:: __file__
+.. attribute:: __cached__
+
+ ``__file__`` is optional. If set, this attribute's value must be a
+ string. The import system may opt to leave ``__file__`` unset if it
+ has no semantic meaning (e.g. a module loaded from a database).
+
+ If ``__file__`` is set, it may also be appropriate to set the
+ ``__cached__`` attribute which is the path to any compiled version of
+ the code (e.g. byte-compiled file). The file does not need to exist
+ to set this attribute; the path can simply point to where the
+ compiled file would exist (see :pep:`3147`).
+
+ It is also appropriate to set ``__cached__`` when ``__file__`` is not
+ set. However, that scenario is quite atypical. Ultimately, the
+ loader is what makes use of ``__file__`` and/or ``__cached__``. So
+ if a loader can load from a cached module but otherwise does not load
+ from a file, that atypical scenario may be appropriate.
.. _package-path-rules:
@@ -462,9 +579,47 @@ A package's ``__init__.py`` file may set or alter the package's ``__path__``
attribute, and this was typically the way namespace packages were implemented
prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no
longer need to supply ``__init__.py`` files containing only ``__path__``
-manipulation code; the namespace loader automatically sets ``__path__``
+manipulation code; the import machinery automatically sets ``__path__``
correctly for the namespace package.
+Module reprs
+------------
+
+By default, all modules have a usable repr, however depending on the
+attributes set above, and in the module's spec, you can more explicitly
+control the repr of module objects.
+
+If the module has a spec (``__spec__``), the import machinery will try
+to generate a repr from it. If that fails or there is no spec, the import
+system will craft a default repr using whatever information is available
+on the module. It will try to use the ``module.__name__``,
+``module.__file__``, and ``module.__loader__`` as input into the repr,
+with defaults for whatever information is missing.
+
+Here are the exact rules used:
+
+ * If the module has a ``__spec__`` attribute, the information in the spec
+ is used to generate the repr. The "name", "loader", "origin", and
+ "has_location" attributes are consulted.
+
+ * If the module has a ``__file__`` attribute, this is used as part of the
+ module's repr.
+
+ * If the module has no ``__file__`` but does have a ``__loader__`` that is not
+ ``None``, then the loader's repr is used as part of the module's repr.
+
+ * Otherwise, just use the module's ``__name__`` in the repr.
+
+.. versionchanged:: 3.4
+ Use of :meth:`loader.module_repr() <importlib.abc.Loader.module_repr>`
+ has been deprecated and the module spec is now used by the import
+ machinery to generate a module repr.
+
+ For backward compatibility with Python 3.3, the module repr will be
+ generated by calling the loader's
+ :meth:`~importlib.abc.Loader.module_repr` method, if defined, before
+ trying either approach described above. However, the method is deprecated.
+
The Path Based Finder
=====================
@@ -473,8 +628,9 @@ The Path Based Finder
single: path based finder
As mentioned previously, Python comes with several default meta path finders.
-One of these, called the :term:`path based finder`, searches an :term:`import
-path`, which contains a list of :term:`path entries <path entry>`. Each path
+One of these, called the :term:`path based finder`
+(:class:`~importlib.machinery.PathFinder`), searches an :term:`import path`,
+which contains a list of :term:`path entries <path entry>`. Each path
entry names a location to search for modules.
The path based finder itself doesn't know how to import anything. Instead, it
@@ -523,15 +679,15 @@ Path entry finders
single: sys.path_importer_cache
single: PYTHONPATH
-The :term:`path based finder` is responsible for finding and loading Python
-modules and packages whose location is specified with a string :term:`path
-entry`. Most path entries name locations in the file system, but they need
-not be limited to this.
+The :term:`path based finder` is responsible for finding and loading
+Python modules and packages whose location is specified with a string
+:term:`path entry`. Most path entries name locations in the file system,
+but they need not be limited to this.
As a meta path finder, the :term:`path based finder` implements the
-:meth:`find_module()` protocol previously described, however it exposes
-additional hooks that can be used to customize how modules are found and
-loaded from the :term:`import path`.
+:meth:`~importlib.abc.MetaPathFinder.find_spec` protocol previously
+described, however it exposes additional hooks that can be used to
+customize how modules are found and loaded from the :term:`import path`.
Three variables are used by the :term:`path based finder`, :data:`sys.path`,
:data:`sys.path_hooks` and :data:`sys.path_importer_cache`. The ``__path__``
@@ -551,14 +707,16 @@ finder>`.
The :term:`path based finder` is a :term:`meta path finder`, so the import
machinery begins the :term:`import path` search by calling the path
-based finder's :meth:`find_module()` method as described previously. When
-the ``path`` argument to :meth:`find_module()` is given, it will be a
+based finder's :meth:`~importlib.machinery.PathFinder.find_spec` method as
+described previously. When the ``path`` argument to
+:meth:`~importlib.machinery.PathFinder.find_spec` is given, it will be a
list of string paths to traverse - typically a package's ``__path__``
-attribute for an import within that package. If the ``path`` argument
-is ``None``, this indicates a top level import and :data:`sys.path` is used.
+attribute for an import within that package. If the ``path`` argument is
+``None``, this indicates a top level import and :data:`sys.path` is used.
The path based finder iterates over every entry in the search path, and
-for each of these, looks for an appropriate :term:`path entry finder` for the
+for each of these, looks for an appropriate :term:`path entry finder`
+(:class:`~importlib.abc.PathEntryFinder`) for the
path entry. Because this can be an expensive operation (e.g. there may be
`stat()` call overheads for this search), the path based finder maintains
a cache mapping path entries to path entry finders. This cache is maintained
@@ -575,59 +733,82 @@ hooks <path entry hook>` in this list is called with a single argument, the
path entry to be searched. This callable may either return a :term:`path
entry finder` that can handle the path entry, or it may raise
:exc:`ImportError`. An :exc:`ImportError` is used by the path based finder to
-signal that the hook cannot find a :term:`path entry finder` for that
-:term:`path entry`. The exception is ignored and :term:`import path`
-iteration continues. The hook should expect either a string or bytes object;
-the encoding of bytes objects is up to the hook (e.g. it may be a file system
-encoding, UTF-8, or something else), and if the hook cannot decode the
-argument, it should raise :exc:`ImportError`.
+signal that the hook cannot find a :term:`path entry finder`.
+for that :term:`path entry`. The
+exception is ignored and :term:`import path` iteration continues. The hook
+should expect either a string or bytes object; the encoding of bytes objects
+is up to the hook (e.g. it may be a file system encoding, UTF-8, or something
+else), and if the hook cannot decode the argument, it should raise
+:exc:`ImportError`.
If :data:`sys.path_hooks` iteration ends with no :term:`path entry finder`
-being returned, then the path based finder's :meth:`find_module()` method
-will store ``None`` in :data:`sys.path_importer_cache` (to indicate that
-there is no finder for this path entry) and return ``None``, indicating that
-this :term:`meta path finder` could not find the module.
+being returned, then the path based finder's
+:meth:`~importlib.machinery.PathFinder.find_spec` method will store ``None``
+in :data:`sys.path_importer_cache` (to indicate that there is no finder for
+this path entry) and return ``None``, indicating that this
+:term:`meta path finder` could not find the module.
If a :term:`path entry finder` *is* returned by one of the :term:`path entry
hook` callables on :data:`sys.path_hooks`, then the following protocol is used
-to ask the finder for a module loader, which is then used to load the module.
-
+to ask the finder for a module spec, which is then used when loading the
+module.
Path entry finder protocol
--------------------------
In order to support imports of modules and initialized packages and also to
contribute portions to namespace packages, path entry finders must implement
-the :meth:`find_loader()` method.
-
-:meth:`find_loader()` takes one argument, the fully qualified name of the
-module being imported. :meth:`find_loader()` returns a 2-tuple where the
-first item is the loader and the second item is a namespace :term:`portion`.
-When the first item (i.e. the loader) is ``None``, this means that while the
-path entry finder does not have a loader for the named module, it knows that the
-path entry contributes to a namespace portion for the named module. This will
-almost always be the case where Python is asked to import a namespace package
-that has no physical presence on the file system. When a path entry finder
-returns ``None`` for the loader, the second item of the 2-tuple return value
-must be a sequence, although it can be empty.
-
-If :meth:`find_loader()` returns a non-``None`` loader value, the portion is
-ignored and the loader is returned from the path based finder, terminating
-the search through the path entries.
-
-For backwards compatibility with other implementations of the import
-protocol, many path entry finders also support the same,
-traditional :meth:`find_module()` method that meta path finders support.
-However path entry finder :meth:`find_module()` methods are never called
-with a ``path`` argument (they are expected to record the appropriate
-path information from the initial call to the path hook).
-
-The :meth:`find_module()` method on path entry finders is deprecated,
-as it does not allow the path entry finder to contribute portions to
-namespace packages. Instead path entry finders should implement the
-:meth:`find_loader()` method as described above. If it exists on the path
-entry finder, the import system will always call :meth:`find_loader()`
-in preference to :meth:`find_module()`.
+the :meth:`~importlib.abc.PathEntryFinder.find_spec` method.
+
+:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two argument, the
+fully qualified name of the module being imported, and the (optional) target
+module. ``find_spec()`` returns a fully populated spec for the module.
+This spec will always have "loader" set (with one exception).
+
+To indicate to the import machinery that the spec represents a namespace
+:term:`portion`. the path entry finder sets "loader" on the spec to
+``None`` and "submodule_search_locations" to a list containing the
+portion.
+
+.. versionchanged:: 3.4
+ :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced
+ :meth:`~importlib.abc.PathEntryFinder.find_loader` and
+ :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which
+ are now deprecated, but will be used if ``find_spec()`` is not defined.
+
+ Older path entry finders may implement one of these two deprecated methods
+ instead of ``find_spec()``. The methods are still respected for the
+ sake of backward compatibility. Howevever, if ``find_spec()`` is
+ implemented on the path entry finder, the legacy methods are ignored.
+
+ :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the
+ fully qualified name of the module being imported. ``find_loader()``
+ returns a 2-tuple where the first item is the loader and the second item
+ is a namespace :term:`portion`. When the first item (i.e. the loader) is
+ ``None``, this means that while the path entry finder does not have a
+ loader for the named module, it knows that the path entry contributes to
+ a namespace portion for the named module. This will almost always be the
+ case where Python is asked to import a namespace package that has no
+ physical presence on the file system. When a path entry finder returns
+ ``None`` for the loader, the second item of the 2-tuple return value must
+ be a sequence, although it can be empty.
+
+ If ``find_loader()`` returns a non-``None`` loader value, the portion is
+ ignored and the loader is returned from the path based finder, terminating
+ the search through the path entries.
+
+ For backwards compatibility with other implementations of the import
+ protocol, many path entry finders also support the same,
+ traditional ``find_module()`` method that meta path finders support.
+ However path entry finder ``find_module()`` methods are never called
+ with a ``path`` argument (they are expected to record the appropriate
+ path information from the initial call to the path hook).
+
+ The ``find_module()`` method on path entry finders is deprecated,
+ as it does not allow the path entry finder to contribute portions to
+ namespace packages. If both ``find_loader()`` and ``find_module()``
+ exist on a path entry finder, the import system will always call
+ ``find_loader()`` in preference to ``find_module()``.
Replacing the standard import system
@@ -646,9 +827,54 @@ import statements within that module.
To selectively prevent import of some modules from a hook early on the
meta path (rather than disabling the standard import system entirely),
it is sufficient to raise :exc:`ImportError` directly from
-:meth:`find_module` instead of returning ``None``. The latter indicates
-that the meta path search should continue. while raising an exception
-terminates it immediately.
+:meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning
+``None``. The latter indicates that the meta path search should continue,
+while raising an exception terminates it immediately.
+
+
+Special considerations for __main__
+===================================
+
+The :mod:`__main__` module is a special case relative to Python's import
+system. As noted :ref:`elsewhere <programs>`, the ``__main__`` module
+is directly initialized at interpreter startup, much like :mod:`sys` and
+:mod:`builtins`. However, unlike those two, it doesn't strictly
+qualify as a built-in module. This is because the manner in which
+``__main__`` is initialized depends on the flags and other options with
+which the interpreter is invoked.
+
+.. _main_spec:
+
+__main__.__spec__
+-----------------
+
+Depending on how :mod:`__main__` is initialized, ``__main__.__spec__``
+gets set appropriately or to ``None``.
+
+When Python is started with the :option:`-m` option, ``__spec__`` is set
+to the module spec of the corresponding module or package. ``__spec__`` is
+also populated when the ``__main__`` module is loaded as part of executing a
+directory, zipfile or other :data:`sys.path` entry.
+
+In :ref:`the remaining cases <using-on-interface-options>`
+``__main__.__spec__`` is set to ``None``, as the code used to populate the
+:mod:`__main__` does not correspond directly with an importable module:
+
+- interactive prompt
+- -c switch
+- running from stdin
+- running directly from a source or bytecode file
+
+Note that ``__main__.__spec__`` is always ``None`` in the last case,
+*even if* the file could technically be imported directly as a module
+instead. Use the :option:`-m` switch if valid module metadata is desired
+in :mod:`__main__`.
+
+Note also that even when ``__main__`` corresponds with an importable module
+and ``__main__.__spec__`` is set accordingly, they're still considered
+*distinct* modules. This is due to the fact that blocks guarded by
+``if __name__ == "__main__":`` checks only execute when the module is used
+to populate the ``__main__`` namespace, and not during normal import.
Open issues
@@ -663,6 +889,12 @@ related entries in the data model reference page?
XXX runpy, pkgutil, et al in the library manual should all get "See Also"
links at the top pointing to the new import system section.
+XXX Add more explanation regarding the different ways in which
+``__main__`` is initialized?
+
+XXX Add more info on ``__main__`` quirks/pitfalls (i.e. copy from
+:pep:`395`).
+
References
==========
@@ -688,6 +920,11 @@ proposed ``__name__`` for semantics :pep:`366` would eventually specify for
:pep:`338` defines executing modules as scripts.
+:pep:`451` adds the encapsulation of per-module import state in spec
+objects. It also off-loads most of the boilerplate responsibilities of
+loaders back onto the import machinery. These changes allow the
+deprecation of several APIs in the import system and also addition of new
+methods to finders and loaders.
.. rubric:: Footnotes
diff --git a/Doc/tools/sphinx-build.py b/Doc/tools/sphinx-build.py
deleted file mode 100644
index d3fe702..0000000
--- a/Doc/tools/sphinx-build.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- Sphinx - Python documentation toolchain
- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
- :copyright: 2007-2010 by Georg Brandl.
- :license: Python license.
-"""
-
-import sys
-import warnings
-
-# Get rid of UserWarnings reported by pkg_resources.
-warnings.filterwarnings('ignore', category=UserWarning, module='jinja2')
-
-if __name__ == '__main__':
-
- if sys.version_info[:3] < (2, 4, 0) or sys.version_info[:3] > (3, 0, 0):
- sys.stderr.write("""\
-Error: Sphinx needs to be executed with Python 2.4 or newer (not 3.x though).
-(If you run this from the Makefile, you can set the PYTHON variable
-to the path of an alternative interpreter executable, e.g.,
-``make html PYTHON=python2.5``).
-""")
- sys.exit(1)
-
- from sphinx import main
- sys.exit(main(sys.argv))
diff --git a/Doc/tools/sphinxext/download.html b/Doc/tools/sphinxext/download.html
index 31a53cf..0550111 100644
--- a/Doc/tools/sphinxext/download.html
+++ b/Doc/tools/sphinxext/download.html
@@ -34,8 +34,8 @@ in the table are the size of the download files in megabytes.</p>
<td><a href="{{ dlbase }}/python-{{ release }}-docs-text.tar.bz2">Download</a> (ca. 1.5 MB)</td>
</tr>
<tr><td>EPUB</td>
- <td><a href="{{ dlbase }}/python-{{ release }}-docs-epub.zip">Download</a> (ca. 3.5 MB)</td>
- <td><a href="{{ dlbase }}/python-{{ release }}-docs-epub.tar.bz2">Download</a> (ca. 3.5 MB)</td>
+ <td><a href="{{ dlbase }}/python-{{ release }}-docs.epub">Download</a> (ca. 4.5 MB)</td>
+ <td></td>
</tr>
</table>
diff --git a/Doc/tools/sphinxext/indexcontent.html b/Doc/tools/sphinxext/indexcontent.html
index 7f85470..969099a 100644
--- a/Doc/tools/sphinxext/indexcontent.html
+++ b/Doc/tools/sphinxext/indexcontent.html
@@ -16,14 +16,14 @@
<p class="biglink"><a class="biglink" href="{{ pathto("howto/index") }}">Python HOWTOs</a><br/>
<span class="linkdescr">in-depth documents on specific topics</span></p>
</td><td width="50%">
+ <p class="biglink"><a class="biglink" href="{{ pathto("installing/index") }}">Installing Python Modules</a><br/>
+ <span class="linkdescr">installing from the Python Package Index &amp; other sources</span></p>
+ <p class="biglink"><a class="biglink" href="{{ pathto("distributing/index") }}">Distributing Python Modules</a><br/>
+ <span class="linkdescr">publishing modules for installation by others</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("extending/index") }}">Extending and Embedding</a><br/>
<span class="linkdescr">tutorial for C/C++ programmers</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("c-api/index") }}">Python/C API</a><br/>
<span class="linkdescr">reference for C/C++ programmers</span></p>
- <p class="biglink"><a class="biglink" href="{{ pathto("install/index") }}">Installing Python Modules</a><br/>
- <span class="linkdescr">information for installers &amp; sys-admins</span></p>
- <p class="biglink"><a class="biglink" href="{{ pathto("distutils/index") }}">Distributing Python Modules</a><br/>
- <span class="linkdescr">sharing modules with others</span></p>
<p class="biglink"><a class="biglink" href="{{ pathto("faq/index") }}">FAQs</a><br/>
<span class="linkdescr">frequently asked questions (with answers!)</span></p>
</td></tr>
diff --git a/Doc/tools/sphinxext/indexsidebar.html b/Doc/tools/sphinxext/indexsidebar.html
index a0ec32f..ed5da0c 100644
--- a/Doc/tools/sphinxext/indexsidebar.html
+++ b/Doc/tools/sphinxext/indexsidebar.html
@@ -3,7 +3,7 @@
<h3>Docs for other versions</h3>
<ul>
<li><a href="http://docs.python.org/2.7/">Python 2.7 (stable)</a></li>
- <li><a href="http://docs.python.org/3.4/">Python 3.4 (in development)</a></li>
+ <li><a href="http://docs.python.org/3.3/">Python 3.3 (stable)</a></li>
<li><a href="http://www.python.org/doc/versions/">Old versions</a></li>
</ul>
diff --git a/Doc/tools/sphinxext/layout.html b/Doc/tools/sphinxext/layout.html
index 16a9212..d3b2801 100644
--- a/Doc/tools/sphinxext/layout.html
+++ b/Doc/tools/sphinxext/layout.html
@@ -12,6 +12,8 @@
{%- endif %}
</li>
{% endblock %}
+{% block relbar1 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %}
+{% block relbar2 %} {% if builder != 'qthelp' %} {{ relbar() }} {% endif %} {% endblock %}
{% block extrahead %}
<link rel="shortcut icon" type="image/png" href="{{ pathto('_static/py.png', 1) }}" />
{% if not embedded %}<script type="text/javascript" src="{{ pathto('_static/copybutton.js', 1) }}"></script>{% endif %}
diff --git a/Doc/tools/sphinxext/pyspecific.py b/Doc/tools/sphinxext/pyspecific.py
index f2d3ca1..31d8c06 100644
--- a/Doc/tools/sphinxext/pyspecific.py
+++ b/Doc/tools/sphinxext/pyspecific.py
@@ -10,7 +10,7 @@
"""
ISSUE_URI = 'http://bugs.python.org/issue%s'
-SOURCE_URI = 'http://hg.python.org/cpython/file/3.3/%s'
+SOURCE_URI = 'http://hg.python.org/cpython/file/3.4/%s'
from docutils import nodes, utils
diff --git a/Doc/tools/sphinxext/susp-ignored.csv b/Doc/tools/sphinxext/susp-ignored.csv
index abc3ca8..1769023 100644
--- a/Doc/tools/sphinxext/susp-ignored.csv
+++ b/Doc/tools/sphinxext/susp-ignored.csv
@@ -79,9 +79,8 @@ howto/logging,,:Started,INFO:root:Started
howto/logging,,:This,DEBUG:root:This message should go to the log file
howto/logging,,:This,DEBUG:This message should appear on the console
howto/logging,,:Watch,WARNING:root:Watch out!
-howto/pyporting,75,::,# make sure to use :: Python *and* :: Python :: 3 so
-howto/pyporting,75,::,"'Programming Language :: Python',"
-howto/pyporting,75,::,'Programming Language :: Python :: 3'
+howto/pyporting,,::,Programming Language :: Python :: 2
+howto/pyporting,,::,Programming Language :: Python :: 3
howto/regex,,::,
howto/regex,,:foo,(?:foo)
howto/urllib2,,:example,"for example ""joe@password:example.com"""
@@ -141,15 +140,8 @@ library/linecache,,:sys,"sys:x:3:3:sys:/dev:/bin/sh"
library/logging.handlers,,:port,host:port
library/mmap,,:i2,obj[i1:i2]
library/multiprocessing,,`,# Add more tasks using `put()`
-library/multiprocessing,,`,# A test file for the `multiprocessing` package
-library/multiprocessing,,`,# A test of `multiprocessing.Pool` class
-library/multiprocessing,,`,# `BaseManager`.
-library/multiprocessing,,`,# in the original order then consider using `Pool.map()` or
library/multiprocessing,,`,">>> l._callmethod('__getitem__', (20,)) # equiv to `l[20]`"
library/multiprocessing,,`,">>> l._callmethod('__getslice__', (2, 7)) # equiv to `l[2:7]`"
-library/multiprocessing,,`,# Not sure if we should synchronize access to `socket.accept()` method by
-library/multiprocessing,,`,# object. (We import `multiprocessing.reduction` to enable this pickling.)
-library/multiprocessing,,`,# `Pool.imap()` (which will save on the amount of code needed anyway).
library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)"
library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy
library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy
@@ -158,20 +150,16 @@ library/nntplib,,:bytes,:bytes
library/nntplib,,:lines,:lines
library/optparse,,:len,"del parser.rargs[:len(value)]"
library/os.path,,:foo,c:foo
+library/pathlib,,:bar,">>> PureWindowsPath('c:/Windows', 'd:bar')"
+library/pathlib,,:bar,PureWindowsPath('d:bar')
+library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').root
+library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').anchor
library/pdb,,:lineno,filename:lineno
library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")"
library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS"""
-library/pprint,209,::,"'classifiers': ['Development Status :: 4 - Beta',"
-library/pprint,209,::,"'Intended Audience :: Developers',"
-library/pprint,209,::,"'License :: OSI Approved :: MIT License',"
-library/pprint,209,::,"'Natural Language :: English',"
-library/pprint,209,::,"'Operating System :: OS Independent',"
-library/pprint,209,::,"'Programming Language :: Python',"
-library/pprint,209,::,"'Programming Language :: Python :: 2',"
-library/pprint,209,::,"'Programming Language :: Python :: 2.6',"
-library/pprint,209,::,"'Programming Language :: Python :: 2.7',"
-library/pprint,209,::,"'Topic :: Software Development :: Libraries',"
-library/pprint,209,::,"'Topic :: Software Development :: Libraries :: Python Modules'],"
+library/pprint,,::,"'Programming Language :: Python :: 2 :: Only'],"
+library/pprint,,::,"'Programming Language :: Python :: 2.6',"
+library/pprint,,::,"'Programming Language :: Python :: 2.7',"
library/profile,,:lineno,filename:lineno(function)
library/pyexpat,,:elem1,<py:elem1 />
library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">"
@@ -183,6 +171,7 @@ library/socket,,:len,fds.fromstring(cmsg_data[:len(cmsg_data) - (len(cmsg_data)
library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})"
library/sqlite3,,:memory,
library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})"
+library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)"
library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group"
library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc."
library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com"
@@ -207,7 +196,12 @@ library/tarfile,,:xz,'r:xz'
library/tarfile,,:xz,'w:xz'
library/time,,:mm,
library/time,,:ss,
+library/tracemalloc,,:limit,"for index, stat in enumerate(top_stats[:limit], 1):"
library/turtle,,::,Example::
+library/unittest,1412,:foo,"self.assertEqual(cm.output, ['INFO:foo:first message',"
+library/unittest,1412,:first,"self.assertEqual(cm.output, ['INFO:foo:first message',"
+library/unittest,1412,:foo,'ERROR:foo.bar:second message'])
+library/unittest,1412,:second,'ERROR:foo.bar:second message'])
library/urllib.request,,:close,Connection:close
library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n<head>\n"
library/urllib.request,,:password,"""joe:password@python.org"""
@@ -283,8 +277,8 @@ whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') a
whatsnew/3.2,,:location,zope9-location = ${zope9:location}
whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf
whatsnew/changelog,,:platform,:platform:
+whatsnew/changelog,,:gz,": TarFile opened with external fileobj and ""w:gz"" mode didn't"
whatsnew/changelog,,:PythonCmd,"With Tk < 8.5 _tkinter.c:PythonCmd() raised UnicodeDecodeError, caused"
whatsnew/changelog,,::,": Fix FTP tests for IPv6, bind to ""::1"" instead of ""localhost""."
whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as"
whatsnew/changelog,,:password,user:password
-whatsnew/changelog,,:gz,w:gz
diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst
index 36acb06..abf30f0 100644
--- a/Doc/tutorial/interactive.rst
+++ b/Doc/tutorial/interactive.rst
@@ -7,140 +7,27 @@ Interactive Input Editing and History Substitution
Some versions of the Python interpreter support editing of the current input
line and history substitution, similar to facilities found in the Korn shell and
the GNU Bash shell. This is implemented using the `GNU Readline`_ library,
-which supports Emacs-style and vi-style editing. This library has its own
-documentation which I won't duplicate here; however, the basics are easily
-explained. The interactive editing and history described here are optionally
-available in the Unix and Cygwin versions of the interpreter.
-
-This chapter does *not* document the editing facilities of Mark Hammond's
-PythonWin package or the Tk-based environment, IDLE, distributed with Python.
-The command line history recall which operates within DOS boxes on NT and some
-other DOS and Windows flavors is yet another beast.
-
-
-.. _tut-lineediting:
-
-Line Editing
-============
-
-If supported, input line editing is active whenever the interpreter prints a
-primary or secondary prompt. The current line can be edited using the
-conventional Emacs control characters. The most important of these are:
-:kbd:`C-A` (Control-A) moves the cursor to the beginning of the line, :kbd:`C-E`
-to the end, :kbd:`C-B` moves it one position to the left, :kbd:`C-F` to the
-right. Backspace erases the character to the left of the cursor, :kbd:`C-D` the
-character to its right. :kbd:`C-K` kills (erases) the rest of the line to the
-right of the cursor, :kbd:`C-Y` yanks back the last killed string.
-:kbd:`C-underscore` undoes the last change you made; it can be repeated for
-cumulative effect.
-
-
-.. _tut-history:
-
-History Substitution
-====================
-
-History substitution works as follows. All non-empty input lines issued are
-saved in a history buffer, and when a new prompt is given you are positioned on
-a new line at the bottom of this buffer. :kbd:`C-P` moves one line up (back) in
-the history buffer, :kbd:`C-N` moves one down. Any line in the history buffer
-can be edited; an asterisk appears in front of the prompt to mark a line as
-modified. Pressing the :kbd:`Return` key passes the current line to the
-interpreter. :kbd:`C-R` starts an incremental reverse search; :kbd:`C-S` starts
-a forward search.
+which supports various styles of editing. This library has its own
+documentation which we won't duplicate here.
.. _tut-keybindings:
-Key Bindings
-============
-
-The key bindings and some other parameters of the Readline library can be
-customized by placing commands in an initialization file called
-:file:`~/.inputrc`. Key bindings have the form ::
-
- key-name: function-name
-
-or ::
-
- "string": function-name
-
-and options can be set with ::
-
- set option-name value
-
-For example::
-
- # I prefer vi-style editing:
- set editing-mode vi
-
- # Edit using a single line:
- set horizontal-scroll-mode On
+Tab Completion and History Editing
+==================================
- # Rebind some keys:
- Meta-h: backward-kill-word
- "\C-u": universal-argument
- "\C-x\C-r": re-read-init-file
-
-Note that the default binding for :kbd:`Tab` in Python is to insert a :kbd:`Tab`
-character instead of Readline's default filename completion function. If you
-insist, you can override this by putting ::
-
- Tab: complete
-
-in your :file:`~/.inputrc`. (Of course, this makes it harder to type indented
-continuation lines if you're accustomed to using :kbd:`Tab` for that purpose.)
-
-.. index::
- module: rlcompleter
- module: readline
-
-Automatic completion of variable and module names is optionally available. To
-enable it in the interpreter's interactive mode, add the following to your
-startup file: [#]_ ::
-
- import rlcompleter, readline
- readline.parse_and_bind('tab: complete')
-
-This binds the :kbd:`Tab` key to the completion function, so hitting the
-:kbd:`Tab` key twice suggests completions; it looks at Python statement names,
-the current local variables, and the available module names. For dotted
-expressions such as ``string.a``, it will evaluate the expression up to the
-final ``'.'`` and then suggest completions from the attributes of the resulting
-object. Note that this may execute application-defined code if an object with a
-:meth:`__getattr__` method is part of the expression.
-
-A more capable startup file might look like this example. Note that this
-deletes the names it creates once they are no longer needed; this is done since
-the startup file is executed in the same namespace as the interactive commands,
-and removing the names avoids creating side effects in the interactive
-environment. You may find it convenient to keep some of the imported modules,
-such as :mod:`os`, which turn out to be needed in most sessions with the
-interpreter. ::
-
- # Add auto-completion and a stored history file of commands to your Python
- # interactive interpreter. Requires Python 2.0+, readline. Autocomplete is
- # bound to the Esc key by default (you can change it - see readline docs).
- #
- # Store the file in ~/.pystartup, and set an environment variable to point
- # to it: "export PYTHONSTARTUP=~/.pystartup" in bash.
-
- import atexit
- import os
- import readline
- import rlcompleter
-
- historyPath = os.path.expanduser("~/.pyhistory")
-
- def save_history(historyPath=historyPath):
- import readline
- readline.write_history_file(historyPath)
-
- if os.path.exists(historyPath):
- readline.read_history_file(historyPath)
-
- atexit.register(save_history)
- del os, atexit, readline, rlcompleter, save_history, historyPath
+Completion of variable and module names is
+:ref:`automatically enabled <rlcompleter-config>` at interpreter startup so
+that the :kbd:`Tab` key invokes the completion function; it looks at
+Python statement names, the current local variables, and the available
+module names. For dotted expressions such as ``string.a``, it will evaluate
+the expression up to the final ``'.'`` and then suggest completions from
+the attributes of the resulting object. Note that this may execute
+application-defined code if an object with a :meth:`__getattr__` method
+is part of the expression. The default configuration also saves your
+history into a file named :file:`.python_history` in your user directory.
+The history will be available again during the next interactive interpreter
+session.
.. _tut-commentary:
@@ -162,14 +49,6 @@ into other applications. Another similar enhanced interactive environment is
bpython_.
-.. rubric:: Footnotes
-
-.. [#] Python will execute the contents of a file identified by the
- :envvar:`PYTHONSTARTUP` environment variable when you start an interactive
- interpreter. To customize Python even for non-interactive mode, see
- :ref:`tut-customize`.
-
-
.. _GNU Readline: http://tiswww.case.edu/php/chet/readline/rltop.html
.. _IPython: http://ipython.scipy.org/
.. _bpython: http://www.bpython-interpreter.org/
diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index cdc2bf2..8e8395a 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -10,13 +10,13 @@ Using the Python Interpreter
Invoking the Interpreter
========================
-The Python interpreter is usually installed as :file:`/usr/local/bin/python3.3`
+The Python interpreter is usually installed as :file:`/usr/local/bin/python3.4`
on those machines where it is available; putting :file:`/usr/local/bin` in your
Unix shell's search path makes it possible to start it by typing the command:
.. code-block:: text
- python3.3
+ python3.4
to the shell. [#]_ Since the choice of the directory where the interpreter lives
is an installation option, other places are possible; check with your local
@@ -24,11 +24,11 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a
popular alternative location.)
On Windows machines, the Python installation is usually placed in
-:file:`C:\\Python33`, though you can change this when you're running the
+:file:`C:\\Python34`, though you can change this when you're running the
installer. To add this directory to your path, you can type the following
command into the command prompt in a DOS box::
- set path=%path%;C:\python33
+ set path=%path%;C:\python34
Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on
Windows) at the primary prompt causes the interpreter to exit with a zero exit
@@ -95,9 +95,9 @@ with the *secondary prompt*, by default three dots (``...``). The interpreter
prints a welcome message stating its version number and a copyright notice
before printing the first prompt::
- $ python3.3
- Python 3.3 (default, Sep 24 2012, 09:25:04)
- [GCC 4.6.3] on linux2
+ $ python3.4
+ Python 3.4 (default, Mar 16 2014, 09:25:04)
+ [GCC 4.8.2] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>>
@@ -106,7 +106,7 @@ before printing the first prompt::
Continuation lines are needed when entering a multi-line construct. As an
example, take a look at this :keyword:`if` statement::
- >>> the_world_is_flat = 1
+ >>> the_world_is_flat = True
>>> if the_world_is_flat:
... print("Be careful not to fall off!")
...
@@ -149,7 +149,7 @@ Executable Python Scripts
On BSD'ish Unix systems, Python scripts can be made directly executable, like
shell scripts, by putting the line ::
- #! /usr/bin/env python3.3
+ #! /usr/bin/env python3.4
(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning
of the script and giving the file an executable mode. The ``#!`` must be the
diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst
index 1902964..fd361ae 100644
--- a/Doc/tutorial/modules.rst
+++ b/Doc/tutorial/modules.rst
@@ -165,10 +165,16 @@ a built-in module with that name. If not found, it then searches for a file
named :file:`spam.py` in a list of directories given by the variable
:data:`sys.path`. :data:`sys.path` is initialized from these locations:
-* the directory containing the input script (or the current directory).
+* The directory containing the input script (or the current directory when no
+ file is specified).
* :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the
shell variable :envvar:`PATH`).
-* the installation-dependent default.
+* The installation-dependent default.
+
+.. note::
+ On file systems which support symlinks, the directory containing the input
+ script is calculated after the symlink is followed. In other words the
+ directory containing the symlink is **not** added to the module search path.
After initialization, Python programs can modify :data:`sys.path`. The
directory containing the script being run is placed at the beginning of the
@@ -278,24 +284,23 @@ defines. It returns a sorted list of strings::
>>> dir(fibo)
['__name__', 'fib', 'fib2']
>>> dir(sys) # doctest: +NORMALIZE_WHITESPACE
- ['__displayhook__', '__doc__', '__egginsert', '__excepthook__',
- '__loader__', '__name__', '__package__', '__plen', '__stderr__',
- '__stdin__', '__stdout__', '_clear_type_cache', '_current_frames',
- '_debugmallocstats', '_getframe', '_home', '_mercurial', '_xoptions',
- 'abiflags', 'api_version', 'argv', 'base_exec_prefix', 'base_prefix',
- 'builtin_module_names', 'byteorder', 'call_tracing', 'callstats',
- 'copyright', 'displayhook', 'dont_write_bytecode', 'exc_info',
- 'excepthook', 'exec_prefix', 'executable', 'exit', 'flags', 'float_info',
- 'float_repr_style', 'getcheckinterval', 'getdefaultencoding',
- 'getdlopenflags', 'getfilesystemencoding', 'getobjects', 'getprofile',
- 'getrecursionlimit', 'getrefcount', 'getsizeof', 'getswitchinterval',
- 'gettotalrefcount', 'gettrace', 'hash_info', 'hexversion',
- 'implementation', 'int_info', 'intern', 'maxsize', 'maxunicode',
- 'meta_path', 'modules', 'path', 'path_hooks', 'path_importer_cache',
- 'platform', 'prefix', 'ps1', 'setcheckinterval', 'setdlopenflags',
- 'setprofile', 'setrecursionlimit', 'setswitchinterval', 'settrace',
- 'stderr', 'stdin', 'stdout', 'thread_info', 'version', 'version_info',
- 'warnoptions']
+ ['__displayhook__', '__doc__', '__excepthook__', '__loader__', '__name__',
+ '__package__', '__stderr__', '__stdin__', '__stdout__',
+ '_clear_type_cache', '_current_frames', '_debugmallocstats', '_getframe',
+ '_home', '_mercurial', '_xoptions', 'abiflags', 'api_version', 'argv',
+ 'base_exec_prefix', 'base_prefix', 'builtin_module_names', 'byteorder',
+ 'call_tracing', 'callstats', 'copyright', 'displayhook',
+ 'dont_write_bytecode', 'exc_info', 'excepthook', 'exec_prefix',
+ 'executable', 'exit', 'flags', 'float_info', 'float_repr_style',
+ 'getcheckinterval', 'getdefaultencoding', 'getdlopenflags',
+ 'getfilesystemencoding', 'getobjects', 'getprofile', 'getrecursionlimit',
+ 'getrefcount', 'getsizeof', 'getswitchinterval', 'gettotalrefcount',
+ 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info',
+ 'intern', 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path',
+ 'path_hooks', 'path_importer_cache', 'platform', 'prefix', 'ps1',
+ 'setcheckinterval', 'setdlopenflags', 'setprofile', 'setrecursionlimit',
+ 'setswitchinterval', 'settrace', 'stderr', 'stdin', 'stdout',
+ 'thread_info', 'version', 'version_info', 'warnoptions']
Without arguments, :func:`dir` lists the names you have defined currently::
diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst
index 7e7a154..2e3ed18 100644
--- a/Doc/tutorial/stdlib.rst
+++ b/Doc/tutorial/stdlib.rst
@@ -15,7 +15,7 @@ operating system::
>>> import os
>>> os.getcwd() # Return the current working directory
- 'C:\\Python33'
+ 'C:\\Python34'
>>> os.chdir('/server/accesslogs') # Change current working directory
>>> os.system('mkdir today') # Run the command mkdir in the system shell
0
diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst
index c1dd69a..c0197ea 100644
--- a/Doc/tutorial/stdlib2.rst
+++ b/Doc/tutorial/stdlib2.rst
@@ -277,7 +277,7 @@ applications include caching objects that are expensive to create::
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
d['primary'] # entry was automatically removed
- File "C:/python33/lib/weakref.py", line 46, in __getitem__
+ File "C:/python34/lib/weakref.py", line 46, in __getitem__
o = self.data[key]()
KeyError: 'primary'
diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst
index 7fcbdc3f..979f587 100644
--- a/Doc/tutorial/whatnow.rst
+++ b/Doc/tutorial/whatnow.rst
@@ -21,8 +21,8 @@ the set are:
and many other tasks. Skimming through the Library Reference will give you an
idea of what's available.
-* :ref:`install-index` explains how to install external modules written by other
- Python users.
+* :ref:`installing-index` explains how to install additional modules written
+ by other Python users.
* :ref:`reference-index`: A detailed explanation of Python's syntax and
semantics. It's heavy reading, but is useful as a complete guide to the
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index 4e7168f..0c3c203 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -24,7 +24,7 @@ Command line
When invoking Python, you may specify any of these options::
- python [-bBdEhiOqsSuvVWx?] [-c command | -m module-name | script | - ] [args]
+ python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args]
The most common use case is, of course, a simple invocation of a script::
@@ -81,7 +81,8 @@ source.
the implementation may not always enforce this (e.g. it may allow you to
use a name that includes a hyphen).
- Package names are also permitted. When a package name is supplied instead
+ Package names (including namespace packages) are also permitted. When a
+ package name is supplied instead
of a normal module, the interpreter will execute ``<pkg>.__main__`` as
the main module. This behaviour is deliberately similar to the handling
of directories and zipfiles that are passed to the interpreter as the
@@ -115,6 +116,9 @@ source.
.. versionchanged:: 3.1
Supply the package name to run a ``__main__`` submodule.
+ .. versionchanged:: 3.4
+ namespace packages are also supported
+
.. describe:: -
@@ -147,7 +151,12 @@ source.
If no interface option is given, :option:`-i` is implied, ``sys.argv[0]`` is
an empty string (``""``) and the current directory will be added to the
-start of :data:`sys.path`.
+start of :data:`sys.path`. Also, tab-completion and history editing is
+automatically enabled, if available on your platform (see
+:ref:`rlcompleter-config`).
+
+.. versionchanged:: 3.4
+ Automatic enabling of tab-completion and history editing.
.. seealso:: :ref:`tut-invoking`
@@ -170,6 +179,8 @@ Generic options
Python 3.0
+.. _using-on-misc-options:
+
Miscellaneous options
~~~~~~~~~~~~~~~~~~~~~
@@ -208,6 +219,17 @@ Miscellaneous options
raises an exception. See also :envvar:`PYTHONINSPECT`.
+.. cmdoption:: -I
+
+ Run Python in isolated mode. This also implies -E and -s.
+ In isolated mode :data:`sys.path` contains neither the script's directory nor
+ the user's site-packages directory. All :envvar:`PYTHON*` environment
+ variables are ignored, too. Further restrictions may be imposed to prevent
+ the user from injecting malicious code.
+
+ .. versionadded:: 3.4
+
+
.. cmdoption:: -O
Turn on basic optimizations. This changes the filename extension for
@@ -358,9 +380,19 @@ Miscellaneous options
.. cmdoption:: -X
Reserved for various implementation-specific options. CPython currently
- defines just one, you can use ``-X faulthandler`` to enable
- :mod:`faulthandler`. It also allows to pass arbitrary values and retrieve
- them through the :data:`sys._xoptions` dictionary.
+ defines the following possible values:
+
+ * ``-X faulthandler`` to enable :mod:`faulthandler`;
+ * ``-X showrefcount`` to enable the output of the total reference count
+ and memory blocks (only works on debug builds);
+ * ``-X tracemalloc`` to start tracing Python memory allocations using the
+ :mod:`tracemalloc` module. By default, only the most recent frame is
+ stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start
+ tracing with a traceback limit of *NFRAME* frames. See the
+ :func:`tracemalloc.start` for more information.
+
+ It also allows to pass arbitrary values and retrieve them through the
+ :data:`sys._xoptions` dictionary.
.. versionchanged:: 3.2
It is now allowed to pass :option:`-X` with CPython.
@@ -368,6 +400,9 @@ Miscellaneous options
.. versionadded:: 3.3
The ``-X faulthandler`` option.
+ .. versionadded:: 3.4
+ The ``-X showrefcount`` and ``-X tracemalloc`` options.
+
Options you shouldn't use
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -385,7 +420,7 @@ Environment variables
---------------------
These environment variables influence Python's behavior, they are processed
-before the command-line switches other than -E. It is customary that
+before the command-line switches other than -E or -I. It is customary that
command-line switches override environmental variables where there is a
conflict.
@@ -430,7 +465,7 @@ conflict.
is executed in the same namespace where interactive commands are executed so
that objects defined or imported in it can be used without qualification in
the interactive session. You can also change the prompts :data:`sys.ps1` and
- :data:`sys.ps2` in this file.
+ :data:`sys.ps2` and the hook :data:`sys.__interactivehook__` in this file.
.. envvar:: PYTHONY2K
@@ -485,9 +520,9 @@ conflict.
.. envvar:: PYTHONDONTWRITEBYTECODE
- If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the
- import of source modules. This is equivalent to specifying the :option:`-B`
- option.
+ If this is set to a non-empty string, Python won't try to write ``.pyc`` or
+ ``.pyo`` files on the import of source modules. This is equivalent to
+ specifying the :option:`-B` option.
.. envvar:: PYTHONHASHSEED
@@ -512,13 +547,16 @@ conflict.
.. envvar:: PYTHONIOENCODING
If this is set before running the interpreter, it overrides the encoding used
- for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. The
- ``:errorhandler`` part is optional and has the same meaning as in
- :func:`str.encode`.
+ for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. Both
+ the ``encodingname`` and the ``:errorhandler`` parts are optional and have
+ the same meaning as in :func:`str.encode`.
For stderr, the ``:errorhandler`` part is ignored; the handler will always be
``'backslashreplace'``.
+ .. versionchanged:: 3.4
+ The ``encodingname`` part is now optional.
+
.. envvar:: PYTHONNOUSERSITE
@@ -556,15 +594,34 @@ conflict.
.. envvar:: PYTHONFAULTHANDLER
- If this environment variable is set, :func:`faulthandler.enable` is called
- at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`,
- :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the
- Python traceback. This is equivalent to :option:`-X` ``faulthandler``
- option.
+ If this environment variable is set to a non-empty string,
+ :func:`faulthandler.enable` is called at startup: install a handler for
+ :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and
+ :const:`SIGILL` signals to dump the Python traceback. This is equivalent to
+ :option:`-X` ``faulthandler`` option.
.. versionadded:: 3.3
+.. envvar:: PYTHONTRACEMALLOC
+
+ If this environment variable is set to a non-empty string, start tracing
+ Python memory allocations using the :mod:`tracemalloc` module. The value of
+ the variable is the maximum number of frames stored in a traceback of a
+ trace. For example, ``PYTHONTRACEMALLOC=1`` stores only the most recent
+ frame. See the :func:`tracemalloc.start` for more information.
+
+ .. versionadded:: 3.4
+
+
+.. envvar:: PYTHONASYNCIODEBUG
+
+ If this environment variable is set to a non-empty string, enable the debug
+ mode of the :mod:`asyncio` module.
+
+ .. versionadded:: 3.4
+
+
Debug-mode variables
~~~~~~~~~~~~~~~~~~~~
diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst
index 3e1b74d..5be439f 100644
--- a/Doc/using/mac.rst
+++ b/Doc/using/mac.rst
@@ -25,7 +25,7 @@ there.
What you get after installing is a number of things:
-* A :file:`MacPython 3.3` folder in your :file:`Applications` folder. In here
+* A :file:`MacPython 3.4` folder in your :file:`Applications` folder. In here
you find IDLE, the development environment that is a standard part of official
Python distributions; PythonLauncher, which handles double-clicking Python
scripts from the Finder; and the "Build Applet" tool, which allows you to
@@ -93,7 +93,7 @@ aware of: programs that talk to the Aqua window manager (in other words,
anything that has a GUI) need to be run in a special way. Use :program:`pythonw`
instead of :program:`python` to start such scripts.
-With Python 3.3, you can use either :program:`python` or :program:`pythonw`.
+With Python 3.4, you can use either :program:`python` or :program:`pythonw`.
Configuration
@@ -158,7 +158,7 @@ http://www.riverbankcomputing.co.uk/software/pyqt/intro.
Distributing Python Applications on the Mac
===========================================
-The "Build Applet" tool that is placed in the MacPython 3.3 folder is fine for
+The "Build Applet" tool that is placed in the MacPython 3.4 folder is fine for
packaging small Python scripts on your own machine to run as a standard Mac
application. This tool, however, is not robust enough to distribute Python
applications to other users.
diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst
index 40635c6..3c7fc41 100644
--- a/Doc/using/unix.rst
+++ b/Doc/using/unix.rst
@@ -60,6 +60,8 @@ To install the newest Python versions on OpenSolaris, install `blastwave
prompt.
+.. _building-python-on-unix:
+
Building Python
===============
diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc
index 5fdbc9b..52cdda0 100644
--- a/Doc/using/venv-create.inc
+++ b/Doc/using/venv-create.inc
@@ -11,6 +11,11 @@ containing a copy of the ``python`` binary (or binaries, in the case of
Windows). It also creates an (initially empty) ``lib/pythonX.Y/site-packages``
subdirectory (on Windows, this is ``Lib\site-packages``).
+.. seealso::
+
+ `Python Packaging User Guide: Creating and using virtual environments
+ <http://packaging.python.org/en/latest/tutorial.html#creating-and-using-virtual-environments>`__
+
.. highlight:: none
On Windows, you may have to invoke the ``pyvenv`` script as follows, if you
@@ -25,7 +30,7 @@ or equivalently::
The command, if run with ``-h``, will show the available options::
usage: pyvenv [-h] [--system-site-packages] [--symlinks] [--clear]
- [--upgrade] ENV_DIR [ENV_DIR ...]
+ [--upgrade] [--without-pip] ENV_DIR [ENV_DIR ...]
Creates virtual Python environments in one or more target directories.
@@ -38,11 +43,19 @@ The command, if run with ``-h``, will show the available options::
virtual environment.
--symlinks Try to use symlinks rather than copies, when symlinks
are not the default for the platform.
+ --copies Try to use copies rather than symlinks, even when
+ symlinks are the default for the platform.
--clear Delete the environment directory if it already exists.
If not specified and the directory exists, an error is
raised.
--upgrade Upgrade the environment directory to use this version
of Python, assuming Python has been upgraded in-place.
+ --without-pip Skips installing or upgrading pip in the virtual
+ environment (pip is bootstrapped by default)
+
+.. versionchanged:: 3.4
+ Installs pip by default, added the ``--without-pip`` and ``--copies``
+ options
If the target directory already exists an error will be raised, unless
the ``--clear`` or ``--upgrade`` option was provided.
@@ -51,25 +64,29 @@ The created ``pyvenv.cfg`` file also includes the
``include-system-site-packages`` key, set to ``true`` if ``venv`` is
run with the ``--system-site-packages`` option, ``false`` otherwise.
+Unless the ``--without-pip`` option is given, :mod:`ensurepip` will be
+invoked to bootstrap ``pip`` into the virtual environment.
+
Multiple paths can be given to ``pyvenv``, in which case an identical
virtualenv will be created, according to the given options, at each
provided path.
Once a venv has been created, it can be "activated" using a script in the
-venv's binary directory. The invocation of the script is platform-specific: on
-a Posix platform, you would typically do::
-
- $ source <venv>/bin/activate
-
-whereas on Windows, you might do::
-
- C:\> <venv>/Scripts/activate
-
-if you are using the ``cmd.exe`` shell, or perhaps::
-
- PS C:\> <venv>/Scripts/Activate.ps1
-
-if you use PowerShell.
+venv's binary directory. The invocation of the script is platform-specific:
+
++-------------+-----------------+-----------------------------------------+
+| Platform | Shell | Command to activate virtual environment |
++=============+=================+=========================================+
+| Posix | bash/zsh | $ source <venv>/bin/activate |
++-------------+-----------------+-----------------------------------------+
+| | fish | $ . <venv>/bin/activate.fish |
++-------------+-----------------+-----------------------------------------+
+| | csh/tcsh | $ source <venv>/bin/activate.csh |
++-------------+-----------------+-----------------------------------------+
+| Windows | cmd.exe | C:\> <venv>/Scripts/activate.bat |
++-------------+-----------------+-----------------------------------------+
+| | PowerShell | PS C:\> <venv>/Scripts/Activate.ps1 |
++-------------+-----------------+-----------------------------------------+
You don't specifically *need* to activate an environment; activation just
prepends the venv's binary directory to your path, so that "python" invokes the
@@ -83,3 +100,5 @@ a "deactivate" function, whereas on Windows there are separate scripts called
``deactivate.bat`` and ``Deactivate.ps1`` which are installed when the venv is
created.
+.. versionadded:: 3.4
+ ``fish`` and ``csh`` activation scripts.
diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index 6c4d16e..42a92af 100644
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -11,6 +11,10 @@
This document aims to give an overview of Windows-specific behaviour you should
know about when using Python on Microsoft Windows.
+.. XXX (ncoghlan)
+
+ This looks rather stale to me...
+
Installing Python
=================
@@ -560,12 +564,6 @@ View the :file:`readme.txt` in their respective directories:
+--------------------+--------------+-----------------------+
| Directory | MSVC version | Visual Studio version |
+====================+==============+=======================+
-| :file:`PC/VC6/` | 6.0 | 97 |
-+--------------------+--------------+-----------------------+
-| :file:`PC/VS7.1/` | 7.1 | 2003 |
-+--------------------+--------------+-----------------------+
-| :file:`PC/VS8.0/` | 8.0 | 2005 |
-+--------------------+--------------+-----------------------+
| :file:`PC/VS9.0/` | 9.0 | 2008 |
+--------------------+--------------+-----------------------+
| :file:`PCbuild/` | 10.0 | 2010 |
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
new file mode 100644
index 0000000..177208a
--- /dev/null
+++ b/Doc/whatsnew/3.4.rst
@@ -0,0 +1,2506 @@
+****************************
+ What's New In Python 3.4
+****************************
+
+:Author: R. David Murray <rdmurray@bitdance.com> (Editor)
+
+.. Rules for maintenance:
+
+ * Anyone can add text to this document, but the maintainer reserves the
+ right to rewrite any additions. In particular, for obscure or esoteric
+ features, the maintainer may reduce any addition to a simple reference to
+ the new documentation rather than explaining the feature inline.
+
+ * While the maintainer will periodically go through Misc/NEWS
+ and add changes, it's best not to rely on this. We know from experience
+ that any changes that aren't in the What's New documentation around the
+ time of the original release will remain largely unknown to the community
+ for years, even if they're added later. We also know from experience that
+ other priorities can arise, and the maintainer will run out of time to do
+ updates -- in such cases, end users will be much better served by partial
+ notifications that at least give a hint about new features to
+ investigate.
+
+ * This is not a complete list of every single change; completeness
+ is the purpose of Misc/NEWS. The What's New should focus on changes that
+ are visible to Python *users* and that *require* a feature release (i.e.
+ most bug fixes should only be recorded in Misc/NEWS)
+
+ * PEPs should not be marked Final until they have an entry in What's New.
+ A placeholder entry that is just a section header and a link to the PEP
+ (e.g ":pep:`397` has been implemented") is acceptable. If a PEP has been
+ implemented and noted in What's New, don't forget to mark it as Final!
+
+ * If you want to draw your new text to the attention of the
+ maintainer, add 'XXX' to the beginning of the paragraph or
+ section.
+
+ * It's OK to add just a very brief note about a change. For
+ example: "The :ref:`~socket.transmogrify()` function was added to the
+ :mod:`socket` module." The maintainer will research the change and
+ write the necessary text (if appropriate). The advantage of doing this
+ is that even if no more descriptive text is ever added, readers will at
+ least have a notification that the new feature exists and a link to the
+ relevant documentation.
+
+ * You can comment out your additions if you like, but it's not
+ necessary (especially when a final release is some months away).
+
+ * Credit the author of a patch or bugfix. Just the name is
+ sufficient; the e-mail address isn't necessary.
+
+ * It's helpful to add the bug/patch number as a comment:
+
+ The :ref:`~socket.transmogrify()` function was added to the
+ :mod:`socket` module. (Contributed by P.Y. Developer in :issue:`12345`.)
+
+ This saves the maintainer the effort of going through the Mercurial log
+ when researching a change.
+
+ * Cross referencing tip: :ref:`mod.attr` will display as ``mod.attr``,
+ while :ref:`~mod.attr` will display as ``attr``.
+
+This article explains the new features in Python 3.4, compared to 3.3.
+Python 3.4 was released on March 16, 2014. For full details, see the
+`changelog <http://docs.python.org/3.4/whatsnew/changelog.html>`_.
+
+
+.. seealso::
+
+ :pep:`429` -- Python 3.4 Release Schedule
+
+
+
+Summary -- Release Highlights
+=============================
+
+.. This section singles out the most important changes in Python 3.4.
+ Brevity is key.
+
+New syntax features:
+
+* No new syntax features were added in Python 3.4.
+
+Other new features:
+
+* :ref:`pip should always be available <whatsnew-pep-453>` (:pep:`453`).
+* :ref:`Newly created file descriptors are non-inheritable <whatsnew-pep-446>`
+ (:pep:`446`).
+* command line option for :ref:`isolated mode <whatsnew-isolated-mode>`
+ (:issue:`16499`).
+* :ref:`improvements in the handling of codecs <codec-handling-improvements>`
+ that are not text encodings (multiple issues).
+* :ref:`A ModuleSpec Type <whatsnew-pep-451>` for the Import System
+ (:pep:`451`). (Affects importer authors.)
+* The :mod:`marshal` format has been made :ref:`more compact and efficient
+ <whatsnew-marshal-3>` (:issue:`16475`).
+
+New library modules:
+
+* :mod:`asyncio`: :ref:`New provisional API for asynchronous IO
+ <whatsnew-asyncio>` (:pep:`3156`).
+* :mod:`ensurepip`: :ref:`Bootstrapping the pip installer <whatsnew-ensurepip>`
+ (:pep:`453`).
+* :mod:`enum`: :ref:`Support for enumeration types <whatsnew-enum>`
+ (:pep:`435`).
+* :mod:`pathlib`: :ref:`Object-oriented filesystem paths <whatsnew-pathlib>`
+ (:pep:`428`).
+* :mod:`selectors`: :ref:`High-level and efficient I/O multiplexing
+ <whatsnew-selectors>`, built upon the :mod:`select` module primitives (part
+ of :pep:`3156`).
+* :mod:`statistics`: A basic :ref:`numerically stable statistics library
+ <whatsnew-statistics>` (:pep:`450`).
+* :mod:`tracemalloc`: :ref:`Trace Python memory allocations
+ <whatsnew-tracemalloc>` (:pep:`454`).
+
+Significantly improved library modules:
+
+* :ref:`Single-dispatch generic functions <whatsnew-singledispatch>` in
+ :mod:`functools` (:pep:`443`).
+* New :mod:`pickle` :ref:`protocol 4 <whatsnew-protocol-4>` (:pep:`3154`).
+* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork
+ on Unix <whatsnew-multiprocessing-no-fork>` (:issue:`8713`).
+* :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and
+ a new :mod:`~email.message.Message` subclass
+ (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME
+ handling <whatsnew_email_contentmanager>` (:issue:`18891`).
+* The :mod:`inspect` and :mod:`pydoc` modules are now capable of
+ correct introspection of a much wider variety of callable objects,
+ which improves the output of the Python :func:`help` system.
+* The :mod:`ipaddress` module API has been declared stable
+
+Security improvements:
+
+* :ref:`Secure and interchangeable hash algorithm <whatsnew-pep-456>`
+ (:pep:`456`).
+* :ref:`Make newly created file descriptors non-inheritable <whatsnew-pep-446>`
+ (:pep:`446`) to avoid leaking file descriptors to child processes.
+* New command line option for :ref:`isolated mode <whatsnew-isolated-mode>`,
+ (:issue:`16499`).
+* :mod:`multiprocessing` now has :ref:`an option to avoid using os.fork
+ on Unix <whatsnew-multiprocessing-no-fork>`. *spawn* and *forkserver* are
+ more secure because they avoid sharing data with child processes.
+* :mod:`multiprocessing` child processes on Windows no longer inherit
+ all of the parent's inheritable handles, only the necessary ones.
+* A new :func:`hashlib.pbkdf2_hmac` function provides
+ the `PKCS#5 password-based key derivation function 2
+ <http://en.wikipedia.org/wiki/PBKDF2>`_.
+* :ref:`TLSv1.1 and TLSv1.2 support <whatsnew-tls-11-12>` for :mod:`ssl`.
+* :ref:`Retrieving certificates from the Windows system cert store support
+ <whatsnew34-win-cert-store>` for :mod:`ssl`.
+* :ref:`Server-side SNI (Server Name Indication) support
+ <whatsnew34-sni>` for :mod:`ssl`.
+* The :class:`ssl.SSLContext` class has a :ref:`lot of improvements
+ <whatsnew34-sslcontext>`.
+* All modules in the standard library that support SSL now support server
+ certificate verification, including hostname matching
+ (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see
+ :func:`ssl.SSLContext.load_verify_locations`).
+
+CPython implementation improvements:
+
+* :ref:`Safe object finalization <whatsnew-pep-442>` (:pep:`442`).
+* Leveraging :pep:`442`, in most cases :ref:`module globals are no longer set
+ to None during finalization <whatsnew-pep-442>` (:issue:`18214`).
+* :ref:`Configurable memory allocators <whatsnew-pep-445>` (:pep:`445`).
+* :ref:`Argument Clinic <whatsnew-pep-436>` (:pep:`436`).
+
+Please read on for a comprehensive list of user-facing changes, including many
+other smaller improvements, CPython optimizations, deprecations, and potential
+porting issues.
+
+
+
+New Features
+============
+
+.. _whatsnew-pep-453:
+
+PEP 453: Explicit Bootstrapping of PIP in Python Installations
+--------------------------------------------------------------
+
+Bootstrapping pip By Default
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The new :mod:`ensurepip` module (defined in :pep:`453`) provides a standard
+cross-platform mechanism to bootstrap the pip installer into Python
+installations and virtual environments. The version of ``pip`` included
+with Python 3.4.0 is ``pip`` 1.5.4, and future 3.4.x maintenance releases
+will update the bundled version to the latest version of ``pip`` that is
+available at the time of creating the release candidate.
+
+By default, the commands ``pipX`` and ``pipX.Y`` will be installed on all
+platforms (where X.Y stands for the version of the Python installation),
+along with the ``pip`` Python package and its dependencies. On Windows and
+in virtual environments on all platforms, the unversioned ``pip`` command
+will also be installed. On other platforms, the system wide unversioned
+``pip`` command typically refers to the separately installed Python 2
+version.
+
+The :ref:`pyvenv <scripts-pyvenv>` command line utility and the :mod:`venv`
+module make use of the :mod:`ensurepip` module to make ``pip`` readily
+available in virtual environments. When using the command line utility,
+``pip`` is installed by default, while when using the :mod:`venv` module
+:ref:`venv-api` installation of ``pip`` must be requested explicitly.
+
+For CPython :ref:`source builds on POSIX systems <building-python-on-unix>`,
+the ``make install`` and ``make altinstall`` commands bootstrap ``pip`` by
+default. This behaviour can be controlled through configure options, and
+overridden through Makefile options.
+
+On Windows and Mac OS X, the CPython installers now default to installing
+``pip`` along with CPython itself (users may opt out of installing it
+during the installation process). Window users will need to opt in to the
+automatic ``PATH`` modifications to have ``pip`` available from the command
+line by default, otherwise it can still be accessed through the Python
+launcher for Windows as ``py -m pip``.
+
+As `discussed in the PEP`__, platform packagers may choose not to install
+these commands by default, as long as, when invoked, they provide clear and
+simple directions on how to install them on that platform (usually using
+the system package manager).
+
+__ http://www.python.org/dev/peps/pep-0453/#recommendations-for-downstream-distributors
+
+.. note::
+
+ To avoid conflicts between parallel Python 2 and Python 3 installations,
+ only the versioned ``pip3`` and ``pip3.4`` commands are bootstrapped by
+ default when ``ensurepip`` is invoked directly - the ``--default-pip``
+ option is needed to also request the unversioned ``pip`` command.
+ ``pyvenv`` and the Windows installer ensure that the unqualified ``pip``
+ command is made available in those environments, and ``pip`` can always be
+ invoked via the ``-m`` switch rather than directly to avoid ambiguity on
+ systems with multiple Python installations.
+
+
+Documentation Changes
+~~~~~~~~~~~~~~~~~~~~~
+
+As part of this change, the :ref:`installing-index` and
+:ref:`distributing-index` sections of the documentation have been
+completely redesigned as short getting started and FAQ documents. Most
+packaging documentation has now been moved out to the Python Packaging
+Authority maintained `Python Packaging User Guide
+<http://packaging.python.org>`__ and the documentation of the individual
+projects.
+
+However, as this migration is currently still incomplete, the legacy
+versions of those guides remaining available as :ref:`install-index`
+and :ref:`distutils-index`.
+
+.. seealso::
+
+ :pep:`453` -- Explicit bootstrapping of pip in Python installations
+ PEP written by Donald Stufft and Nick Coghlan, implemented by
+ Donald Stufft, Nick Coghlan, Martin von Löwis and Ned Deily.
+
+
+.. _whatsnew-pep-446:
+
+PEP 446: Newly Created File Descriptors Are Non-Inheritable
+-----------------------------------------------------------
+
+:pep:`446` makes newly created file descriptors :ref:`non-inheritable
+<fd_inheritance>`. In general, this is the behavior an application will
+want: when launching a new process, having currently open files also
+open in the new process can lead to all sorts of hard to find bugs,
+and potentially to security issues.
+
+However, there are occasions when inheritance is desired. To support
+these cases, the following new functions and methods are available:
+
+* :func:`os.get_inheritable`, :func:`os.set_inheritable`
+* :func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable`
+* :meth:`socket.socket.get_inheritable`, :meth:`socket.socket.set_inheritable`
+
+.. seealso::
+
+ :pep:`446` -- Make newly created file descriptors non-inheritable
+ PEP written and implemented by Victor Stinner.
+
+
+.. _codec-handling-improvements:
+
+Improvements to Codec Handling
+------------------------------
+
+Since it was first introduced, the :mod:`codecs` module has always been
+intended to operate as a type-neutral dynamic encoding and decoding
+system. However, its close coupling with the Python text model, especially
+the type restricted convenience methods on the builtin :class:`str`,
+:class:`bytes` and :class:`bytearray` types, has historically obscured that
+fact.
+
+As a key step in clarifying the situation, the :meth:`codecs.encode` and
+:meth:`codecs.decode` convenience functions are now properly documented in
+Python 2.7, 3.3 and 3.4. These functions have existed in the :mod:`codecs`
+module (and have been covered by the regression test suite) since Python 2.4,
+but were previously only discoverable through runtime introspection.
+
+Unlike the convenience methods on :class:`str`, :class:`bytes` and
+:class:`bytearray`, the :mod:`codecs` convenience functions support arbitrary
+codecs in both Python 2 and Python 3, rather than being limited to Unicode text
+encodings (in Python 3) or ``basestring`` <-> ``basestring`` conversions (in
+Python 2).
+
+In Python 3.4, the interpreter is able to identify the known non-text
+encodings provided in the standard library and direct users towards these
+general purpose convenience functions when appropriate::
+
+ >>> b"abcdef".decode("hex")
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ LookupError: 'hex' is not a text encoding; use codecs.decode() to handle arbitrary codecs
+
+ >>> "hello".encode("rot13")
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ LookupError: 'rot13' is not a text encoding; use codecs.encode() to handle arbitrary codecs
+
+ >>> open("foo.txt", encoding="hex")
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ LookupError: 'hex' is not a text encoding; use codecs.open() to handle arbitrary codecs
+
+In a related change, whenever it is feasible without breaking backwards
+compatibility, exceptions raised during encoding and decoding operations
+are wrapped in a chained exception of the same type that mentions the
+name of the codec responsible for producing the error::
+
+ >>> import codecs
+
+ >>> codecs.decode(b"abcdefgh", "hex")
+ Traceback (most recent call last):
+ File "/usr/lib/python3.4/encodings/hex_codec.py", line 20, in hex_decode
+ return (binascii.a2b_hex(input), len(input))
+ binascii.Error: Non-hexadecimal digit found
+
+ The above exception was the direct cause of the following exception:
+
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ binascii.Error: decoding with 'hex' codec failed (Error: Non-hexadecimal digit found)
+
+ >>> codecs.encode("hello", "bz2")
+ Traceback (most recent call last):
+ File "/usr/lib/python3.4/encodings/bz2_codec.py", line 17, in bz2_encode
+ return (bz2.compress(input), len(input))
+ File "/usr/lib/python3.4/bz2.py", line 498, in compress
+ return comp.compress(data) + comp.flush()
+ TypeError: 'str' does not support the buffer interface
+
+ The above exception was the direct cause of the following exception:
+
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ TypeError: encoding with 'bz2' codec failed (TypeError: 'str' does not support the buffer interface)
+
+Finally, as the examples above show, these improvements have permitted
+the restoration of the convenience aliases for the non-Unicode codecs that
+were themselves restored in Python 3.2. This means that encoding binary data
+to and from its hexadecimal representation (for example) can now be written
+as::
+
+ >>> from codecs import encode, decode
+ >>> encode(b"hello", "hex")
+ b'68656c6c6f'
+ >>> decode(b"68656c6c6f", "hex")
+ b'hello'
+
+The binary and text transforms provided in the standard library are detailed
+in :ref:`binary-transforms` and :ref:`text-transforms`.
+
+(Contributed by Nick Coghlan in :issue:`7475`, :issue:`17827`,
+:issue:`17828` and :issue:`19619`)
+
+
+.. _whatsnew-pep-451:
+
+PEP 451: A ModuleSpec Type for the Import System
+------------------------------------------------
+
+:pep:`451` provides an encapsulation of the information about a module that the
+import machinery will use to load it (that is, a module specification). This
+helps simplify both the import implementation and several import-related APIs.
+The change is also a stepping stone for `several future import-related
+improvements`__.
+
+__ https://mail.python.org/pipermail/python-dev/2013-November/130111.html
+
+The public-facing changes from the PEP are entirely backward-compatible.
+Furthermore, they should be transparent to everyone but importer authors. Key
+finder and loader methods have been deprecated, but they will continue working.
+New importers should use the new methods described in the PEP. Existing
+importers should be updated to implement the new methods. See the
+:ref:`deprecated-3.4` section for a list of methods that should be replaced and
+their replacements.
+
+
+Other Language Changes
+----------------------
+
+Some smaller changes made to the core Python language are:
+
+* Unicode database updated to UCD version 6.3.
+
+* :func:`min` and :func:`max` now accept a *default* keyword-only argument that
+ can be used to specify the value they return if the iterable they are
+ evaluating has no elements. (Contributed by Julian Berman in
+ :issue:`18111`.)
+
+* Module objects are now :mod:`weakref`'able.
+
+* Module ``__file__`` attributes (and related values) should now always
+ contain absolute paths by default, with the sole exception of
+ ``__main__.__file__`` when a script has been executed directly using
+ a relative path (Contributed by Brett Cannon in :issue:`18416`).
+
+* All the UTF-\* codecs (except UTF-7) now reject surrogates during both
+ encoding and decoding unless the ``surrogatepass`` error handler is used,
+ with the exception of the UTF-16 decoder (which accepts valid surrogate pairs)
+ and the UTF-16 encoder (which produces them while encoding non-BMP characters).
+ Contributed by Victor Stinner, Kang-Hao (Kenny) Lu and Serhiy Storchaka in
+ :issue:`12892`.
+
+* New German EBCDIC :ref:`codec <standard-encodings>` ``cp273``. (Contributed
+ by Michael Bierenfeld and Andrew Kuchling in :issue:`1097797`.)
+
+* New Ukrainian :ref:`codec <standard-encodings>` ``cp1125``. (Contributed by
+ Serhiy Storchaka in :issue:`19668`.)
+
+* :class:`bytes`.join() and :class:`bytearray`.join() now accept arbitrary
+ buffer objects as arguments. (Contributed by Antoine Pitrou in
+ :issue:`15958`.)
+
+* The :class:`int` constructor now accepts any object that has an ``__index__``
+ method for its *base* argument. (Contributed by Mark Dickinson in
+ :issue:`16772`.)
+
+* Frame objects now have a :func:`~frame.clear` method that clears all
+ references to local variables from the frame. (Contributed by Antoine Pitrou
+ in :issue:`17934`.)
+
+* :class:`memoryview` is now registered as a :class:`Sequence <collections.abc>`,
+ and supports the :func:`reversed` builtin. (Contributed by Nick Coghlan
+ and Claudiu Popa in :issue:`18690` and :issue:`19078`.)
+
+* Signatures reported by :func:`help` have been modified and improved in
+ several cases as a result of the introduction of Argument Clinic and other
+ changes to the :mod:`inspect` and :mod:`pydoc` modules.
+
+* :meth:`~object.__length_hint__` is now part of the formal language
+ specification (see :pep:`424`). (Contributed by Armin Ronacher in
+ :issue:`16148`.)
+
+
+New Modules
+===========
+
+
+.. _whatsnew-asyncio:
+
+asyncio
+-------
+
+The new :mod:`asyncio` module (defined in :pep:`3156`) provides a standard
+pluggable event loop model for Python, providing solid asynchronous IO
+support in the standard library, and making it easier for other event loop
+implementations to interoperate with the standard library and each other.
+
+For Python 3.4, this module is considered a :term:`provisional API`.
+
+.. seealso::
+
+ :pep:`3156` -- Asynchronous IO Support Rebooted: the "asyncio" Module
+ PEP written and implementation led by Guido van Rossum.
+
+
+.. _whatsnew-ensurepip:
+
+ensurepip
+---------
+
+The new :mod:`ensurepip` module is the primary infrastructure for the
+:pep:`453` implementation. In the normal course of events end users will not
+need to interact with this module, but it can be used to manually bootstrap
+``pip`` if the automated bootstrapping into an installation or virtual
+environment was declined.
+
+:mod:`ensurepip` includes a bundled copy of ``pip``, up-to-date as of the first
+release candidate of the release of CPython with which it ships (this applies
+to both maintenance releases and feature releases). ``ensurepip`` does not
+access the internet. If the installation has Internet access, after
+``ensurepip`` is run the bundled ``pip`` can be used to upgrade ``pip`` to a
+more recent release than the bundled one. (Note that such an upgraded version
+of ``pip`` is considered to be a separately installed package and will not be
+removed if Python is uninstalled.)
+
+The module is named *ensure*\ pip because if called when ``pip`` is already
+installed, it does nothing. It also has an ``--upgrade`` option that will
+cause it to install the bundled copy of ``pip`` if the existing installed
+version of ``pip`` is older than the bundled copy.
+
+
+.. _whatsnew-enum:
+
+enum
+----
+
+The new :mod:`enum` module (defined in :pep:`435`) provides a standard
+implementation of enumeration types, allowing other modules (such as
+:mod:`socket`) to provide more informative error messages and better
+debugging support by replacing opaque integer constants with backwards
+compatible enumeration values.
+
+.. seealso::
+
+ :pep:`435` -- Adding an Enum type to the Python standard library
+ PEP written by Barry Warsaw, Eli Bendersky and Ethan Furman,
+ implemented by Ethan Furman.
+
+
+.. _whatsnew-pathlib:
+
+pathlib
+-------
+
+The new :mod:`pathlib` module offers classes representing filesystem paths
+with semantics appropriate for different operating systems. Path classes are
+divided between *pure paths*, which provide purely computational operations
+without I/O, and *concrete paths*, which inherit from pure paths but also
+provide I/O operations.
+
+For Python 3.4, this module is considered a :term:`provisional API`.
+
+.. seealso::
+
+ :pep:`428` -- The pathlib module -- object-oriented filesystem paths
+ PEP written and implemented by Antoine Pitrou.
+
+
+.. _whatsnew-selectors:
+
+selectors
+---------
+
+The new :mod:`selectors` module (created as part of implementing :pep:`3156`)
+allows high-level and efficient I/O multiplexing, built upon the
+:mod:`select` module primitives.
+
+
+.. _whatsnew-statistics:
+
+statistics
+----------
+
+The new :mod:`statistics` module (defined in :pep:`450`) offers some core
+statistics functionality directly in the standard library. This module
+supports calculation of the mean, median, mode, variance and standard
+deviation of a data series.
+
+.. seealso::
+
+ :pep:`450` -- Adding A Statistics Module To The Standard Library
+ PEP written and implemented by Steven D'Aprano
+
+.. _whatsnew-tracemalloc:
+
+
+tracemalloc
+-----------
+
+The new :mod:`tracemalloc` module (defined in :pep:`454`) is a debug tool to
+trace memory blocks allocated by Python. It provides the following information:
+
+* Trace where an object was allocated
+* Statistics on allocated memory blocks per filename and per line number:
+ total size, number and average size of allocated memory blocks
+* Compute the differences between two snapshots to detect memory leaks
+
+.. seealso::
+
+ :pep:`454` -- Add a new tracemalloc module to trace Python memory allocations
+ PEP written and implemented by Victor Stinner
+
+
+
+Improved Modules
+================
+
+
+abc
+---
+
+New function :func:`abc.get_cache_token` can be used to know when to invalidate
+caches that are affected by changes in the object graph. (Contributed
+by Łukasz Langa in :issue:`16832`.)
+
+New class :class:`~abc.ABC` has :class:`~abc.ABCMeta` as its meta class.
+Using ``ABC`` as a base class has essentially the same effect as specifying
+``metaclass=abc.ABCMeta``, but is simpler to type and easier to read.
+(Contributed by Bruno Dupuis in :issue:`16049`.)
+
+
+aifc
+----
+
+The :meth:`~aifc.aifc.getparams` method now returns a namedtuple rather than a
+plain tuple. (Contributed by Claudiu Popa in :issue:`17818`.)
+
+:func:`aifc.open` now supports the context manager protocol: when used in a
+:keyword:`with` block, the :meth:`~aifc.aifc.close` method of the returned
+object will be called automatically at the end of the block. (Contributed by
+Serhiy Storchacha in :issue:`16486`.)
+
+The :meth:`~aifc.aifc.writeframesraw` and :meth:`~aifc.aifc.writeframes`
+methods now accept any :term:`bytes-like object`. (Contributed by Serhiy
+Storchaka in :issue:`8311`.)
+
+
+argparse
+--------
+
+The :class:`~argparse.FileType` class now accepts *encoding* and
+*errors* arguments, which are passed through to :func:`open`. (Contributed
+by Lucas Maystre in :issue:`11175`.)
+
+
+audioop
+-------
+
+:mod:`audioop` now supports 24-bit samples. (Contributed by Serhiy Storchaka
+in :issue:`12866`.)
+
+New :func:`~audioop.byteswap` function converts big-endian samples to
+little-endian and vice versa (Contributed by Serhiy Storchaka in
+:issue:`19641`).
+
+All :mod:`audioop` functions now accept any :term:`bytes-like object`. Strings
+are not accepted: they didn't work before, now they raise an error right away.
+(Contributed by Serhiy Storchaka in :issue:`16685`.)
+
+
+base64
+------
+
+The encoding and decoding functions in :mod:`base64` now accept any
+:term:`bytes-like object` in cases where it previously required a
+:class:`bytes` or :class:`bytearray` instance. (Contributed by Nick Coghlan in
+:issue:`17839`.)
+
+New functions :func:`~base64.a85encode`, :func:`~base64.a85decode`,
+:func:`~base64.b85encode`, and :func:`~base64.b85decode` provide the ability to
+encode and decode binary data from and to ``Ascii85`` and the git/mercurial
+``Base85`` formats, respectively. The ``a85`` functions have options that can
+be used to make them compatible with the variants of the ``Ascii85`` encoding,
+including the Adobe variant. (Contributed by Martin Morrison, the Mercurial
+project, Serhiy Storchaka, and Antoine Pitrou in :issue:`17618`.)
+
+
+collections
+-----------
+
+The :meth:`.ChainMap.new_child` method now accepts an *m* argument specifying
+the child map to add to the chain. This allows an existing mapping and/or a
+custom mapping type to be used for the child. (Contributed by Vinay Sajip in
+:issue:`16613`.)
+
+
+colorsys
+--------
+
+The number of digits in the coefficients for the RGB --- YIQ conversions have
+been expanded so that they match the FCC NTSC versions. The change in
+results should be less than 1% and may better match results found elsewhere.
+(Contributed by Brian Landers and Serhiy Storchaka in :issue:`14323`.)
+
+
+contextlib
+----------
+
+The new :class:`contextlib.suppress` context manager helps to clarify the
+intent of code that deliberately suppresses exceptions from a single
+statement. (Contributed by Raymond Hettinger in :issue:`15806` and
+Zero Piraeus in :issue:`19266`)
+
+The new :func:`contextlib.redirect_stdout` context manager makes it easier
+for utility scripts to handle inflexible APIs that write their output to
+:data:`sys.stdout` and don't provide any options to redirect it. Using the
+context manager, the :data:`sys.stdout` output can be redirected to any
+other stream or, in conjunction with :class:`io.StringIO`, to a string.
+The latter can be especially useful, for example, to capture output
+from a function that was written to implement a command line interface.
+It is recommended only for utility scripts because it affects the
+global state of :data:`sys.stdout`. (Contributed by Raymond Hettinger
+in :issue:`15805`)
+
+The :mod:`contextlib` documentation has also been updated to include a
+:ref:`discussion <single-use-reusable-and-reentrant-cms>` of the
+differences between single use, reusable and reentrant context managers.
+
+
+dbm
+---
+
+:func:`dbm.open` objects now support the context management protocol. When
+used in a :keyword:`with` statement, the ``close`` method of the database
+object will be called automatically at the end of the block. (Contributed by
+Claudiu Popa and Nick Coghlan in :issue:`19282`.)
+
+
+dis
+---
+
+Functions :func:`~dis.show_code`, :func:`~dis.dis`, :func:`~dis.distb`, and
+:func:`~dis.disassemble` now accept a keyword-only *file* argument that
+controls where they write their output.
+
+The :mod:`dis` module is now built around an :class:`~dis.Instruction` class
+that provides object oriented access to the details of each individual bytecode
+operation.
+
+A new method, :func:`~dis.get_instructions`, provides an iterator that emits
+the Instruction stream for a given piece of Python code. Thus it is now
+possible to write a program that inspects and manipulates a bytecode
+object in ways different from those provided by the :mod:`~dis` module
+itself. For example::
+
+ >>> import dis
+ >>> for instr in dis.get_instructions(lambda x: x + 1):
+ ... print(instr.opname)
+ LOAD_FAST
+ LOAD_CONST
+ BINARY_ADD
+ RETURN_VALUE
+
+The various display tools in the :mod:`dis` module have been rewritten to use
+these new components.
+
+In addition, a new application-friendly class :class:`~dis.Bytecode` provides
+an object-oriented API for inspecting bytecode in both in human-readable form
+and for iterating over instructions. The :class:`~dis.Bytecode` constructor
+takes the same arguments that :func:`~dis.get_instruction` does (plus an
+optional *current_offset*), and the resulting object can be iterated to produce
+:class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis`
+method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but
+returned as a multi-line string::
+
+ >>> bytecode = dis.Bytecode(lambda x: x +1, current_offset=3)
+ >>> for instr in bytecode:
+ ... print('{} ({})'.format(instr.opname, instr.opcode))
+ LOAD_FAST (124)
+ LOAD_CONST (100)
+ BINARY_ADD (23)
+ RETURN_VALUE (83)
+ >>> bytecode.dis().splitlines() # doctest: +NORMALIZE_WHITESPACE
+ [' 1 0 LOAD_FAST 0 (x)',
+ ' --> 3 LOAD_CONST 1 (1)',
+ ' 6 BINARY_ADD',
+ ' 7 RETURN_VALUE']
+
+:class:`~dis.Bytecode` also has a class method,
+:meth:`~dis.Bytecode.from_traceback`, that provides the ability to manipulate a
+traceback (that is, ``print(Bytecode.from_traceback(tb).dis())`` is equivalent
+to ``distb(tb)``).
+
+(Contributed by Nick Coghlan, Ryan Kelly and Thomas Kluyver in :issue:`11816`
+and Claudiu Popa in :issue:`17916`)
+
+New function :func:`~dis.stack_effect` computes the effect on the Python stack
+of a given opcode and argument, information that is not otherwise available.
+(Contributed by Larry Hastings in :issue:`19722`.)
+
+
+doctest
+-------
+
+A new :ref:`option flag <doctest-options>`, :data:`~doctest.FAIL_FAST`, halts
+test running as soon as the first failure is detected. (Contributed by R.
+David Murray and Daniel Urban in :issue:`16522`.)
+
+The :mod:`doctest` command line interface now uses :mod:`argparse`, and has two
+new options, ``-o`` and ``-f``. ``-o`` allows :ref:`doctest options
+<doctest-options>` to be specified on the command line, and ``-f`` is a
+shorthand for ``-o FAIL_FAST`` (to parallel the similar option supported by the
+:mod:`unittest` CLI). (Contributed by R. David Murray in :issue:`11390`.)
+
+:mod:`doctest` will now find doctests in extension module ``__doc__`` strings.
+(Contributed by Zachary Ware in :issue:`3158`.)
+
+
+email
+-----
+
+:meth:`~email.message.Message.as_string` now accepts a *policy* argument to
+override the default policy of the message when generating a string
+representation of it. This means that ``as_string`` can now be used in more
+circumstances, instead of having to create and use a :mod:`~email.generator` in
+order to pass formatting parameters to its ``flatten`` method. (Contributed by
+R. David Murray in :issue:`18600`.)
+
+New method :meth:`~email.message.Message.as_bytes` added to produce a bytes
+representation of the message in a fashion similar to how ``as_string``
+produces a string representation. It does not accept the *maxheaderlen*
+argument, but does accept the *unixfrom* and *policy* arguments. The
+:class:`~email.message.Message` :meth:`~email.message.Message.__bytes__` method
+calls it, meaning that ``bytes(mymsg)`` will now produce the intuitive
+result: a bytes object containing the fully formatted message. (Contributed
+by R. David Murray in :issue:`18600`.)
+
+The :meth:`.Message.set_param` message now accepts a *replace* keyword argument.
+When specified, the associated header will be updated without changing
+its location in the list of headers. For backward compatibility, the default
+is ``False``. (Contributed by R. David Murray in :issue:`18891`.)
+
+
+.. _whatsnew_email_contentmanager:
+
+A pair of new subclasses of :class:`~email.message.Message` have been added
+(:class:`.EmailMessage` and :class:`.MIMEPart`), along with a new sub-module,
+:mod:`~email.contentmanager` and a new :mod:`~email.policy` attribute
+:attr:`~email.policy.EmailPolicy.content_manager`. All documentation is
+currently in the new module, which is being added as part of email's new
+:term:`provisional API`. These classes provide a number of new methods that
+make extracting content from and inserting content into email messages much
+easier. For details, see the :mod:`~email.contentmanager` documentation and
+the :ref:`email-contentmanager-api-examples`. These API additions complete the
+bulk of the work that was planned as part of the email6 project. The currently
+provisional API is scheduled to become final in Python 3.5 (possibly with a few
+minor additions in the area of error handling). (Contributed by R. David
+Murray in :issue:`18891`.)
+
+
+filecmp
+-------
+
+A new :func:`~filecmp.clear_cache` function provides the ability to clear the
+:mod:`filecmp` comparison cache, which uses :func:`os.stat` information to
+determine if the file has changed since the last compare. This can be used,
+for example, if the file might have been changed and re-checked in less time
+than the resolution of a particular filesystem's file modification time field.
+(Contributed by Mark Levitt in :issue:`18149`.)
+
+New module attribute :data:`~filecmp.DEFAULT_IGNORES` provides the list of
+directories that are used as the default value for the *ignore* parameter of
+the :func:`~filecmp.dircmp` function. (Contributed by Eli Bendersky in
+:issue:`15442`.)
+
+
+functools
+---------
+
+The new :func:`~functools.partialmethod` descriptor brings partial argument
+application to descriptors, just as :func:`~functools.partial` provides
+for normal callables. The new descriptor also makes it easier to get
+arbitrary callables (including :func:`~functools.partial` instances)
+to behave like normal instance methods when included in a class definition.
+(Contributed by Alon Horev and Nick Coghlan in :issue:`4331`)
+
+.. _whatsnew-singledispatch:
+
+The new :func:`~functools.singledispatch` decorator brings support for
+single-dispatch generic functions to the Python standard library. Where
+object oriented programming focuses on grouping multiple operations on a
+common set of data into a class, a generic function focuses on grouping
+multiple implementations of an operation that allows it to work with
+*different* kinds of data.
+
+.. seealso::
+
+ :pep:`443` -- Single-dispatch generic functions
+ PEP written and implemented by Łukasz Langa.
+
+:func:`~functools.total_ordering` now supports a return value of
+:const:`NotImplemented` from the underlying comparison function. (Contributed
+by Katie Miller in :issue:`10042`.)
+
+A pure-python version of the :func:`~functools.partial` function is now in the
+stdlib; in CPython it is overridden by the C accelerated version, but it is
+available for other implementations to use. (Contributed by Brian Thorne in
+:issue:`12428`.)
+
+
+gc
+--
+
+New function :func:`~gc.get_stats` returns a list of three per-generation
+dictionaries containing the collections statistics since interpreter startup.
+(Contributed by Antoine Pitrou in :issue:`16351`.)
+
+
+glob
+----
+
+A new function :func:`~glob.escape` provides a way to escape special characters
+in a filename so that they do not become part of the globbing expansion but are
+instead matched literally. (Contributed by Serhiy Storchaka in :issue:`8402`.)
+
+
+hashlib
+-------
+
+A new :func:`hashlib.pbkdf2_hmac` function provides
+the `PKCS#5 password-based key derivation function 2
+<http://en.wikipedia.org/wiki/PBKDF2>`_. (Contributed by Christian
+Heimes in :issue:`18582`)
+
+The :attr:`~hashlib.hash.name` attribute of :mod:`hashlib` hash objects is now
+a formally supported interface. It has always existed in CPython's
+:mod:`hashlib` (although it did not return lower case names for all supported
+hashes), but it was not a public interface and so some other Python
+implementations have not previously supported it. (Contributed by Jason R.
+Coombs in :issue:`18532`.)
+
+
+hmac
+----
+
+:mod:`hmac` now accepts ``bytearray`` as well as ``bytes`` for the *key*
+argument to the :func:`~hmac.new` function, and the *msg* parameter to both the
+:func:`~hmac.new` function and the :meth:`~hmac.HMAC.update` method now
+accepts any type supported by the :mod:`hashlib` module. (Contributed
+by Jonas Borgström in :issue:`18240`.)
+
+The *digestmod* argument to the :func:`hmac.new` function may now be any hash
+digest name recognized by :mod:`hashlib`. In addition, the current behavior in
+which the value of *digestmod* defaults to ``MD5`` is deprecated: in a
+future version of Python there will be no default value. (Contributed by
+Christian Heimes in :issue:`17276`.)
+
+With the addition of :attr:`~hmac.HMAC.block_size` and :attr:`~hmac.HMAC.name`
+attributes (and the formal documentation of the :attr:`~hmac.HMAC.digest_size`
+attribute), the :mod:`hmac` module now conforms fully to the :pep:`247` API.
+(Contributed by Christian Heimes in :issue:`18775`.)
+
+
+html
+----
+
+New function :func:`~html.unescape` function converts HTML5 character references to
+the corresponding Unicode characters. (Contributed by Ezio Melotti in
+:issue:`2927`)
+
+:class:`~html.parser.HTMLParser` accepts a new keyword argument
+*convert_charrefs* that, when ``True``, automatically converts all character
+references. For backward-compatibility, its value defaults to ``False``, but
+it will change to ``True`` in a future version of Python, so you are invited to
+set it explicitly and update your code to use this new feature. (Contributed
+by Ezio Melotti in :issue:`13633`)
+
+The *strict* argument of :class:`~html.parser.HTMLParser` is now deprecated.
+(Contributed by Ezio Melotti in :issue:`15114`)
+
+
+http
+----
+
+:meth:`~http.server.BaseHTTPRequestHandler.send_error` now accepts an
+optional additional *explain* parameter which can be used to provide an
+extended error description, overriding the hardcoded default if there is one.
+This extended error description will be formatted using the
+:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body
+of the error response. (Contributed by Karl Cow in :issue:`12921`.)
+
+The :mod:`http.server` :ref:`command line interface <http-server-cli>` now has
+a ``-b/--bind`` option that causes the server to listen on a specific address.
+(Contributed by Malte Swart in :issue:`17764`.)
+
+
+importlib
+---------
+
+The :class:`~importlib.abc.InspectLoader` ABC defines a new method,
+:meth:`~importlib.abc.InspectLoader.source_to_code` that accepts source
+data and a path and returns a code object. The default implementation
+is equivalent to ``compile(data, path, 'exec', dont_inherit=True)``.
+(Contributed by Eric Snow and Brett Cannon in :issue:`15627`.)
+
+:class:`~importlib.abc.InspectLoader` also now has a default implementation
+for the :meth:`~importlib.abc.InspectLoader.get_code` method. However,
+it will normally be desirable to override the default implementation
+for performance reasons. (Contributed by Brett Cannon in :issue:`18072`.)
+
+The :func:`~importlib.reload` function has been moved from :mod:`imp` to
+:mod:`importlib` as part of the :mod:`imp` module deprecation. (Contributed by
+Berker Peksag in :issue:`18193`.)
+
+:mod:`importlib.util` now has a :data:`~importlib.util.MAGIC_NUMBER` attribute
+providing access to the bytecode version number. This replaces the
+:func:`~imp.get_magic` function in the deprecated :mod:`imp` module.
+(Contributed by Brett Cannon in :issue:`18192`.)
+
+New :mod:`importlib.util` functions :func:`~importlib.util.cache_from_source`
+and :func:`~importlib.util.source_from_cache` replace the same-named functions
+in the deprecated :mod:`imp` module. (Contributed by Brett Cannon in
+:issue:`18194`.)
+
+The :mod:`importlib` bootstrap :class:`.NamespaceLoader` now conforms to
+the :class:`.InspectLoader` ABC, which means that ``runpy`` and
+``python -m`` can now be used with namespace packages. (Contributed
+by Brett Cannon in :issue:`18058`.)
+
+:mod:`importlib.util` has a new function :func:`~importlib.util.decode_source`
+that decodes source from bytes using universal newline processing. This is
+useful for implementing :meth:`.InspectLoader.get_source` methods.
+
+:class:`importlib.machinery.ExtensionFileLoader` now has a
+:meth:`~importlib.machinery.ExtensionFileLoader.get_filename` method. This was
+inadvertently omitted in the original implementation. (Contributed by Eric
+Snow in :issue:`19152`.)
+
+
+inspect
+-------
+
+The :mod:`inspect` module now offers a basic :ref:`command line interface
+<inspect-module-cli>` to quickly display source code and other
+information for modules, classes and functions. (Contributed by Claudiu Popa
+and Nick Coghlan in :issue:`18626`)
+
+:func:`~inspect.unwrap` makes it easy to unravel wrapper function chains
+created by :func:`functools.wraps` (and any other API that sets the
+``__wrapped__`` attribute on a wrapper function). (Contributed by
+Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`)
+
+As part of the implementation of the new :mod:`enum` module, the
+:mod:`inspect` module now has substantially better support for custom
+``__dir__`` methods and dynamic class attributes provided through
+metaclasses (Contributed by Ethan Furman in :issue:`18929` and
+:issue:`19030`)
+
+:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec`
+now use the :func:`~inspect.signature` API. This allows them to
+support a much broader range of callables, including those with
+``__signature__`` attributes, those with metadata provided by argument
+clinic, :func:`functools.partial` objects and more. Note that, unlike
+:func:`~inspect.signature`, these functions still ignore ``__wrapped__``
+attributes, and report the already bound first argument for bound methods,
+so it is still necessary to update your code to use
+:func:`~inspect.signature` directly if those features are desired.
+(Contributed by Yury Selivanov in :issue:`17481`)
+
+:func:`~inspect.signature` now supports duck types of CPython functions,
+which adds support for functions compiled with Cython. (Contributed
+by Stefan Behnel and Yury Selivanov in :issue:`17159`)
+
+
+ipaddress
+---------
+
+:mod:`ipaddress` was added to the standard library in Python 3.3 as a
+:term:`provisional API`. With the release of Python 3.4, this qualification
+has been removed: :mod:`ipaddress` is now considered a stable API, covered
+by the normal standard library requirements to maintain backwards
+compatibility.
+
+A new :attr:`~ipaddress.IPv4Address.is_global` property is ``True`` if
+an address is globally routeable. (Contributed by Peter Moody in
+:issue:`17400`.)
+
+
+logging
+-------
+
+The :class:`~logging.handlers.TimedRotatingFileHandler` has a new *atTime*
+parameter that can be used to specify the time of day when rollover should
+happen. (Contributed by Ronald Oussoren in :issue:`9556`.)
+
+:class:`~logging.handlers.SocketHandler` and
+:class:`~logging.handlers.DatagramHandler` now support Unix domain sockets (by
+setting *port* to ``None``). (Contributed by Vinay Sajip in commit
+ce46195b56a9.)
+
+:func:`~logging.config.fileConfig` now accepts a
+:class:`configparser.RawConfigParser` subclass instance for the *fname*
+parameter. This facilitates using a configuration file when logging
+configuration is just a part of the overall application configuration, or where
+the application modifies the configuration before passing it to
+:func:`~logging.config.fileConfig`. (Contributed by Vinay Sajip in
+:issue:`16110`.)
+
+Logging configuration data received from a socket via the
+:func:`logging.config.listen` function can now be validated before being
+processed by supplying a verification function as the argument to the new
+*verify* keyword argument. (Contributed by Vinay Sajip in :issue:`15452`.)
+
+
+.. _whatsnew-marshal-3:
+
+marshal
+-------
+
+The default :mod:`marshal` version has been bumped to 3. The code implementing
+the new version restores the Python2 behavior of recording only one copy of
+interned strings and preserving the interning on deserialization, and extends
+this "one copy" ability to any object type (including handling recursive
+references). This reduces both the size of ``.pyc`` files and the amount of
+memory a module occupies in memory when it is loaded from a ``.pyc`` (or
+``.pyo``) file. (Contributed by Kristján Valur Jónsson in :issue:`16475`,
+with additional speedups by Antoine Pitrou in :issue:`19219`.)
+
+
+mmap
+----
+
+mmap objects can now be :mod:`weakref`\ ed. (Contributed by Valerie Lambert in
+:issue:`4885`.)
+
+
+multiprocessing
+---------------
+
+.. _whatsnew-multiprocessing-no-fork:
+
+On Unix two new :ref:`start methods <multiprocessing-start-methods>`,
+(``spawn`` and ``forkserver``, have been added for starting processes using
+:mod:`multiprocessing`. These make the mixing of processes with threads more
+robust, and the ``spawn`` method matches the semantics that multiprocessing has
+always used on Windows. New function
+:func:`~multiprocessing.get_all_start_methods` reports all start methods
+available on the platform, :func:`~multiprocessing.get_start_method` reports
+the current start method, and :func:`~multiprocessing.set_start_method` sets
+the start method. (Contributed by Richard Oudkerk in :issue:`8713`).
+
+:mod:`multiprocessing` also now has the concept of a ``context``, which
+determines how child processes are created. New function
+:func:`~multiprocessing.get_context` returns a context that uses a specified
+start method. It has the same API as the :mod:`multiprocessing` module itself,
+so you can use it to create :class:`~multiprocessing.pool.Pool`\ s and other
+objects that will operate within that context. This allows a framework and an
+application or different parts of the same application to use multiprocessing
+without interfering with each other. (Contributed by Richard Oudkerk in
+:issue:`18999`.)
+
+Except when using the old *fork* start method, child processes no longer
+inherit unneeded handles/file descriptors from their parents (part of
+:issue:`8713`).
+
+:mod:`multiprocessing` now relies on :mod:`runpy` (which implements the
+``-m`` switch) to initialise ``__main__`` appropriately in child processes
+when using the ``spawn`` or ``forkserver`` start methods. This resolves some
+edge cases where combining multiprocessing, the ``-m`` command line switch,
+and explicit relative imports could cause obscure failures in child
+processes. (Contributed by Nick Coghlan in :issue:`19946`)
+
+
+operator
+--------
+
+New function :func:`~operator.length_hint` provides an implementation of the
+specification for how the :meth:`~object.__length_hint__` special method should
+be used, as part of the :pep:`424` formal specification of this language
+feature. (Contributed by Armin Ronacher in :issue:`16148`.)
+
+There is now a pure-python version of the :mod:`operator` module available for
+reference and for use by alternate implementations of Python. (Contributed by
+Zachary Ware in :issue:`16694`.)
+
+
+os
+--
+
+There are new functions to get and set the :ref:`inheritable flag
+<fd_inheritance>` of a file descriptor (:func:`os.get_inheritable`,
+:func:`os.set_inheritable`) or a Windows handle
+(:func:`os.get_handle_inheritable`, :func:`os.set_handle_inheritable`).
+
+New function :func:`~os.cpu_count` reports the number of CPUs available on the
+platform on which Python is running (or ``None`` if the count can't be
+determined). The :func:`multiprocessing.cpu_count` function is now implemented
+in terms of this function). (Contributed by Trent Nelson, Yogesh Chaudhari,
+Victor Stinner, and Charles-François Natali in :issue:`17914`.)
+
+:func:`os.path.samestat` is now available on the Windows platform (and the
+:func:`os.path.samefile` implementation is now shared between Unix and
+Windows). (Contributed by Brian Curtin in :issue:`11939`.)
+
+:func:`os.path.ismount` now recognizes volumes mounted below a drive
+root on Windows. (Contributed by Tim Golden in :issue:`9035`.)
+
+:func:`os.open` supports two new flags on platforms that provide them,
+:data:`~os.O_PATH` (un-opened file descriptor), and :data:`~os.O_TMPFILE`
+(unnamed temporary file; as of 3.4.0 release available only on Linux systems
+with a kernel version of 3.11 or newer that have uapi headers). (Contributed
+by Christian Heimes in :issue:`18673` and Benjamin Peterson, respectively.)
+
+
+pdb
+---
+
+:mod:`pdb` has been enhanced to handle generators, :keyword:`yield`, and
+``yield from`` in a more useful fashion. This is especially helpful when
+debugging :mod:`asyncio` based programs. (Contributed by Andrew Svetlov and
+Xavier de Gaye in :issue:`16596`.)
+
+The ``print`` command has been removed from :mod:`pdb`, restoring access to the
+Python :func:`print` function from the pdb command line. Python2's ``pdb`` did
+not have a ``print`` command; instead, entering ``print`` executed the
+``print`` statement. In Python3 ``print`` was mistakenly made an alias for the
+pdb :pdbcmd:`p` command. ``p``, however, prints the ``repr`` of its argument,
+not the ``str`` like the Python2 ``print`` command did. Worse, the Python3
+``pdb print`` command shadowed the Python3 ``print`` function, making it
+inaccessible at the ``pdb`` prompt. (Contributed by Connor Osborn in
+:issue:`18764`.)
+
+
+.. _whatsnew-protocol-4:
+
+pickle
+------
+
+:mod:`pickle` now supports (but does not use by default) a new pickle protocol,
+protocol 4. This new protocol addresses a number of issues that were present
+in previous protocols, such as the serialization of nested classes, very large
+strings and containers, and classes whose :meth:`__new__` method takes
+keyword-only arguments. It also provides some efficiency improvements.
+
+.. seealso::
+
+ :pep:`3154` -- Pickle protocol 4
+ PEP written by Antoine Pitrou and implemented by Alexandre Vassalotti.
+
+
+plistlib
+--------
+
+:mod:`plistlib` now has an API that is similar to the standard pattern for
+stdlib serialization protocols, with new :func:`~plistlib.load`,
+:func:`~plistlib.dump`, :func:`~plistlib.loads`, and :func:`~plistlib.dumps`
+functions. (The older API is now deprecated.) In addition to the already
+supported XML plist format (:data:`~plistlib.FMT_XML`), it also now supports
+the binary plist format (:data:`~plistlib.FMT_BINARY`). (Contributed by Ronald
+Oussoren and others in :issue:`14455`).
+
+
+poplib
+------
+
+Two new methods have been added to :mod:`poplib`: :meth:`~poplib.POP3.capa`,
+which returns the list of capabilities advertised by the POP server, and
+:meth:`~poplib.POP3.stls`, which switches a clear-text POP3 session into an
+encrypted POP3 session if the POP server supports it. (Contributed by Lorenzo
+Catucci in :issue:`4473`.)
+
+
+pprint
+------
+
+The :mod:`pprint` module's :class:`~pprint.PrettyPrinter` class and its
+:func:`~pprint.pformat`, and :func:`~pprint.pprint` functions have a new
+option, *compact*, that controls how the output is formatted. Currently
+setting *compact* to ``True`` means that sequences will be printed with as many
+sequence elements as will fit within *width* on each (indented) line.
+(Contributed by Serhiy Storchaka in :issue:`19132`.)
+
+Long strings are now wrapped using Python's normal line continuation
+syntax. (Contributed by Antoine Pitrou in :issue:`17150`).
+
+
+pty
+---
+
+:func:`pty.spawn` now returns the status value from :func:`os.waitpid` on
+the child process, instead of ``None``. (Contributed by Gregory P. Smith.)
+
+
+pydoc
+-----
+
+The :mod:`pydoc` module is now based directly on the :func:`inspect.signature`
+introspection API, allowing it to provide signature information for a wider
+variety of callable objects. This change also means that ``__wrapped__``
+attributes are now taken into account when displaying help information
+(Contributed by Larry Hastings in :issue:`19674`)
+
+The :mod:`pydoc` module no longer displays the ``self`` parameter for
+already bound methods. Instead, it aims to always display the exact current
+signature of the supplied callable (Contributed by Larry Hastings in
+:issue:`20710`)
+
+In addition to the changes that have been made to :mod:`pydoc` directly,
+its handling of custom ``__dir__`` methods and various descriptor
+behaviours has also been improved substantially by the underlying changes in
+the :mod:`inspect` module.
+
+As the :func:`help` builtin is based on :mod:`pydoc`, the above changes also
+affect the behaviour of :func:`help`.
+
+
+re
+--
+
+New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor
+the pattern at both ends of the string to match. This provides a way to be
+explicit about the goal of the match, which avoids a class of subtle bugs where
+``$`` characters get lost during code changes or the addition of alternatives
+to an existing regular expression. (Contributed by Matthew Barnett in
+:issue:`16203`.)
+
+The repr of :ref:`regex objects <re-objects>` now includes the pattern
+and the flags; the repr of :ref:`match objects <match-objects>` now
+includes the start, end, and the part of the string that matched. (Contributed
+by Hugo Lopes Tavares and Serhiy Storchaka in :issue:`13592` and
+:issue:`17087`.)
+
+
+resource
+--------
+
+New :func:`~resource.prlimit` function, available on Linux platforms with a
+kernel version of 2.6.36 or later and glibc of 2.13 or later, provides the
+ability to query or set the resource limits for processes other than the one
+making the call. (Contributed by Christian Heimes in :issue:`16595`.)
+
+On Linux kernel version 2.6.36 or later, there are there are also some new
+Linux specific constants: :attr:`~resource.RLIMIT_MSGQUEUE`,
+:attr:`~resource.RLIMIT_NICE`, :attr:`~resource.RLIMIT_RTPRIO`,
+:attr:`~resource.RLIMIT_RTTIME`, and :attr:`~resource.RLIMIT_SIGPENDING`.
+(Contributed by Christian Heimes in :issue:`19324`.)
+
+On FreeBSD version 9 and later, there some new FreeBSD specific constants:
+:attr:`~resource.RLIMIT_SBSIZE`, :attr:`~resource.RLIMIT_SWAP`, and
+:attr:`~resource.RLIMIT_NPTS`. (Contributed by Claudiu Popa in
+:issue:`19343`.)
+
+
+select
+------
+
+:class:`~select.epoll` objects now support the context management protocol.
+When used in a :keyword:`with` statement, the :meth:`~select.epoll.close`
+method will be called automatically at the end of the block. (Contributed
+by Serhiy Storchaka in :issue:`16488`.)
+
+:class:`~select.devpoll` objects now have :meth:`~select.devpoll.fileno` and
+:meth:`~select.devpoll.close` methods, as well as a new attribute
+:attr:`~select.devpoll.closed`. (Contributed by Victor Stinner in
+:issue:`18794`.)
+
+
+shelve
+------
+
+:class:`~shelve.Shelf` instances may now be used in :keyword:`with` statements,
+and will be automatically closed at the end of the :keyword:`with` block.
+(Contributed by Filip Gruszczyński in :issue:`13896`.)
+
+
+shutil
+------
+
+:func:`~shutil.copyfile` now raises a specific :exc:`~shutil.Error` subclass,
+:exc:`~shutil.SameFileError`, when the source and destination are the same
+file, which allows an application to take appropriate action on this specific
+error. (Contributed by Atsuo Ishimoto and Hynek Schlawack in
+:issue:`1492704`.)
+
+
+smtpd
+-----
+
+The :class:`~smtpd.SMTPServer` and :class:`~smtpd.SMTPChannel` classes now
+accept a *map* keyword argument which, if specified, is passed in to
+:class:`asynchat.async_chat` as its *map* argument. This allows an application
+to avoid affecting the global socket map. (Contributed by Vinay Sajip in
+:issue:`11959`.)
+
+
+smtplib
+-------
+
+:exc:`~smtplib.SMTPException` is now a subclass of :exc:`OSError`, which allows
+both socket level errors and SMTP protocol level errors to be caught in one
+try/except statement by code that only cares whether or not an error occurred.
+(Contributed by Ned Jackson Lovely in :issue:`2118`).
+
+
+socket
+------
+
+The socket module now supports the :data:`~socket.CAN_BCM` protocol on
+platforms that support it. (Contributed by Brian Thorne in :issue:`15359`.)
+
+Socket objects have new methods to get or set their :ref:`inheritable flag
+<fd_inheritance>`, :meth:`~socket.socket.get_inheritable` and
+:meth:`~socket.socket.set_inheritable`.
+
+The ``socket.AF_*`` and ``socket.SOCK_*`` constants are now enumeration values
+using the new :mod:`enum` module. This allows meaningful names to be printed
+during debugging, instead of integer "magic numbers".
+
+The :data:`~socket.AF_LINK` constant is now available on BSD and OSX.
+
+:func:`~socket.inet_pton` and :func:`~socket.inet_ntop` are now supported
+on Windows. (Contributed by Atsuo Ishimoto in :issue:`7171`.)
+
+
+sqlite3
+-------
+
+A new boolean parameter to the :func:`~sqlite3.connect` function, *uri*, can be
+used to indicate that the *database* parameter is a ``uri`` (see the `SQLite
+URI documentation <http://www.sqlite.org/uri.html>`_). (Contributed by poq in
+:issue:`13773`.)
+
+
+ssl
+---
+
+.. _whatsnew-tls-11-12:
+
+:data:`~ssl.PROTOCOL_TLSv1_1` and :data:`~ssl.PROTOCOL_TLSv1_2` (TLSv1.1 and
+TLSv1.2 support) have been added; support for these protocols is only available if
+Python is linked with OpenSSL 1.0.1 or later. (Contributed by Michele Orrù and
+Antoine Pitrou in :issue:`16692`)
+
+.. _whatsnew34-sslcontext:
+
+New function :func:`~ssl.create_default_context` provides a standard way to
+obtain an :class:`~ssl.SSLContext` whose settings are intended to be a
+reasonable balance between compatibility and security. These settings are
+more stringent than the defaults provided by the :class:`~ssl.SSLContext`
+constructor, and may be adjusted in the future, without prior deprecation, if
+best-practice security requirements change. The new recommended best
+practice for using stdlib libraries that support SSL is to use
+:func:`~ssl.create_default_context` to obtain an :class:`~ssl.SSLContext`
+object, modify it if needed, and then pass it as the *context* argument
+of the appropriate stdlib API. (Contributed by Christian Heimes
+in :issue:`19689`.)
+
+:class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_verify_locations`
+accepts a new optional argument *cadata*, which can be used to provide PEM or
+DER encoded certificates directly via strings or bytes, respectively.
+(Contributed by Christian Heimes in :issue:`18138`.)
+
+New function :func:`~ssl.get_default_verify_paths` returns
+a named tuple of the paths and environment variables that the
+:meth:`~ssl.SSLContext.set_default_verify_paths` method uses to set
+OpenSSL's default ``cafile`` and ``capath``. This can be an aid in
+debugging default verification issues. (Contributed by Christian Heimes
+in :issue:`18143`.)
+
+:class:`~ssl.SSLContext` has a new method,
+:meth:`~ssl.SSLContext.cert_store_stats`, that reports the number of loaded
+``X.509`` certs, ``X.509 CA`` certs, and certificate revocation lists (``crl``\
+s), as well as a :meth:`~ssl.SSLContext.get_ca_certs` method that returns a
+list of the loaded ``CA`` certificates. (Contributed by Christian Heimes in
+:issue:`18147`.)
+
+If OpenSSL 0.9.8 or later is available, :class:`~ssl.SSLContext` has an new
+attribute :attr:`~ssl.SSLContext.verify_flags` that can be used to control the
+certificate verification process by setting it to some combination of the new
+constants :data:`~ssl.VERIFY_DEFAULT`, :data:`~ssl.VERIFY_CRL_CHECK_LEAF`,
+:data:`~ssl.VERIFY_CRL_CHECK_CHAIN`, or :data:`~ssl.VERIFY_X509_STRICT`.
+OpenSSL does not do any CRL verification by default. (Contributed by
+Christien Heimes in :issue:`8813`.)
+
+New :class:`~ssl.SSLContext` method :meth:`~ssl.SSLContext.load_default_certs`
+loads a set of default "certificate authority" (CA) certificates from default
+locations, which vary according to the platform. It can be used to load both
+TLS web server authentication certificates
+(``purpose=``:data:`~ssl.Purpose.SERVER_AUTH`) for a client to use to verify a
+server, and certificates for a server to use in verifying client certificates
+(``purpose=``:data:`~ssl.Purpose.CLIENT_AUTH`). (Contributed by Christian
+Heimes in :issue:`19292`.)
+
+.. _whatsnew34-win-cert-store:
+
+Two new windows-only functions, :func:`~ssl.enum_certificates` and
+:func:`~ssl.enum_crls` provide the ability to retrieve certificates,
+certificate information, and CRLs from the Windows cert store. (Contributed
+by Christian Heimes in :issue:`17134`.)
+
+.. _whatsnew34-sni:
+
+Support for server-side SNI (Server Name Indication) using the new
+:meth:`ssl.SSLContext.set_servername_callback` method.
+(Contributed by Daniel Black in :issue:`8109`.)
+
+The dictionary returned by :meth:`.SSLSocket.getpeercert` contains additional
+``X509v3`` extension items: ``crlDistributionPoints``, ``calIssuers``, and
+``OCSP`` URIs. (Contributed by Christian Heimes in :issue:`18379`.)
+
+
+stat
+----
+
+The :mod:`stat` module is now backed by a C implementation in :mod:`_stat`. A C
+implementation is required as most of the values aren't standardized and
+are platform-dependent. (Contributed by Christian Heimes in :issue:`11016`.)
+
+The module supports new :mod:`~stat.ST_MODE` flags, :mod:`~stat.S_IFDOOR`,
+:attr:`~stat.S_IFPORT`, and :attr:`~stat.S_IFWHT`. (Contributed by
+Christian Hiemes in :issue:`11016`.)
+
+
+struct
+------
+
+New function :mod:`~struct.iter_unpack` and a new
+:meth:`struct.Struct.iter_unpack` method on compiled formats provide streamed
+unpacking of a buffer containing repeated instances of a given format of data.
+(Contributed by Antoine Pitrou in :issue:`17804`.)
+
+
+subprocess
+----------
+
+:func:`~subprocess.check_output` now accepts an *input* argument that can
+be used to provide the contents of ``stdin`` for the command that is run.
+(Contributed by Zack Weinberg in :issue:`16624`.)
+
+:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now
+work on Windows. This change was actually inadvertently made in 3.3.4.
+(Contributed by Tim Golden in :issue:`10197`.)
+
+
+sunau
+-----
+
+The :meth:`~sunau.getparams` method now returns a namedtuple rather than a
+plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.)
+
+:meth:`sunau.open` now supports the context manager protocol: when used in a
+:keyword:`with` block, the ``close`` method of the returned object will be
+called automatically at the end of the block. (Contributed by Serhiy Storchaka
+in :issue:`18878`.)
+
+:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding
+support for writing 24 sample using the module. (Contributed by
+Serhiy Storchaka in :issue:`19261`.)
+
+The :meth:`~sunau.AU_write.writeframesraw` and
+:meth:`~sunau.AU_write.writeframes` methods now accept any :term:`bytes-like
+object`. (Contributed by Serhiy Storchaka in :issue:`8311`.)
+
+
+sys
+---
+
+New function :func:`sys.getallocatedblocks` returns the current number of
+blocks allocated by the interpreter. (In CPython with the default
+``--with-pymalloc`` setting, this is allocations made through the
+:c:func:`PyObject_Malloc` API.) This can be useful for tracking memory leaks,
+especially if automated via a test suite. (Contributed by Antoine Pitrou
+in :issue:`13390`.)
+
+When the Python interpreter starts in :ref:`interactive mode
+<tut-interactive>`, it checks for an :data:`~sys.__interactivehook__` attribute
+on the :mod:`sys` module. If the attribute exists, its value is called with no
+arguments just before interactive mode is started. The check is made after the
+:envvar:`PYTHONSTARTUP` file is read, so it can be set there. The :mod:`site`
+module :ref:`sets it <rlcompleter-config>` to a function that enables tab
+completion and history saving (in :file:`~/.python-history`) if the platform
+supports :mod:`readline`. If you do not want this (new) behavior, you can
+override it in :envvar:`PYTHONSTARTUP`, :mod:`sitecustomize`, or
+:mod:`usercustomize` by deleting this attribute from :mod:`sys` (or setting it
+to some other callable). (Contributed by Éric Araujo and Antoine Pitrou in
+:issue:`5845`.)
+
+
+tarfile
+-------
+
+The :mod:`tarfile` module now supports a simple :ref:`tarfile-commandline` when
+called as a script directly or via ``-m``. This can be used to create and
+extract tarfile archives. (Contributed by Berker Peksag in :issue:`13477`.)
+
+
+textwrap
+--------
+
+The :class:`~textwrap.TextWrapper` class has two new attributes/constructor
+arguments: :attr:`~textwrap.TextWrapper.max_lines`, which limits the number of
+lines in the output, and :attr:`~textwrap.TextWrapper.placeholder`, which is a
+string that will appear at the end of the output if it has been truncated
+because of *max_lines*. Building on these capabilities, a new convenience
+function :func:`~textwrap.shorten` collapses all of the whitespace in the input
+to single spaces and produces a single line of a given *width* that ends with
+the *placeholder* (by default, ``[...]``). (Contributed by Antoine Pitrou and
+Serhiy Storchaka in :issue:`18585` and :issue:`18725`.)
+
+
+threading
+---------
+
+The :class:`~threading.Thread` object representing the main thread can be
+obtained from the new :func:`~threading.main_thread` function. In normal
+conditions this will be the thread from which the Python interpreter was
+started. (Contributed by Andrew Svetlov in :issue:`18882`.)
+
+
+traceback
+---------
+
+A new :func:`traceback.clear_frames` function takes a traceback object
+and clears the local variables in all of the frames it references,
+reducing the amount of memory consumed. (Contributed by Andrew Kuchling in
+:issue:`1565525`).
+
+
+types
+-----
+
+A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define
+an attribute that acts normally when looked up through an instance object, but
+which is routed to the *class* ``__getattr__`` when looked up through the
+class. This allows one to have properties active on a class, and have virtual
+attributes on the class with the same name (see :mod:`Enum` for an example).
+(Contributed by Ethan Furman in :issue:`19030`.)
+
+
+urllib
+------
+
+:mod:`urllib.request` now supports ``data:`` URLs via the
+:class:`~urllib.request.DataHandler` class. (Contributed by Mathias Panzenböck
+in :issue:`16423`.)
+
+The http method that will be used by a :class:`~urllib.request.Request` class
+can now be specified by setting a :class:`~urllib.request.Request.method`
+class attribute on the subclass. (Contributed by Jason R Coombs in
+:issue:`18978`.)
+
+:class:`~urllib.request.Request` objects are now reusable: if the
+:attr:`~urllib.request.Request.full_url` or :attr:`~urllib.request.Request.data`
+attributes are modified, all relevant internal properties are updated. This
+means, for example, that it is now possible to use the same
+:class:`~urllib.request.Request` object in more than one
+:meth:`.OpenerDirector.open` call with different *data* arguments, or to
+modify a :class:`~urllib.request.Request`\ 's ``url`` rather than recomputing it
+from scratch. There is also a new
+:meth:`~urllib.request.Request.remove_header` method that can be used to remove
+headers from a :class:`~urllib.request.Request`. (Contributed by Alexey
+Kachayev in :issue:`16464`, Daniel Wozniak in :issue:`17485`, and Damien Brecht
+and Senthil Kumaran in :issue:`17272`.)
+
+:class:`~urllib.error.HTTPError` objects now have a
+:attr:`~urllib.error.HTTPError.headers` attribute that provides access to the
+HTTP response headers associated with the error. (Contributed by
+Berker Peksag in :issue:`15701`.)
+
+
+unittest
+--------
+
+The :class:`~unittest.TestCase` class has a new method,
+:meth:`~unittest.TestCase.subTest`, that produces a context manager whose
+:keyword:`with` block becomes a "sub-test". This context manager allows a test
+method to dynamically generate subtests by, say, calling the ``subTest``
+context manager inside a loop. A single test method can thereby produce an
+indefinite number of separately-identified and separately-counted tests, all of
+which will run even if one or more of them fail. For example::
+
+ class NumbersTest(unittest.TestCase):
+ def test_even(self):
+ for i in range(6):
+ with self.subTest(i=i):
+ self.assertEqual(i % 2, 0)
+
+will result in six subtests, each identified in the unittest verbose output
+with a label consisting of the variable name ``i`` and a particular value for
+that variable (``i=0``, ``i=1``, etc). See :ref:`subtests` for the full
+version of this example. (Contributed by Antoine Pitrou in :issue:`16997`.)
+
+:func:`unittest.main` now accepts an iterable of test names for
+*defaultTest*, where previously it only accepted a single test name as a
+string. (Contributed by Jyrki Pulliainen in :issue:`15132`.)
+
+If :class:`~unittest.SkipTest` is raised during test discovery (that is, at the
+module level in the test file), it is now reported as a skip instead of an
+error. (Contributed by Zach Ware in :issue:`16935`.)
+
+:meth:`~unittest.TestLoader.discover` now sorts the discovered files to provide
+consistent test ordering. (Contributed by Martin Melin and Jeff Ramnani in
+:issue:`16709`.)
+
+:class:`~unittest.TestSuite` now drops references to tests as soon as the test
+has been run, if the test is successful. On Python interpreters that do
+garbage collection, this allows the tests to be garbage collected if nothing
+else is holding a reference to the test. It is possible to override this
+behavior by creating a :class:`~unittest.TestSuite` subclass that defines a
+custom ``_removeTestAtIndex`` method. (Contributed by Tom Wardill, Matt
+McClure, and Andrew Svetlov in :issue:`11798`.)
+
+A new test assertion context-manager, :meth:`~unittest.TestCase.assertLogs`,
+will ensure that a given block of code emits a log message using the
+:mod:`logging` module. By default the message can come from any logger and
+have a priority of ``INFO`` or higher, but both the logger name and an
+alternative minimum logging level may be specified. The object returned by the
+context manager can be queried for the :class:`~logging.LogRecord`\ s and/or
+formatted messages that were logged. (Contributed by Antoine Pitrou in
+:issue:`18937`.)
+
+Test discovery now works with namespace packages (Contributed by Claudiu Popa
+in :issue:`17457`.)
+
+:mod:`unittest.mock` objects now inspect their specification signatures when
+matching calls, which means an argument can now be matched by either position
+or name, instead of only by position. (Contributed by Antoine Pitrou in
+:issue:`17015`.)
+
+:func:`~mock.mock_open` objects now have ``readline`` and ``readlines``
+methods. (Contributed by Toshio Kuratomi in :issue:`17467`.)
+
+
+venv
+----
+
+:mod:`venv` now includes activation scripts for the ``csh`` and ``fish``
+shells (Contributed by Andrew Svetlov in :issue:`15417`.)
+
+:class:`~venv.EnvBuilder` and the :func:`~venv.create` convenience function
+take a new keyword argument *with_pip*, which defaults to ``False``, that
+controls whether or not :class:`~venv.EnvBuilder` ensures that ``pip`` is
+installed in the virtual environment. (Contributed by Nick Coghlan in
+:issue:`19552` as part of the :pep:`453` implementation.)
+
+
+wave
+----
+
+The :meth:`~wave.getparams` method now returns a namedtuple rather than a
+plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.)
+
+:meth:`wave.open` now supports the context manager protocol. (Contributed
+by Claudiu Popa in :issue:`17616`.)
+
+:mod:`wave` can now :ref:`write output to unseekable files
+<wave-write-objects>`. (Contributed by David Jones, Guilherme Polo, and Serhiy
+Storchaka in :issue:`5202`.)
+
+The :meth:`~wave.Wave_write.writeframesraw` and
+:meth:`~wave.Wave_write.writeframes` methods now accept any :term:`bytes-like
+object`. (Contributed by Serhiy Storchaka in :issue:`8311`.)
+
+
+weakref
+-------
+
+New :class:`~weakref.WeakMethod` class simulates weak references to bound
+methods. (Contributed by Antoine Pitrou in :issue:`14631`.)
+
+New :class:`~weakref.finalize` class makes it possible to register a callback
+to be invoked when an object is garbage collected, without needing to
+carefully manage the lifecycle of the weak reference itself. (Contributed by
+Richard Oudkerk in :issue:`15528`)
+
+The callback, if any, associated with a :class:`~weakref.ref` is now
+exposed via the :attr:`~weakref.ref.__callback__` attribute. (Contributed
+by Mark Dickinson in :issue:`17643`.)
+
+
+xml.etree
+---------
+
+A new parser, :class:`~xml.etree.ElementTree.XMLPullParser`, allows a
+non-blocking applications to parse XML documents. An example can be
+seen at :ref:`elementtree-pull-parsing`. (Contributed by Antoine
+Pitrou in :issue:`17741`.)
+
+The :mod:`xml.etree.ElementTree` :func:`~xml.etree.ElementTree.tostring` and
+:func:`~xml.etree.ElementTree.tostringlist` functions, and the
+:class:`~xml.etree.ElementTree.ElementTree`
+:meth:`~xml.etree.ElementTree.ElementTree.write` method, now have a
+*short_empty_elements* :ref:`keyword-only parameter <keyword-only_parameter>`
+providing control over whether elements with no content are written in
+abbreviated (``<tag />``) or expanded (``<tag></tag>``) form. (Contributed by
+Ariel Poliak and Serhiy Storchaka in :issue:`14377`.)
+
+
+zipfile
+-------
+
+The :meth:`~zipfile.PyZipFile.writepy` method of the
+:class:`~zipfile.PyZipFile` class has a new *filterfunc* option that can be
+used to control which directories and files are added to the archive. For
+example, this could be used to exclude test files from the archive.
+(Contributed by Christian Tismer in :issue:`19274`.)
+
+The *allowZip64* parameter to :class:`~zipfile.ZipFile` and
+:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by
+William Mallard in :issue:`17201`.)
+
+
+
+CPython Implementation Changes
+==============================
+
+
+.. _whatsnew-pep-445:
+
+PEP 445: Customization of CPython Memory Allocators
+---------------------------------------------------
+
+:pep:`445` adds new C level interfaces to customize memory allocation in
+the CPython interpreter.
+
+.. seealso::
+
+ :pep:`445` -- Add new APIs to customize Python memory allocators
+ PEP written and implemented by Victor Stinner.
+
+
+.. _whatsnew-pep-442:
+
+PEP 442: Safe Object Finalization
+---------------------------------
+
+:pep:`442` removes the current limitations and quirks of object finalization
+in CPython. With it, objects with :meth:`__del__` methods, as well as
+generators with :keyword:`finally` clauses, can be finalized when they are
+part of a reference cycle.
+
+As part of this change, module globals are no longer forcibly set to
+:const:`None` during interpreter shutdown in most cases, instead relying
+on the normal operation of the cyclic garbage collector. This avoids a
+whole class of interpreter-shutdown-time errors, usually involving
+``__del__`` methods, that have plagued Python since the cyclic GC
+was first introduced.
+
+.. seealso::
+
+ :pep:`442` -- Safe object finalization
+ PEP written and implemented by Antoine Pitrou.
+
+
+.. _whatsnew-pep-456:
+
+PEP 456: Secure and Interchangeable Hash Algorithm
+--------------------------------------------------
+
+:pep:`456` follows up on earlier security fix work done on Python's hash
+algorithm to address certain DOS attacks to which public facing APIs backed by
+dictionary lookups may be subject. (See :issue:`14621` for the start of the
+current round of improvements.) The PEP unifies CPython's hash code to make it
+easier for a packager to substitute a different hash algorithm, and switches
+Python's default implementation to a SipHash implementation on platforms that
+have a 64 bit data type. Any performance differences in comparison with the
+older FNV algorithm are trivial.
+
+The PEP adds additional fields to the :attr:`sys.hash_info` struct sequence to
+describe the hash algorithm in use by the currently executing binary. Otherwise,
+the PEP does not alter any existing CPython APIs.
+
+
+.. _whatsnew-pep-436:
+
+PEP 436: Argument Clinic
+------------------------
+
+"Argument Clinic" (:pep:`436`) is now part of the CPython build process
+and can be used to simplify the process of defining and maintaining
+accurate signatures for builtins and standard library extension modules
+implemented in C.
+
+Some standard library extension modules have been converted to use Argument
+Clinic in Python 3.4, and :mod:`pydoc` and :mod:`inspect` have been updated
+accordingly.
+
+It is expected that signature metadata for programmatic introspection will
+be added to additional callables implemented in C as part of Python 3.4
+maintenance releases.
+
+.. note::
+ The Argument Clinic PEP is not fully up to date with the state of the
+ implementation. This has been deemed acceptable by the release manager
+ and core development team in this case, as Argument Clinic will not
+ be made available as a public API for third party use in Python 3.4.
+
+.. seealso::
+
+ :pep:`436` -- The Argument Clinic DSL
+ PEP written and implemented by Larry Hastings.
+
+
+Other Build and C API Changes
+-----------------------------
+
+* The new :c:func:`PyType_GetSlot` function has been added to the stable ABI,
+ allowing retrieval of function pointers from named type slots when using
+ the limited API. (Contributed by Martin von Löwis in :issue:`17162`)
+
+* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API
+ allows applications embedding the CPython interpreter to reliably force
+ a particular encoding and error handler for the standard streams
+ (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`)
+
+* Most Python C APIs that don't mutate string arguments are now correctly
+ marked as accepting ``const char *`` rather than ``char *`` (Contributed
+ by Serhiy Storchaka in :issue:`1772673`).
+
+* A new shell version of ``python-config`` can be used even when a python
+ interpreter is not available (for example, in cross compilation scenarios).
+
+* :c:func:`PyUnicode_FromFormat` now supports width and precision
+ specifications for ``%s``, ``%A``, ``%U``, ``%V``, ``%S``, and ``%R``.
+ (Contributed by Ysj Ray and Victor Stinner in :issue:`7330`.)
+
+* New function :c:func:`PyStructSequence_InitType2` supplements the
+ existing :c:func:`PyStructSequence_InitType` function. The difference
+ is that it returns ``0`` on success and ``-1`` on failure.
+
+* The CPython source can now be compiled using the address sanity checking
+ features of recent versions of GCC and clang: the false alarms in the small
+ object allocator have been silenced. (Contributed by Dhiru Kholia in
+ :issue:`18596`.)
+
+* The Windows build now uses `Address Space Layout Randomization
+ <http://en.wikipedia.org/wiki/ASLR>`_ and `Data Execution Prevention
+ <http://en.wikipedia.org/wiki/Data_Execution_Prevention>`_. (Contributed by
+ Christian Heimes in :issue:`16632`.)
+
+* New function :c:func:`PyObject_LengthHint` is the C API equivalent
+ of :func:`operator.length_hint`. (Contributed by Armin Ronacher in
+ :issue:`16148`.)
+
+
+.. _other-improvements-3.4:
+
+Other Improvements
+------------------
+
+.. _whatsnew-isolated-mode:
+
+* The :ref:`python <using-on-cmdline>` command has a new :ref:`option
+ <using-on-misc-options>`, ``-I``, which causes it to run in "isolated mode",
+ which means that :data:`sys.path` contains neither the script's directory nor
+ the user's ``site-packages`` directory, and all :envvar:`PYTHON*` environment
+ variables are ignored (it implies both ``-s`` and ``-E``). Other
+ restrictions may also be applied in the future, with the goal being to
+ isolate the execution of a script from the user's environment. This is
+ appropriate, for example, when Python is used to run a system script. On
+ most POSIX systems it can and should be used in the ``#!`` line of system
+ scripts. (Contributed by Christian Heimes in :issue:`16499`.)
+
+* Tab-completion is now enabled by default in the interactive interpreter
+ on systems that support :mod:`readline`. History is also enabled by default,
+ and is written to (and read from) the file :file:`~/.python-history`.
+ (Contributed by Antoine Pitrou and Éric Araujo in :issue:`5845`.)
+
+* Invoking the Python interpreter with ``--version`` now outputs the version to
+ standard output instead of standard error (:issue:`18338`). Similar changes
+ were made to :mod:`argparse` (:issue:`18920`) and other modules that have
+ script-like invocation capabilities (:issue:`18922`).
+
+* The CPython Windows installer now adds ``.py`` to the :envvar:`PATHEXT`
+ variable when extensions are registered, allowing users to run a python
+ script at the windows command prompt by just typing its name without the
+ ``.py`` extension. (Contributed by Paul Moore in :issue:`18569`.)
+
+* A new ``make`` target `coverage-report
+ <http://docs.python.org/devguide/coverage.html#measuring-coverage-of-c-code-with-gcov-and-lcov>`_
+ will build python, run the test suite, and generate an HTML coverage report
+ for the C codebase using ``gcov`` and `lcov
+ <http://ltp.sourceforge.net/coverage/lcov.php>`_.
+
+* The ``-R`` option to the :ref:`python regression test suite <regrtest>` now
+ also checks for memory allocation leaks, using
+ :func:`sys.getallocatedblocks()`. (Contributed by Antoine Pitrou in
+ :issue:`13390`).
+
+* ``python -m`` now works with namespace packages.
+
+* The :mod:`stat` module is now implemented in C, which means it gets the
+ values for its constants from the C header files, instead of having the
+ values hard-coded in the python module as was previously the case.
+
+* Loading multiple python modules from a single OS module (``.so``, ``.dll``)
+ now works correctly (previously it silently returned the first python
+ module in the file). (Contributed by Václav Šmilauer in :issue:`16421`.)
+
+* A new opcode, :opcode:`LOAD_CLASSDEREF`, has been added to fix a bug in the
+ loading of free variables in class bodies that could be triggered by certain
+ uses of :ref:`__prepare__ <prepare>`. (Contributed by Benjamin Peterson in
+ :issue:`17853`.)
+
+* A number of MemoryError-related crashes were identified and fixed by Victor
+ Stinner using his :pep:`445`-based ``pyfailmalloc`` tool (:issue:`18408`,
+ :issue:`18520`).
+
+* The :ref:`pyvenv <scripts-pyvenv>` command now accepts a ``--copies`` option
+ to use copies rather than symlinks even on systems where symlinks are the
+ default. (Contributed by Vinay Sajip in :issue:`18807`.)
+
+* The :ref:`pyvenv <scripts-pyvenv>` command also accepts a ``--without-pip``
+ option to suppress the otherwise-automatic bootstrapping of pip into
+ the virtual environment. (Contributed by Nick Coghlan in :issue:`19552`
+ as part of the :pep:`453` implementation.)
+
+* The encoding name is now optional in the value set for the
+ :envvar:`PYTHONIOENCODING` environment variable. This makes it possible to
+ set just the error handler, without changing the default encoding.
+ (Contributed by Serhiy Storchaka in :issue:`18818`.)
+
+* The :mod:`bz2`, :mod:`lzma`, and :mod:`gzip` module ``open`` functions now
+ support ``x`` (exclusive creation) mode. (Contributed by Tim Heaney and
+ Vajrasky Kok in :issue:`19201`, :issue:`19222`, and :issue:`19223`.)
+
+
+Significant Optimizations
+-------------------------
+
+* The UTF-32 decoder is now 3x to 4x faster. (Contributed by Serhiy Storchaka
+ in :issue:`14625`.)
+
+* The cost of hash collisions for sets is now reduced. Each hash table
+ probe now checks a series of consecutive, adjacent key/hash pairs before
+ continuing to make random probes through the hash table. This exploits
+ cache locality to make collision resolution less expensive.
+ The collision resolution scheme can be described as a hybrid of linear
+ probing and open addressing. The number of additional linear probes
+ defaults to nine. This can be changed at compile-time by defining
+ LINEAR_PROBES to be any value. Set LINEAR_PROBES=0 to turn-off
+ linear probing entirely. (Contributed by Raymond Hettinger in
+ :issue:`18771`.)
+
+* The interpreter starts about 30% faster. A couple of measures lead to the
+ speedup. The interpreter loads fewer modules on startup, e.g. the :mod:`re`,
+ :mod:`collections` and :mod:`locale` modules and their dependencies are no
+ longer imported by default. The marshal module has been improved to load
+ compiled Python code faster. (Contributed by Antoine Pitrou, Christian
+ Heimes and Victor Stinner in :issue:`19219`, :issue:`19218`, :issue:`19209`,
+ :issue:`19205` and :issue:`9548`)
+
+* :class:`bz2.BZ2File` is now as fast or faster than the Python2 version for
+ most cases. :class:`lzma.LZMAFile` has also been optimized. (Contributed by
+ Serhiy Storchaka and Nadeem Vawda in :issue:`16034`.)
+
+* :func:`random.getrandbits` is 20%-40% faster for small integers (the most
+ common use case). (Contributed by Serhiy Storchaka in :issue:`16674`).
+
+* By taking advantage of the new storage format for strings, pickling of
+ strings is now significantly faster. (Contributed by Victor Stinner and
+ Antoine Pitrou in :issue:`15596`.)
+
+* A performance issue in :meth:`io.FileIO.readall` has been solved. This
+ particularly affects Windows, and significantly speeds up the case of piping
+ significant amounts of data through :mod:`subprocess`. (Contributed
+ by Richard Oudkerk in :issue:`15758`.)
+
+* :func:`html.escape` is now 10x faster. (Contributed by Matt Bryant in
+ :issue:`18020`.)
+
+* On Windows, the native ``VirtualAlloc`` is now used instead of the CRT
+ ``malloc`` in ``obmalloc``. Artificial benchmarks show about a 3% memory
+ savings.
+
+* :func:`os.urandom` now uses a lazily-opened persistent file descriptor
+ so as to avoid using many file descriptors when run in parallel from
+ multiple threads. (Contributed by Antoine Pitrou in :issue:`18756`.)
+
+
+.. _deprecated-3.4:
+
+Deprecated
+==========
+
+This section covers various APIs and other features that have been deprecated
+in Python 3.4, and will be removed in Python 3.5 or later. In most (but not
+all) cases, using the deprecated APIs will produce a :exc:`DeprecationWarning`
+when the interpreter is run with deprecation warnings enabled (for example, by
+using ``-Wd``).
+
+
+Deprecations in the Python API
+------------------------------
+
+* As mentioned in :ref:`whatsnew-pep-451`, a number of :mod:`importilb`
+ methods and functions are deprecated: :meth:`importlib.find_loader` is
+ replaced by :func:`importlib.util.find_spec`;
+ :meth:`importlib.machinery.PathFinder.find_module` is replaced by
+ :meth:`importlib.machinery.PathFinder.find_spec`;
+ :meth:`importlib.abc.MetaPathFinder.find_module` is replaced by
+ :meth:`importlib.abc.MetaPathFinder.find_spec`;
+ :meth:`importlib.abc.PathEntryFinder.find_loader` and
+ :meth:`~importlib.abc.PathEntryFinder.find_module` are replaced by
+ :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC
+ ``load_module`` methods (:meth:`importlib.abc.Loader.load_module`,
+ :meth:`importlib.abc.InspectLoader.load_module`,
+ :meth:`importlib.abc.FileLoader.load_module`,
+ :meth:`importlib.abc.SourceLoader.load_module`) should no longer be
+ implemented, instead loaders should implement an
+ ``exec_module`` method
+ (:meth:`importlib.abc.Loader.exec_module`,
+ :meth:`importlib.abc.InspectLoader.exec_module`
+ :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system
+ take care of the rest; and
+ :meth:`importlib.abc.Loader.module_repr`,
+ :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`,
+ and :meth:`importlib.util.set_package` are no longer needed because their
+ functions are now handled automatically by the import system.
+
+* The :mod:`imp` module is pending deprecation. To keep compatibility with
+ Python 2/3 code bases, the module's removal is currently not scheduled.
+
+* The :mod:`formatter` module is pending deprecation and is slated for removal
+ in Python 3.6.
+
+* ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is
+ deprecated. Python 3.6 will require an explicit digest name or constructor as
+ *digestmod* argument.
+
+* The internal ``Netrc`` class in the :mod:`ftplib` module has been documented
+ as deprecated in its docstring for quite some time. It now emits a
+ :exc:`DeprecationWarning` and will be removed completely in Python 3.5.
+
+* The undocumented *endtime* argument to :meth:`subprocess.Popen.wait` should
+ not have been exposed and is hopefully not in use; it is deprecated and
+ will mostly likely be removed in Python 3.5.
+
+* The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated.
+
+* The :mod:`plistlib` :func:`~plistlib.readPlist`,
+ :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and
+ :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the
+ corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`,
+ :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data`
+ is deprecated in favor of just using the :class:`bytes` constructor.
+
+* The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by
+ ``EXT_SUFFIX``.
+
+* The ``U`` mode accepted by various ``open`` functions is deprecated.
+ In Python3 it does not do anything useful, and should be replaced by
+ appropriate uses of :class:`io.TextIOWrapper` (if needed) and its *newline*
+ argument.
+
+* The *parser* argument of :func:`xml.etree.ElementTree.iterparse` has
+ been deprecated, as has the *html* argument of
+ :func:`~xml.etree.ElementTree.XMLParser`. To prepare for the removal of the
+ latter, all arguments to ``XMLParser`` should be passed by keyword.
+
+
+Deprecated Features
+-------------------
+
+* Running :ref:`idle` with the ``-n`` flag (no subprocess) is deprecated.
+ However, the feature will not be removed until :issue:`18823` is resolved.
+
+* The site module adding a "site-python" directory to sys.path, if it
+ exists, is deprecated (:issue:`19375`).
+
+
+
+Removed
+=======
+
+
+Operating Systems No Longer Supported
+-------------------------------------
+
+Support for the following operating systems has been removed from the source
+and build tools:
+
+* OS/2 (:issue:`16135`).
+* Windows 2000 (changeset e52df05b496a).
+* Windows systems where ``COMSPEC`` points to ``command.com`` (:issue:`14470`).
+* VMS (:issue:`16136`).
+
+
+API and Feature Removals
+------------------------
+
+The following obsolete and previously deprecated APIs and features have been
+removed:
+
+* The unmaintained ``Misc/TextMate`` and ``Misc/vim`` directories have been
+ removed (see the `devguide <http://docs.python.org/devguide>`_
+ for suggestions on what to use instead).
+
+* The ``SO`` makefile macro is removed (it was replaced by the
+ ``SHLIB_SUFFIX`` and ``EXT_SUFFIX`` macros) (:issue:`16754`).
+
+* The ``PyThreadState.tick_counter`` field has been removed; its value has
+ been meaningless since Python 3.2, when the "new GIL" was introduced
+ (:issue:`19199`).
+
+* ``PyLoader`` and ``PyPycLoader`` have been removed from :mod:`importlib`.
+ (Contributed by Taras Lyapun in :issue:`15641`.)
+
+* The *strict* argument to :class:`~http.client.HTTPConnection` and
+ :class:`~http.client.HTTPSConnection` has been removed. HTTP 0.9-style
+ "Simple Responses" are no longer supported.
+
+* The deprecated :mod:`urllib.request.Request` getter and setter methods
+ ``add_data``, ``has_data``, ``get_data``, ``get_type``, ``get_host``,
+ ``get_selector``, ``set_proxy``, ``get_origin_req_host``, and
+ ``is_unverifiable`` have been removed (use direct attribute access instead).
+
+* Support for loading the deprecated ``TYPE_INT64`` has been removed from
+ :mod:`marshal`. (Contributed by Dan Riti in :issue:`15480`.)
+
+* :class:`inspect.Signature`: positional-only parameters are now required
+ to have a valid name.
+
+* :meth:`object.__format__` no longer accepts non-empty format strings, it now
+ raises a :exc:`TypeError` instead. Using a non-empty string has been
+ deprecated since Python 3.2. This change has been made to prevent a
+ situation where previously working (but incorrect) code would start failing
+ if an object gained a __format__ method, which means that your code may now
+ raise a :exc:`TypeError` if you are using an ``'s'`` format code with objects
+ that do not have a __format__ method that handles it. See :issue:`7994` for
+ background.
+
+* :meth:`difflib.SequenceMatcher.isbjunk` and
+ :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have
+ now been removed: use ``x in sm.bjunk`` and
+ ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object
+ (:issue:`13248`).
+
+
+Code Cleanups
+-------------
+
+* The unused and undocumented internal ``Scanner`` class has been removed from
+ the :mod:`pydoc` module.
+
+* The private and effectively unused ``_gestalt`` module has been removed,
+ along with the private :mod:`platform` functions ``_mac_ver_lookup``,
+ ``_mac_ver_gstalt``, and ``_bcd2str``, which would only have ever been called
+ on badly broken OSX systems (see :issue:`18393`).
+
+* The hardcoded copies of certain :mod:`stat` constants that were included in
+ the :mod:`tarfile` module namespace have been removed.
+
+
+
+Porting to Python 3.4
+=====================
+
+This section lists previously described changes and other bugfixes
+that may require changes to your code.
+
+
+Changes in 'python' Command Behavior
+------------------------------------
+
+* In a posix shell, setting the :envvar:`PATH` environment variable to
+ an empty value is equivalent to not setting it at all. However, setting
+ :envvar:`PYTHONPATH` to an empty value was *not* equivalent to not setting it
+ at all: setting :envvar:`PYTHONPATH` to an empty value was equivalent to
+ setting it to ``.``, which leads to confusion when reasoning by analogy to
+ how :envvar:`PATH` works. The behavior now conforms to the posix convention
+ for :envvar:`PATH`.
+
+* The [X refs, Y blocks] output of a debug (``--with-pydebug``) build of the
+ CPython interpreter is now off by default. It can be re-enabled using the
+ ``-X showrefcount`` option. (Contributed by Ezio Melotti in :issue:`17323`.)
+
+* The python command and most stdlib scripts (as well as :mod:`argparse`) now
+ output ``--version`` information to ``stdout`` instead of ``stderr`` (for
+ issue list see :ref:`other-improvements-3.4` above).
+
+
+Changes in the Python API
+-------------------------
+
+* The ABCs defined in :mod:`importlib.abc` now either raise the appropriate
+ exception or return a default value instead of raising
+ :exc:`NotImplementedError` blindly. This will only affect code calling
+ :func:`super` and falling through all the way to the ABCs. For compatibility,
+ catch both :exc:`NotImplementedError` or the appropriate exception as needed.
+
+* The module type now initializes the :attr:`__package__` and :attr:`__loader__`
+ attributes to ``None`` by default. To determine if these attributes were set
+ in a backwards-compatible fashion, use e.g.
+ ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.)
+
+* :meth:`importlib.util.module_for_loader` now sets ``__loader__`` and
+ ``__package__`` unconditionally to properly support reloading. If this is not
+ desired then you will need to set these attributes manually. You can use
+ :func:`importlib.util.module_to_load` for module management.
+
+* Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``,
+ ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading.
+ Note that this restores a pre-3.3 behavior in that it means a module is
+ re-found when re-loaded (:issue:`19413`).
+
+* Frozen packages no longer set ``__path__`` to a list containing the package
+ name, they now set it to an empty list. The previous behavior could cause
+ the import system to do the wrong thing on submodule imports if there was
+ also a directory with the same name as the frozen package. The correct way
+ to determine if a module is a package or not is to use ``hasattr(module,
+ '__path__')`` (:issue:`18065`).
+
+* Frozen modules no longer define a ``__file__`` attribute. It's semantically
+ incorrect for frozen modules to set the attribute as they are not loaded from
+ any explicit location. If you must know that a module comes from frozen code
+ then you can see if the module's ``__spec__.location`` is set to ``'frozen'``,
+ check if the loader is a subclass of
+ :class:`importlib.machinery.FrozenImporter`,
+ or if Python 2 compatibility is necessary you can use :func:`imp.is_frozen`.
+
+* :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path
+ it would write to is a symlink or a non-regular file. This is to act as a
+ warning that import will overwrite those files with a regular file regardless
+ of what type of file path they were originally.
+
+* :meth:`importlib.abc.SourceLoader.get_source` no longer raises
+ :exc:`ImportError` when the source code being loaded triggers a
+ :exc:`SyntaxError` or :exc:`UnicodeDecodeError`. As :exc:`ImportError` is
+ meant to be raised only when source code cannot be found but it should, it was
+ felt to be over-reaching/overloading of that meaning when the source code is
+ found but improperly structured. If you were catching ImportError before and
+ wish to continue to ignore syntax or decoding issues, catch all three
+ exceptions now.
+
+* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly
+ set the ``__wrapped__`` attribute to the function being wrapped, even if
+ that function also had its ``__wrapped__`` attribute set. This means
+ ``__wrapped__`` attributes now correctly link a stack of decorated
+ functions rather than every ``__wrapped__`` attribute in the chain
+ referring to the innermost function. Introspection libraries that
+ assumed the previous behaviour was intentional can use
+ :func:`inspect.unwrap` to access the first function in the chain that has
+ no ``__wrapped__`` attribute.
+
+* :func:`inspect.getfullargspec` has been reimplemented on top of
+ :func:`inspect.signature` and hence handles a much wider variety of callable
+ objects than it did in the past. It is expected that additional builtin and
+ extension module callables will gain signature metadata over the course of
+ the Python 3.4 series. Code that assumes that
+ :func:`inspect.getfullargspec` will fail on non-Python callables may need
+ to be adjusted accordingly.
+
+* :class:`importlib.machinery.PathFinder` now passes on the current working
+ directory to objects in :data:`sys.path_hooks` for the empty string. This
+ results in :data:`sys.path_importer_cache` never containing ``''``, thus
+ iterating through :data:`sys.path_importer_cache` based on :data:`sys.path`
+ will not find all keys. A module's ``__file__`` when imported in the current
+ working directory will also now have an absolute path, including when using
+ ``-m`` with the interpreter (except for ``__main__.__file__`` when a script
+ has been executed directly using a relative path) (Contributed by Brett
+ Cannon in :issue:`18416`). is specified on the command-line)
+ (:issue:`18416`).
+
+* The removal of the *strict* argument to :class:`~http.client.HTTPConnection`
+ and :class:`~http.client.HTTPSConnection` changes the meaning of the
+ remaining arguments if you are specifying them positionally rather than by
+ keyword. If you've been paying attention to deprecation warnings your code
+ should already be specifying any additional arguments via keywords.
+
+* Strings between ``from __future__ import ...`` statements now *always* raise
+ a :exc:`SyntaxError`. Previously if there was no leading docstring, an
+ interstitial string would sometimes be ignored. This brings CPython into
+ compliance with the language spec; Jython and PyPy already were.
+ (:issue:`17434`).
+
+* :meth:`ssl.SSLSocket.getpeercert` and :meth:`ssl.SSLSocket.do_handshake`
+ now raise an :exc:`OSError` with ``ENOTCONN`` when the ``SSLSocket`` is not
+ connected, instead of the previous behavior of raising an
+ :exc:`AttributeError`. In addition, :meth:`~ssl.SSLSocket.getpeercert`
+ will raise a :exc:`ValueError` if the handshake has not yet been done.
+
+* :func:`base64.b32decode` now raises a :exc:`binascii.Error` when the
+ input string contains non-b32-alphabet characters, instead of a
+ :exc:`TypeError`. This particular :exc:`TypeError` was missed when the other
+ :exc:`TypeError`\ s were converted. (Contributed by Serhiy Storchaka in
+ :issue:`18011`.) Note: this change was also inadvertently applied in Python
+ 3.3.3.
+
+* The :attr:`~cgi.FieldStorage.file` attribute is now automatically closed when
+ the creating :class:`cgi.FieldStorage` instance is garbage collected. If you
+ were pulling the file object out separately from the :class:`cgi.FieldStorage`
+ instance and not keeping the instance alive, then you should either store the
+ entire :class:`cgi.FieldStorage` instance or read the contents of the file
+ before the :class:`cgi.FieldStorage` instance is garbage collected.
+
+* Calling ``read`` or ``write`` on a closed SSL socket now raises an
+ informative :exc:`ValueError` rather than the previous more mysterious
+ :exc:`AttributeError` (:issue:`9177`).
+
+* :meth:`slice.indices` no longer produces an :exc:`OverflowError` for huge
+ values. As a consequence of this fix, :meth:`slice.indices` now raises a
+ :exc:`ValueError` if given a negative length; previously it returned nonsense
+ values (:issue:`14794`).
+
+* The :class:`complex` constructor, unlike the :mod:`cmath` functions, was
+ incorrectly accepting :class:`float` values if an object's ``__complex__``
+ special method returned one. This now raises a :exc:`TypeError`.
+ (:issue:`16290`.)
+
+* The :class:`int` constructor in 3.2 and 3.3 erroneously accepts :class:`float`
+ values for the *base* parameter. It is unlikely anyone was doing this, but
+ if so, it will now raise a :exc:`TypeError` (:issue:`16772`).
+
+* Defaults for keyword-only arguments are now evaluated *after* defaults for
+ regular keyword arguments, instead of before. Hopefully no one wrote any
+ code that depends on the previous buggy behavior (:issue:`16967`).
+
+* Stale thread states are now cleared after :func:`~os.fork`. This may cause
+ some system resources to be released that previously were incorrectly kept
+ perpetually alive (for example, database connections kept in thread-local
+ storage). (:issue:`17094`.)
+
+* Parameter names in ``__annotations__`` dicts are now mangled properly,
+ similarly to ``__kwdefaults__``. (Contributed by Yury Selivanov in
+ :issue:`20625`).
+
+* :attr:`hashlib.hash.name` now always returns the identifier in lower case.
+ Previously some builtin hashes had uppercase names, but now that it is a
+ formal public interface the naming has been made consistent (:issue:`18532`).
+
+* Because :mod:`unittest.TestSuite` now drops references to tests after they
+ are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run
+ a set of tests may fail. Test suites should not be re-used in this fashion
+ since it means state is retained between test runs, breaking the test
+ isolation that :mod:`unittest` is designed to provide. However, if the lack
+ of isolation is considered acceptable, the old behavior can be restored by
+ creating a :mod:`~unittest.TestSuite` subclass that defines a
+ ``_removeTestAtIndex`` method that does nothing (see
+ :meth:`.TestSuite.__iter__`) (:issue:`11798`).
+
+* :mod:`unittest` now uses :mod:`argparse` for command line parsing. There are
+ certain invalid command forms that used to work that are no longer allowed;
+ in theory this should not cause backward compatibility issues since the
+ disallowed command forms didn't make any sense and are unlikely to be in use.
+
+* The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and
+ the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of
+ ``match`` objects now always return a *bytes* object when the string
+ to be matched is a :term:`bytes-like object`. Previously the return type
+ matched the input type, so if your code was depending on the return value
+ being, say, a ``bytearray``, you will need to change your code.
+
+* :mod:`audioop` functions now raise an error immediately if passed string
+ input, instead of failing randomly later on (:issue:`16685`).
+
+* The new *convert_charrefs* argument to :class:`~html.parser.HTMLParser`
+ currently defaults to ``False`` for backward compatibility, but will
+ eventually be changed to default to ``True``. It is recommended that you add
+ this keyword, with the appropriate value, to any
+ :class:`~html.parser.HTMLParser` calls in your code (:issue:`13633`).
+
+* Since the *digestmod* argument to the :func:`hmac.new` function will in the
+ future have no default, all calls to :func:`hmac.new` should be changed to
+ explicitly specify a *digestmod* (:issue:`17276`).
+
+* Calling :func:`sysconfig.get_config_var` with the ``SO`` key, or looking
+ ``SO`` up in the results of a call to :func:`sysconfig.get_config_vars`
+ is deprecated. This key should be replaced by ``EXT_SUFFIX`` or
+ ``SHLIB_SUFFIX``, depending on the context (:issue:`19555`).
+
+* Any calls to ``open`` functions that specify ``U`` should be modified.
+ ``U`` is ineffective in Python3 and will eventually raise an error if used.
+ Depending on the function, the equivalent of its old Python2 behavior can be
+ achieved using either a *newline* argument, or if necessary by wrapping the
+ stream in :mod:`~io.TextIOWrapper` to use its *newline* argument
+ (:issue:`15204`).
+
+* If you use :ref:`pyvenv <scripts-pyvenv>` in a script and desire that pip
+ *not* be installed, you must add ``--without-pip`` to your command
+ invocation.
+
+* The default behavior of :func:`json.dump` and :func:`json.dumps` when
+ an indent is specified has changed: it no longer produces trailing
+ spaces after the item separating commas at the ends of lines. This
+ will matter only if you have tests that are doing white-space-sensitive
+ comparisons of such output (:issue:`16333`).
+
+* :mod:`doctest` now looks for doctests in extension module ``__doc__``
+ strings, so if your doctest test discovery includes extension modules that
+ have things that look like doctests in them you may see test failures you've
+ never seen before when running your tests (:issue:`3158`).
+
+* The :mod:`collections.abc` module has been slightly refactored as
+ part of the Python startup improvements. As a consequence of this, it is no
+ longer the case that importing :mod:`collections` automatically imports
+ :mod:`collections.abc`. If your program depended on the (undocumented)
+ implicit import, you will need to add an explicit ``import collections.abc``
+ (:issue:`20784`).
+
+
+Changes in the C API
+--------------------
+
+* :c:func:`PyEval_EvalFrameEx`, :c:func:`PyObject_Repr`, and
+ :c:func:`PyObject_Str`, along with some other internal C APIs, now include
+ a debugging assertion that ensures they are not used in situations where
+ they may silently discard a currently active exception. In cases where
+ discarding the active exception is expected and desired (for example,
+ because it has already been saved locally with :c:func:`PyErr_Fetch` or
+ is being deliberately replaced with a different exception), an explicit
+ :c:func:`PyErr_Clear` call will be needed to avoid triggering the
+ assertion when invoking these operations (directly or indirectly) and
+ running against a version of Python that is compiled with assertions
+ enabled.
+
+* :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg**
+ argument is not set. Previously only ``NULL`` was returned with no exception
+ set.
+
+* The result of the :c:data:`PyOS_ReadlineFunctionPointer` callback must
+ now be a string allocated by :c:func:`PyMem_RawMalloc` or
+ :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred, instead of a
+ string allocated by :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`
+ (:issue:`16742`)
+
+* :c:func:`PyThread_set_key_value` now always set the value. In Python
+ 3.3, the function did nothing if the key already exists (if the current
+ value is a non-NULL pointer).
+
+* The ``f_tstate`` (thread state) field of the :c:type:`PyFrameObject`
+ structure has been removed to fix a bug: see :issue:`14432` for the
+ rationale.
diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst
index bc1206b..29902e4 100644
--- a/Doc/whatsnew/index.rst
+++ b/Doc/whatsnew/index.rst
@@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release.
.. toctree::
:maxdepth: 2
+ 3.4.rst
3.3.rst
3.2.rst
3.1.rst
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 00e92d0..67d677b 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -182,8 +182,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
- Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
- Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
+ NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
+ Subscript_kind=22, Starred_kind=23, Name_kind=24,
+ List_kind=25, Tuple_kind=26};
struct _expr {
enum _expr_kind kind;
union {
@@ -279,6 +280,10 @@ struct _expr {
} Bytes;
struct {
+ singleton value;
+ } NameConstant;
+
+ struct {
expr_ty value;
identifier attr;
expr_context_ty ctx;
@@ -359,18 +364,18 @@ struct _excepthandler {
struct _arguments {
asdl_seq *args;
- identifier vararg;
- expr_ty varargannotation;
+ arg_ty vararg;
asdl_seq *kwonlyargs;
- identifier kwarg;
- expr_ty kwargannotation;
- asdl_seq *defaults;
asdl_seq *kw_defaults;
+ arg_ty kwarg;
+ asdl_seq *defaults;
};
struct _arg {
identifier arg;
expr_ty annotation;
+ int lineno;
+ int col_offset;
};
struct _keyword {
@@ -509,6 +514,9 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
#define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)
+expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
+ *arena);
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
@@ -542,11 +550,10 @@ comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_seq *
excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
body, int lineno, int col_offset, PyArena
*arena);
-#define arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_arguments(a0, a1, a2, a3, a4, a5, a6, a7, a8)
-arguments_ty _Py_arguments(asdl_seq * args, identifier vararg, expr_ty
- varargannotation, asdl_seq * kwonlyargs, identifier
- kwarg, expr_ty kwargannotation, asdl_seq * defaults,
- asdl_seq * kw_defaults, PyArena *arena);
+#define arguments(a0, a1, a2, a3, a4, a5, a6) _Py_arguments(a0, a1, a2, a3, a4, a5, a6)
+arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
+ kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
+ asdl_seq * defaults, PyArena *arena);
#define arg(a0, a1, a2) _Py_arg(a0, a1, a2)
arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
#define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
diff --git a/Include/Python.h b/Include/Python.h
index a78a721..2dd8290 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -68,6 +68,7 @@
#include "object.h"
#include "objimpl.h"
#include "typeslots.h"
+#include "pyhash.h"
#include "pydebug.h"
diff --git a/Include/abstract.h b/Include/abstract.h
index c9624f3..4bc9952a 100644
--- a/Include/abstract.h
+++ b/Include/abstract.h
@@ -144,7 +144,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
/* Implemented elsewhere:
- int PyObject_HasAttrString(PyObject *o, char *attr_name);
+ int PyObject_HasAttrString(PyObject *o, const char *attr_name);
Returns 1 if o has the attribute attr_name, and 0 otherwise.
This is equivalent to the Python expression:
@@ -156,7 +156,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
/* Implemented elsewhere:
- PyObject* PyObject_GetAttrString(PyObject *o, char *attr_name);
+ PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name);
Retrieve an attributed named attr_name form object o.
Returns the attribute value on success, or NULL on failure.
@@ -189,7 +189,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
/* Implemented elsewhere:
- int PyObject_SetAttrString(PyObject *o, char *attr_name, PyObject *v);
+ int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v);
Set the value of the attribute named attr_name, for object o,
to the value, v. Returns -1 on failure. This is
@@ -209,7 +209,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
/* implemented as a macro:
- int PyObject_DelAttrString(PyObject *o, char *attr_name);
+ int PyObject_DelAttrString(PyObject *o, const char *attr_name);
Delete attribute named attr_name, for object o. Returns
-1 on failure. This is the equivalent of the Python
@@ -284,7 +284,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
*/
PyAPI_FUNC(PyObject *) PyObject_CallFunction(PyObject *callable_object,
- char *format, ...);
+ const char *format, ...);
/*
Call a callable Python object, callable_object, with a
@@ -296,8 +296,9 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
*/
- PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *o, char *method,
- char *format, ...);
+ PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *o,
+ const char *method,
+ const char *format, ...);
/*
Call the method named m of object o with a variable number of
@@ -308,8 +309,9 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
Python expression: o.method(args).
*/
- PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o, _Py_Identifier *method,
- char *format, ...);
+ PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o,
+ _Py_Identifier *method,
+ const char *format, ...);
/*
Like PyObject_CallMethod, but expect a _Py_Identifier* as the
@@ -317,13 +319,16 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
*/
PyAPI_FUNC(PyObject *) _PyObject_CallFunction_SizeT(PyObject *callable,
- char *format, ...);
+ const char *format,
+ ...);
PyAPI_FUNC(PyObject *) _PyObject_CallMethod_SizeT(PyObject *o,
- char *name,
- char *format, ...);
+ const char *name,
+ const char *format,
+ ...);
PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *o,
_Py_Identifier *name,
- char *format, ...);
+ const char *format,
+ ...);
PyAPI_FUNC(PyObject *) PyObject_CallFunctionObjArgs(PyObject *callable,
...);
@@ -339,11 +344,10 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(PyObject *o,
PyObject *method, ...);
- PyAPI_FUNC(PyObject *) _PyObject_CallMethodObjIdArgs(PyObject *o,
+ PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs(PyObject *o,
struct _Py_Identifier *method,
...);
-
/*
Call the method named m of object o with a variable number of
C arguments. The C arguments are provided as PyObject *
@@ -404,7 +408,8 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
#define PyObject_Length PyObject_Size
#ifndef Py_LIMITED_API
- PyAPI_FUNC(Py_ssize_t) _PyObject_LengthHint(PyObject *o, Py_ssize_t);
+ PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o);
+ PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t);
#endif
/*
@@ -429,7 +434,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
statement: o[key]=v.
*/
- PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, char *key);
+ PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, const char *key);
/*
Remove the mapping for object, key, from the object *o.
@@ -1151,7 +1156,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
/* implemented as a macro:
- int PyMapping_DelItemString(PyObject *o, char *key);
+ int PyMapping_DelItemString(PyObject *o, const char *key);
Remove the mapping for object, key, from the object *o.
Returns -1 on failure. This is equivalent to
@@ -1169,7 +1174,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
*/
#define PyMapping_DelItem(O,K) PyObject_DelItem((O),(K))
- PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, char *key);
+ PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, const char *key);
/*
On success, return 1 if the mapping object has the key, key,
@@ -1213,7 +1218,8 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
*/
- PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o, char *key);
+ PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o,
+ const char *key);
/*
Return element of o corresponding to the object, key, or NULL
@@ -1221,7 +1227,7 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
o[key].
*/
- PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, char *key,
+ PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, const char *key,
PyObject *value);
/*
diff --git a/Include/asdl.h b/Include/asdl.h
index 6bf618f..495153c 100644
--- a/Include/asdl.h
+++ b/Include/asdl.h
@@ -5,6 +5,7 @@ typedef PyObject * identifier;
typedef PyObject * string;
typedef PyObject * bytes;
typedef PyObject * object;
+typedef PyObject * singleton;
/* It would be nice if the code generated by asdl_c.py was completely
independent of Python, but it is a goal the requires too much work
@@ -24,17 +25,19 @@ typedef struct {
int elements[1];
} asdl_int_seq;
-asdl_seq *asdl_seq_new(Py_ssize_t size, PyArena *arena);
-asdl_int_seq *asdl_int_seq_new(Py_ssize_t size, PyArena *arena);
+asdl_seq *_Py_asdl_seq_new(Py_ssize_t size, PyArena *arena);
+asdl_int_seq *_Py_asdl_int_seq_new(Py_ssize_t size, PyArena *arena);
#define asdl_seq_GET(S, I) (S)->elements[(I)]
#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
#ifdef Py_DEBUG
-#define asdl_seq_SET(S, I, V) { \
- int _asdl_i = (I); \
- assert((S) && _asdl_i < (S)->size); \
+#define asdl_seq_SET(S, I, V) \
+ do { \
+ Py_ssize_t _asdl_i = (I); \
+ assert((S) != NULL); \
+ assert(_asdl_i < (S)->size); \
(S)->elements[_asdl_i] = (V); \
-}
+ } while (0)
#else
#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
#endif
diff --git a/Include/ast.h b/Include/ast.h
index 055e8dc..6a8c816 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -10,6 +10,11 @@ PyAPI_FUNC(mod_ty) PyAST_FromNode(
PyCompilerFlags *flags,
const char *filename, /* decoded from the filesystem encoding */
PyArena *arena);
+PyAPI_FUNC(mod_ty) PyAST_FromNodeObject(
+ const node *n,
+ PyCompilerFlags *flags,
+ PyObject *filename,
+ PyArena *arena);
#ifdef __cplusplus
}
diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h
index eccd44c..a757b88 100644
--- a/Include/bytearrayobject.h
+++ b/Include/bytearrayobject.h
@@ -22,10 +22,11 @@ extern "C" {
#ifndef Py_LIMITED_API
typedef struct {
PyObject_VAR_HEAD
+ Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */
+ char *ob_bytes; /* Physical backing buffer */
+ char *ob_start; /* Logical start inside ob_bytes */
/* XXX(nnorwitz): should ob_exports be Py_ssize_t? */
- int ob_exports; /* how many buffer exports */
- Py_ssize_t ob_alloc; /* How many bytes allocated */
- char *ob_bytes;
+ int ob_exports; /* How many buffer exports */
} PyByteArrayObject;
#endif
@@ -49,8 +50,8 @@ PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t);
#ifndef Py_LIMITED_API
#define PyByteArray_AS_STRING(self) \
(assert(PyByteArray_Check(self)), \
- Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_bytes : _PyByteArray_empty_string)
-#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)),Py_SIZE(self))
+ Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string)
+#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self))
PyAPI_DATA(char) _PyByteArray_empty_string[];
#endif
diff --git a/Include/bytesobject.h b/Include/bytesobject.h
index d7c7ffd..0ee8d36 100644
--- a/Include/bytesobject.h
+++ b/Include/bytesobject.h
@@ -86,11 +86,11 @@ PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x);
0-terminated (passing a string with embedded NULL characters will
cause an exception). */
PyAPI_FUNC(int) PyBytes_AsStringAndSize(
- register PyObject *obj, /* string or Unicode object */
- register char **s, /* pointer to buffer variable */
- register Py_ssize_t *len /* pointer to length variable or NULL
- (only possible for 0-terminated
- strings) */
+ PyObject *obj, /* string or Unicode object */
+ char **s, /* pointer to buffer variable */
+ Py_ssize_t *len /* pointer to length variable or NULL
+ (only possible for 0-terminated
+ strings) */
);
/* Using the current locale, insert the thousands grouping
diff --git a/Include/compile.h b/Include/compile.h
index ac2636d..c6650d7f 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -36,11 +36,27 @@ PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
PyCompilerFlags *flags,
int optimize,
PyArena *arena);
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject(
+ struct _mod *mod,
+ PyObject *filename,
+ PyCompilerFlags *flags,
+ int optimize,
+ PyArena *arena);
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(
+ struct _mod * mod,
+ const char *filename /* decoded from the filesystem encoding */
+ );
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject(
+ struct _mod * mod,
+ PyObject *filename
+ );
/* _Py_Mangle is defined in compile.c */
PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
+#define PY_INVALID_STACK_EFFECT INT_MAX
+PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/dictobject.h b/Include/dictobject.h
index d89aac8..ef122bd 100644
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -53,6 +53,10 @@ PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key);
PyAPI_FUNC(PyObject *) PyDict_GetItemWithError(PyObject *mp, PyObject *key);
PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp,
struct _Py_Identifier *key);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) PyDict_SetDefault(
+ PyObject *mp, PyObject *key, PyObject *defaultobj);
+#endif
PyAPI_FUNC(int) PyDict_SetItem(PyObject *mp, PyObject *key, PyObject *item);
PyAPI_FUNC(int) PyDict_DelItem(PyObject *mp, PyObject *key);
PyAPI_FUNC(void) PyDict_Clear(PyObject *mp);
@@ -109,9 +113,11 @@ PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, struct _Py_Identifier *key, PyOb
PyAPI_FUNC(int) PyDict_DelItemString(PyObject *dp, const char *key);
#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, struct _Py_Identifier *key);
+PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out);
+
int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value);
PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
-PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out);
#endif
#ifdef __cplusplus
diff --git a/Include/fileobject.h b/Include/fileobject.h
index a99c94d..0939744 100644
--- a/Include/fileobject.h
+++ b/Include/fileobject.h
@@ -8,8 +8,9 @@ extern "C" {
#define PY_STDIOTEXTMODE "b"
-PyAPI_FUNC(PyObject *) PyFile_FromFd(int, char *, char *, int, char *, char *,
- char *, int);
+PyAPI_FUNC(PyObject *) PyFile_FromFd(int, const char *, const char *, int,
+ const char *, const char *,
+ const char *, int);
PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *);
diff --git a/Include/fileutils.h b/Include/fileutils.h
index 7c18cf2..e9bad80 100644
--- a/Include/fileutils.h
+++ b/Include/fileutils.h
@@ -27,11 +27,21 @@ PyAPI_FUNC(int) _Py_stat(
struct stat *statbuf);
#endif
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _Py_open(
+ const char *pathname,
+ int flags);
+#endif
+
PyAPI_FUNC(FILE *) _Py_wfopen(
const wchar_t *path,
const wchar_t *mode);
PyAPI_FUNC(FILE*) _Py_fopen(
+ const char *pathname,
+ const char *mode);
+
+PyAPI_FUNC(FILE*) _Py_fopen_obj(
PyObject *path,
const char *mode);
@@ -53,6 +63,15 @@ PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
wchar_t *buf,
size_t size);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _Py_get_inheritable(int fd);
+
+PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable,
+ int *atomic_flag_works);
+
+PyAPI_FUNC(int) _Py_dup(int fd);
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/frameobject.h b/Include/frameobject.h
index 33f73af..966ff1f 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -36,8 +36,9 @@ typedef struct _frame {
non-generator frames. See the save_exc_state and swap_exc_state
functions in ceval.c for details of their use. */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
+ /* Borrowed reference to a generator, or NULL */
+ PyObject *f_gen;
- PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
/* Call PyFrame_GetLineNumber() instead of reading this field
directly. As of 2.3 f_lineno is only valid when tracing is
@@ -46,6 +47,7 @@ typedef struct _frame {
bytecode index. */
int f_lineno; /* Current line number */
int f_iblock; /* index in f_blockstack */
+ char f_executing; /* whether the frame is still executing */
PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */
} PyFrameObject;
@@ -75,6 +77,8 @@ PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int);
/* Conversions between "fast locals" and locals in dictionary */
PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
+
+PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f);
PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
diff --git a/Include/genobject.h b/Include/genobject.h
index ed451ba..65f1ecf 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -36,6 +36,8 @@ PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);
+PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
+
#ifdef __cplusplus
}
diff --git a/Include/grammar.h b/Include/grammar.h
index 8426da3..ba7d19d 100644
--- a/Include/grammar.h
+++ b/Include/grammar.h
@@ -69,14 +69,14 @@ typedef struct {
/* FUNCTIONS */
grammar *newgrammar(int start);
-dfa *adddfa(grammar *g, int type, char *name);
+dfa *adddfa(grammar *g, int type, const char *name);
int addstate(dfa *d);
void addarc(dfa *d, int from, int to, int lbl);
dfa *PyGrammar_FindDFA(grammar *g, int type);
-int addlabel(labellist *ll, int type, char *str);
-int findlabel(labellist *ll, int type, char *str);
-char *PyGrammar_LabelRepr(label *lb);
+int addlabel(labellist *ll, int type, const char *str);
+int findlabel(labellist *ll, int type, const char *str);
+const char *PyGrammar_LabelRepr(label *lb);
void translatelabels(grammar *g);
void addfirstsets(grammar *g);
diff --git a/Include/import.h b/Include/import.h
index 73c86ee..afdfac2 100644
--- a/Include/import.h
+++ b/Include/import.h
@@ -13,19 +13,19 @@ PyMODINIT_FUNC PyInit_imp(void);
PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
PyAPI_FUNC(const char *) PyImport_GetMagicTag(void);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModule(
- char *name, /* UTF-8 encoded string */
+ const char *name, /* UTF-8 encoded string */
PyObject *co
);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleEx(
- char *name, /* UTF-8 encoded string */
+ const char *name, /* UTF-8 encoded string */
PyObject *co,
- char *pathname /* decoded from the filesystem encoding */
+ const char *pathname /* decoded from the filesystem encoding */
);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames(
- char *name, /* UTF-8 encoded string */
+ const char *name, /* UTF-8 encoded string */
PyObject *co,
- char *pathname, /* decoded from the filesystem encoding */
- char *cpathname /* decoded from the filesystem encoding */
+ const char *pathname, /* decoded from the filesystem encoding */
+ const char *cpathname /* decoded from the filesystem encoding */
);
PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject(
PyObject *name,
@@ -72,7 +72,7 @@ PyAPI_FUNC(int) PyImport_ImportFrozenModuleObject(
PyObject *name
);
PyAPI_FUNC(int) PyImport_ImportFrozenModule(
- char *name /* UTF-8 encoded string */
+ const char *name /* UTF-8 encoded string */
);
#ifndef Py_LIMITED_API
@@ -92,12 +92,12 @@ PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin(
PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *);
PyAPI_FUNC(int) _PyImport_FixupBuiltin(
PyObject *mod,
- char *name /* UTF-8 encoded string */
+ const char *name /* UTF-8 encoded string */
);
PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *);
struct _inittab {
- char *name; /* ASCII encoded string */
+ const char *name; /* ASCII encoded string */
PyObject* (*initfunc)(void);
};
PyAPI_DATA(struct _inittab *) PyImport_Inittab;
@@ -113,15 +113,15 @@ PyAPI_FUNC(int) PyImport_AppendInittab(
#ifndef Py_LIMITED_API
struct _frozen {
- char *name; /* ASCII encoded string */
- unsigned char *code;
+ const char *name; /* ASCII encoded string */
+ const unsigned char *code;
int size;
};
/* Embedding apps may change this pointer to point to their favorite
collection of frozen modules: */
-PyAPI_DATA(struct _frozen *) PyImport_FrozenModules;
+PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules;
#endif
#ifdef __cplusplus
diff --git a/Include/longobject.h b/Include/longobject.h
index 1c1c5de..ff43309 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -52,6 +52,19 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void);
#error "sizeof(pid_t) is neither sizeof(int), sizeof(long) or sizeof(long long)"
#endif /* SIZEOF_PID_T */
+#if SIZEOF_VOID_P == SIZEOF_INT
+# define _Py_PARSE_INTPTR "i"
+# define _Py_PARSE_UINTPTR "I"
+#elif SIZEOF_VOID_P == SIZEOF_LONG
+# define _Py_PARSE_INTPTR "l"
+# define _Py_PARSE_UINTPTR "k"
+#elif defined(SIZEOF_LONG_LONG) && SIZEOF_VOID_P == SIZEOF_LONG_LONG
+# define _Py_PARSE_INTPTR "L"
+# define _Py_PARSE_UINTPTR "K"
+#else
+# error "void* different in size from int, long and long long"
+#endif /* SIZEOF_VOID_P */
+
/* Used by Python/mystrtoul.c. */
#ifndef Py_LIMITED_API
PyAPI_DATA(unsigned char) _PyLong_DigitValue[256];
@@ -80,7 +93,7 @@ PyAPI_FUNC(unsigned PY_LONG_LONG) PyLong_AsUnsignedLongLongMask(PyObject *);
PyAPI_FUNC(PY_LONG_LONG) PyLong_AsLongLongAndOverflow(PyObject *, int *);
#endif /* HAVE_LONG_LONG */
-PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int);
+PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int);
PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base);
@@ -182,8 +195,8 @@ PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter(
/* These aren't really part of the int object, but they're handy. The
functions are in Python/mystrtoul.c.
*/
-PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
-PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
+PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int);
+PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int);
#ifdef __cplusplus
}
diff --git a/Include/marshal.h b/Include/marshal.h
index e96d062..09d9337 100644
--- a/Include/marshal.h
+++ b/Include/marshal.h
@@ -7,7 +7,7 @@
extern "C" {
#endif
-#define Py_MARSHAL_VERSION 2
+#define Py_MARSHAL_VERSION 4
PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int);
PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int);
@@ -19,7 +19,8 @@ PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *);
PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *);
PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *);
#endif
-PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(char *, Py_ssize_t);
+PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *,
+ Py_ssize_t);
#ifdef __cplusplus
}
diff --git a/Include/modsupport.h b/Include/modsupport.h
index ecf1dcc..5de0458 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -26,7 +26,7 @@ PyAPI_FUNC(PyObject *) _Py_VaBuildValue_SizeT(const char *, va_list);
/* Due to a glitch in 3.2, the _SizeT versions weren't exported from the DLL. */
#if !defined(PY_SSIZE_T_CLEAN) || !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
PyAPI_FUNC(int) PyArg_Parse(PyObject *, const char *, ...);
-PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...) Py_FORMAT_PARSETUPLE(PyArg_ParseTuple, 2, 3);
+PyAPI_FUNC(int) PyArg_ParseTuple(PyObject *, const char *, ...);
PyAPI_FUNC(int) PyArg_ParseTupleAndKeywords(PyObject *, PyObject *,
const char *, char **, ...);
PyAPI_FUNC(int) PyArg_ValidateKeywordArguments(PyObject *);
@@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...);
#endif
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _PyArg_NoKeywords(const char *funcname, PyObject *kw);
+PyAPI_FUNC(int) _PyArg_NoPositional(const char *funcname, PyObject *args);
PyAPI_FUNC(int) PyArg_VaParse(PyObject *, const char *, va_list);
PyAPI_FUNC(int) PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *,
diff --git a/Include/moduleobject.h b/Include/moduleobject.h
index 8013dd9..f119364 100644
--- a/Include/moduleobject.h
+++ b/Include/moduleobject.h
@@ -25,6 +25,7 @@ PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *);
PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyModule_Clear(PyObject *);
+PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *);
#endif
PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);
PyAPI_FUNC(void*) PyModule_GetState(PyObject*);
diff --git a/Include/object.h b/Include/object.h
index cdeb06c..7584d4c 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -143,7 +143,8 @@ typedef struct _Py_Identifier {
PyObject *object;
} _Py_Identifier;
-#define _Py_static_string(varname, value) static _Py_Identifier varname = { 0, value, 0 }
+#define _Py_static_string_init(value) { 0, value, 0 }
+#define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value)
#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname)
/*
@@ -362,7 +363,7 @@ typedef struct _typeobject {
PyBufferProcs *tp_as_buffer;
/* Flags to define presence of optional/expanded features */
- long tp_flags;
+ unsigned long tp_flags;
const char *tp_doc; /* Documentation string */
@@ -408,6 +409,8 @@ typedef struct _typeobject {
/* Type attribute cache version tag. Added in version 2.6 */
unsigned int tp_version_tag;
+ destructor tp_finalize;
+
#ifdef COUNT_ALLOCS
/* these must be last and never explicitly initialized */
Py_ssize_t tp_allocs;
@@ -428,7 +431,7 @@ typedef struct{
const char* name;
int basicsize;
int itemsize;
- int flags;
+ unsigned int flags;
PyType_Slot *slots; /* terminated by slot==0. */
} PyType_Spec;
@@ -436,6 +439,9 @@ PyAPI_FUNC(PyObject*) PyType_FromSpec(PyType_Spec*);
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
PyAPI_FUNC(PyObject*) PyType_FromSpecWithBases(PyType_Spec*, PyObject*);
#endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03040000
+PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int);
+#endif
#ifndef Py_LIMITED_API
/* The *real* layout of a type object when allocated on the heap */
@@ -470,7 +476,7 @@ PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */
PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */
PyAPI_DATA(PyTypeObject) PySuper_Type; /* built-in 'super' */
-PyAPI_FUNC(long) PyType_GetFlags(PyTypeObject*);
+PyAPI_FUNC(unsigned long) PyType_GetFlags(PyTypeObject*);
#define PyType_Check(op) \
PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
@@ -489,6 +495,11 @@ PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *
PyAPI_FUNC(unsigned int) PyType_ClearCache(void);
PyAPI_FUNC(void) PyType_Modified(PyTypeObject *);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *);
+PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *);
+#endif
+
/* Generic operations on objects */
struct _Py_Identifier;
#ifndef Py_LIMITED_API
@@ -530,6 +541,10 @@ PyAPI_FUNC(int) PyObject_Not(PyObject *);
PyAPI_FUNC(int) PyCallable_Check(PyObject *);
PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *);
+PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
+#endif
/* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes
dict as the last parameter. */
@@ -557,23 +572,6 @@ PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *);
PyAPI_FUNC(int) Py_ReprEnter(PyObject *);
PyAPI_FUNC(void) Py_ReprLeave(PyObject *);
-/* Helpers for hash functions */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double);
-PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*);
-PyAPI_FUNC(Py_hash_t) _Py_HashBytes(unsigned char*, Py_ssize_t);
-#endif
-
-typedef struct {
- Py_hash_t prefix;
- Py_hash_t suffix;
-} _Py_HashSecret_t;
-PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret;
-
-#ifdef Py_DEBUG
-PyAPI_DATA(int) _Py_HashSecret_Initialized;
-#endif
-
/* Helper for passing objects to printf and the like */
#define PyObject_REPR(obj) _PyUnicode_AsString(PyObject_Repr(obj))
@@ -604,50 +602,55 @@ given type object has a specified feature.
*/
/* Set if the type object is dynamically allocated */
-#define Py_TPFLAGS_HEAPTYPE (1L<<9)
+#define Py_TPFLAGS_HEAPTYPE (1UL << 9)
/* Set if the type allows subclassing */
-#define Py_TPFLAGS_BASETYPE (1L<<10)
+#define Py_TPFLAGS_BASETYPE (1UL << 10)
/* Set if the type is 'ready' -- fully initialized */
-#define Py_TPFLAGS_READY (1L<<12)
+#define Py_TPFLAGS_READY (1UL << 12)
/* Set while the type is being 'readied', to prevent recursive ready calls */
-#define Py_TPFLAGS_READYING (1L<<13)
+#define Py_TPFLAGS_READYING (1UL << 13)
/* Objects support garbage collection (see objimp.h) */
-#define Py_TPFLAGS_HAVE_GC (1L<<14)
+#define Py_TPFLAGS_HAVE_GC (1UL << 14)
/* These two bits are preserved for Stackless Python, next after this is 17 */
#ifdef STACKLESS
-#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3L<<15)
+#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15)
#else
#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0
#endif
/* Objects support type attribute cache */
-#define Py_TPFLAGS_HAVE_VERSION_TAG (1L<<18)
-#define Py_TPFLAGS_VALID_VERSION_TAG (1L<<19)
+#define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18)
+#define Py_TPFLAGS_VALID_VERSION_TAG (1UL << 19)
/* Type is abstract and cannot be instantiated */
-#define Py_TPFLAGS_IS_ABSTRACT (1L<<20)
+#define Py_TPFLAGS_IS_ABSTRACT (1UL << 20)
/* These flags are used to determine if a type is a subclass. */
-#define Py_TPFLAGS_INT_SUBCLASS (1L<<23)
-#define Py_TPFLAGS_LONG_SUBCLASS (1L<<24)
-#define Py_TPFLAGS_LIST_SUBCLASS (1L<<25)
-#define Py_TPFLAGS_TUPLE_SUBCLASS (1L<<26)
-#define Py_TPFLAGS_BYTES_SUBCLASS (1L<<27)
-#define Py_TPFLAGS_UNICODE_SUBCLASS (1L<<28)
-#define Py_TPFLAGS_DICT_SUBCLASS (1L<<29)
-#define Py_TPFLAGS_BASE_EXC_SUBCLASS (1L<<30)
-#define Py_TPFLAGS_TYPE_SUBCLASS (1L<<31)
+#define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24)
+#define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25)
+#define Py_TPFLAGS_TUPLE_SUBCLASS (1UL << 26)
+#define Py_TPFLAGS_BYTES_SUBCLASS (1UL << 27)
+#define Py_TPFLAGS_UNICODE_SUBCLASS (1UL << 28)
+#define Py_TPFLAGS_DICT_SUBCLASS (1UL << 29)
+#define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30)
+#define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31)
#define Py_TPFLAGS_DEFAULT ( \
Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \
Py_TPFLAGS_HAVE_VERSION_TAG | \
0)
+/* NOTE: The following flags reuse lower bits (removed as part of the
+ * Python 3.0 transition). */
+
+/* Type structure has tp_finalize member (3.4) */
+#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
+
#ifdef Py_LIMITED_API
#define PyType_HasFeature(t,f) ((PyType_GetFlags(t) & (f)) != 0)
#else
@@ -682,12 +685,6 @@ is not considered to be a reference to the type object, to save
complications in the deallocation function. (This is actually a
decision that's up to the implementer of each new type so if you want,
you can count such references to the type object.)
-
-*** WARNING*** The Py_DECREF macro must have a side-effect-free argument
-since it may evaluate its argument multiple times. (The alternative
-would be to mace it a proper function or assign it to a global temporary
-variable first, both of which are slower; and in a multi-threaded
-environment the global variable trick is not safe.)
*/
/* First define a pile of simple helper macros, one set per special
@@ -704,7 +701,6 @@ PyAPI_DATA(Py_ssize_t) _Py_RefTotal;
PyAPI_FUNC(void) _Py_NegativeRefcount(const char *fname,
int lineno, PyObject *op);
PyAPI_FUNC(PyObject *) _PyDict_Dummy(void);
-PyAPI_FUNC(PyObject *) _PySet_Dummy(void);
PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
#define _Py_INC_REFTOTAL _Py_RefTotal++
#define _Py_DEC_REFTOTAL _Py_RefTotal--
@@ -766,15 +762,16 @@ PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
#define Py_INCREF(op) ( \
_Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA \
- ((PyObject*)(op))->ob_refcnt++)
+ ((PyObject *)(op))->ob_refcnt++)
#define Py_DECREF(op) \
do { \
+ PyObject *_py_decref_tmp = (PyObject *)(op); \
if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA \
- --((PyObject*)(op))->ob_refcnt != 0) \
- _Py_CHECK_REFCNT(op) \
+ --(_py_decref_tmp)->ob_refcnt != 0) \
+ _Py_CHECK_REFCNT(_py_decref_tmp) \
else \
- _Py_Dealloc((PyObject *)(op)); \
+ _Py_Dealloc(_py_decref_tmp); \
} while (0)
/* Safely decref `op` and set `op` to NULL, especially useful in tp_clear
@@ -813,16 +810,27 @@ PyAPI_FUNC(void) _Py_Dealloc(PyObject *);
*/
#define Py_CLEAR(op) \
do { \
- if (op) { \
- PyObject *_py_tmp = (PyObject *)(op); \
+ PyObject *_py_tmp = (PyObject *)(op); \
+ if (_py_tmp != NULL) { \
(op) = NULL; \
Py_DECREF(_py_tmp); \
} \
} while (0)
/* Macros to use in case the object pointer may be NULL: */
-#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
-#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+#define Py_XINCREF(op) \
+ do { \
+ PyObject *_py_xincref_tmp = (PyObject *)(op); \
+ if (_py_xincref_tmp != NULL) \
+ Py_INCREF(_py_xincref_tmp); \
+ } while (0)
+
+#define Py_XDECREF(op) \
+ do { \
+ PyObject *_py_xdecref_tmp = (PyObject *)(op); \
+ if (_py_xdecref_tmp != NULL) \
+ Py_DECREF(_py_xdecref_tmp); \
+ } while (0)
/*
These are provided as conveniences to Python runtime embedders, so that
diff --git a/Include/objimpl.h b/Include/objimpl.h
index c6b7df4..3f21b70 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -94,51 +94,27 @@ PyObject_{New, NewVar, Del}.
the object gets initialized via PyObject_{Init, InitVar} after obtaining
the raw memory.
*/
-PyAPI_FUNC(void *) PyObject_Malloc(size_t);
-PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyObject_Free(void *);
+PyAPI_FUNC(void *) PyObject_Malloc(size_t size);
+PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyObject_Free(void *ptr);
+/* This function returns the number of allocated memory blocks, regardless of size */
+PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
/* Macros */
#ifdef WITH_PYMALLOC
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
#endif /* #ifndef Py_LIMITED_API */
-#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */
-PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
-PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
-PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
-PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
-PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
-PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
-PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
-PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
-#define PyObject_MALLOC _PyObject_DebugMalloc
-#define PyObject_Malloc _PyObject_DebugMalloc
-#define PyObject_REALLOC _PyObject_DebugRealloc
-#define PyObject_Realloc _PyObject_DebugRealloc
-#define PyObject_FREE _PyObject_DebugFree
-#define PyObject_Free _PyObject_DebugFree
-
-#else /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */
+#endif
+
+/* Macros */
#define PyObject_MALLOC PyObject_Malloc
#define PyObject_REALLOC PyObject_Realloc
#define PyObject_FREE PyObject_Free
-#endif
-
-#else /* ! WITH_PYMALLOC */
-#define PyObject_MALLOC PyMem_MALLOC
-#define PyObject_REALLOC PyMem_REALLOC
-#define PyObject_FREE PyMem_FREE
-
-#endif /* WITH_PYMALLOC */
-
#define PyObject_Del PyObject_Free
-#define PyObject_DEL PyObject_FREE
+#define PyObject_DEL PyObject_Free
+
/*
* Generic object allocator interface
@@ -222,6 +198,26 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
constructor you would start directly with PyObject_Init/InitVar
*/
+#ifndef Py_LIMITED_API
+typedef struct {
+ /* user context passed as the first argument to the 2 functions */
+ void *ctx;
+
+ /* allocate an arena of size bytes */
+ void* (*alloc) (void *ctx, size_t size);
+
+ /* free an arena */
+ void (*free) (void *ctx, void *ptr, size_t size);
+} PyObjectArenaAllocator;
+
+/* Get the arena allocator. */
+PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
+
+/* Set the arena allocator. */
+PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
+#endif
+
+
/*
* Garbage Collection Support
* ==========================
@@ -230,6 +226,10 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t);
/* C equivalent of gc.collect(). */
PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_ssize_t) _PyGC_CollectNoFail(void);
+#endif
+
/* Test if a type has a GC head */
#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
@@ -249,13 +249,37 @@ typedef union _gc_head {
union _gc_head *gc_prev;
Py_ssize_t gc_refs;
} gc;
- long double dummy; /* force worst-case alignment */
+ double dummy; /* force worst-case alignment */
} PyGC_Head;
extern PyGC_Head *_PyGC_generation0;
#define _Py_AS_GC(o) ((PyGC_Head *)(o)-1)
+/* Bit 0 is set when tp_finalize is called */
+#define _PyGC_REFS_MASK_FINALIZED (1 << 0)
+/* The (N-1) most significant bits contain the gc state / refcount */
+#define _PyGC_REFS_SHIFT (1)
+#define _PyGC_REFS_MASK (((size_t) -1) << _PyGC_REFS_SHIFT)
+
+#define _PyGCHead_REFS(g) ((g)->gc.gc_refs >> _PyGC_REFS_SHIFT)
+#define _PyGCHead_SET_REFS(g, v) do { \
+ (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK) \
+ | (((size_t)(v)) << _PyGC_REFS_SHIFT); \
+ } while (0)
+#define _PyGCHead_DECREF(g) ((g)->gc.gc_refs -= 1 << _PyGC_REFS_SHIFT)
+
+#define _PyGCHead_FINALIZED(g) (((g)->gc.gc_refs & _PyGC_REFS_MASK_FINALIZED) != 0)
+#define _PyGCHead_SET_FINALIZED(g, v) do { \
+ (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \
+ | (v != 0); \
+ } while (0)
+
+#define _PyGC_FINALIZED(o) _PyGCHead_FINALIZED(_Py_AS_GC(o))
+#define _PyGC_SET_FINALIZED(o, v) _PyGCHead_SET_FINALIZED(_Py_AS_GC(o), v)
+
+#define _PyGC_REFS(o) _PyGCHead_REFS(_Py_AS_GC(o))
+
#define _PyGC_REFS_UNTRACKED (-2)
#define _PyGC_REFS_REACHABLE (-3)
#define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4)
@@ -264,9 +288,9 @@ extern PyGC_Head *_PyGC_generation0;
* collector it must be safe to call the ob_traverse method. */
#define _PyObject_GC_TRACK(o) do { \
PyGC_Head *g = _Py_AS_GC(o); \
- if (g->gc.gc_refs != _PyGC_REFS_UNTRACKED) \
+ if (_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED) \
Py_FatalError("GC object already tracked"); \
- g->gc.gc_refs = _PyGC_REFS_REACHABLE; \
+ _PyGCHead_SET_REFS(g, _PyGC_REFS_REACHABLE); \
g->gc.gc_next = _PyGC_generation0; \
g->gc.gc_prev = _PyGC_generation0->gc.gc_prev; \
g->gc.gc_prev->gc.gc_next = g; \
@@ -279,8 +303,8 @@ extern PyGC_Head *_PyGC_generation0;
*/
#define _PyObject_GC_UNTRACK(o) do { \
PyGC_Head *g = _Py_AS_GC(o); \
- assert(g->gc.gc_refs != _PyGC_REFS_UNTRACKED); \
- g->gc.gc_refs = _PyGC_REFS_UNTRACKED; \
+ assert(_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED); \
+ _PyGCHead_SET_REFS(g, _PyGC_REFS_UNTRACKED); \
g->gc.gc_prev->gc.gc_next = g->gc.gc_next; \
g->gc.gc_next->gc.gc_prev = g->gc.gc_prev; \
g->gc.gc_next = NULL; \
@@ -288,7 +312,7 @@ extern PyGC_Head *_PyGC_generation0;
/* True if the object is currently tracked by the GC. */
#define _PyObject_GC_IS_TRACKED(o) \
- ((_Py_AS_GC(o))->gc.gc_refs != _PyGC_REFS_UNTRACKED)
+ (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED)
/* True if the object may be tracked by the GC in the future, or already is.
This can be useful to implement some optimizations. */
diff --git a/Include/opcode.h b/Include/opcode.h
index a90184d..0936f2d 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -49,7 +49,6 @@ extern "C" {
#define BINARY_OR 66
#define INPLACE_POWER 67
#define GET_ITER 68
-#define STORE_LOCALS 69
#define PRINT_EXPR 70
#define LOAD_BUILD_CLASS 71
#define YIELD_FROM 72
@@ -140,6 +139,7 @@ extern "C" {
#define SET_ADD 146
#define MAP_ADD 147
+#define LOAD_CLASSDEREF 148
/* EXCEPT_HANDLER is a special, implicit block type which is created when
entering an except handler. It is not an opcode but we define it here
diff --git a/Include/osdefs.h b/Include/osdefs.h
index 05c0c8e..0c2e34b 100644
--- a/Include/osdefs.h
+++ b/Include/osdefs.h
@@ -9,16 +9,10 @@ extern "C" {
/* Mod by chrish: QNX has WATCOM, but isn't DOS */
#if !defined(__QNX__)
-#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__) || defined(PYOS_OS2)
-#if defined(PYOS_OS2) && defined(PYCC_GCC)
-#define MAXPATHLEN 260
-#define SEP L'/'
-#define ALTSEP L'\\'
-#else
+#if defined(MS_WINDOWS) || defined(__BORLANDC__) || defined(__WATCOMC__) || defined(__DJGPP__)
#define SEP L'\\'
#define ALTSEP L'/'
#define MAXPATHLEN 256
-#endif
#define DELIM L';'
#endif
#endif
diff --git a/Include/parsetok.h b/Include/parsetok.h
index 911dfc1..2acb854 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -38,29 +38,49 @@ typedef struct {
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
perrdetail *);
PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
- char *, char *, perrdetail *);
+ const char *, const char *,
+ perrdetail *);
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
- const char*, grammar *,
- int, char *, char *,
- perrdetail *, int);
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ const char *enc,
+ grammar *g,
+ int start,
+ const char *ps1,
+ const char *ps2,
+ perrdetail *err_ret,
+ int flags);
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
FILE *fp,
const char *filename, /* decoded from the filesystem encoding */
const char *enc,
grammar *g,
int start,
- char *ps1,
- char *ps2,
+ const char *ps1,
+ const char *ps2,
+ perrdetail *err_ret,
+ int *flags);
+PyAPI_FUNC(node *) PyParser_ParseFileObject(
+ FILE *fp,
+ PyObject *filename,
+ const char *enc,
+ grammar *g,
+ int start,
+ const char *ps1,
+ const char *ps2,
perrdetail *err_ret,
int *flags);
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
- const char *,
- grammar *, int,
- perrdetail *, int);
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(
+ const char *s,
+ const char *filename, /* decoded from the filesystem encoding */
+ grammar *g,
+ int start,
+ perrdetail *err_ret,
+ int flags);
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
const char *s,
const char *filename, /* decoded from the filesystem encoding */
@@ -68,6 +88,13 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
int start,
perrdetail *err_ret,
int *flags);
+PyAPI_FUNC(node *) PyParser_ParseStringObject(
+ const char *s,
+ PyObject *filename,
+ grammar *g,
+ int start,
+ perrdetail *err_ret,
+ int *flags);
/* Note that the following functions are defined in pythonrun.c,
not in parsetok.c */
diff --git a/Include/patchlevel.h b/Include/patchlevel.h
index 61a3ba0..72f540d 100644
--- a/Include/patchlevel.h
+++ b/Include/patchlevel.h
@@ -17,13 +17,13 @@
/* Version parsed out into numeric values */
/*--start constants--*/
#define PY_MAJOR_VERSION 3
-#define PY_MINOR_VERSION 3
-#define PY_MICRO_VERSION 5
+#define PY_MINOR_VERSION 4
+#define PY_MICRO_VERSION 0
#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL
#define PY_RELEASE_SERIAL 0
/* Version as a string */
-#define PY_VERSION "3.3.5+"
+#define PY_VERSION "3.4.0+"
/*--end constants--*/
/* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.
diff --git a/Include/pydebug.h b/Include/pydebug.h
index 97c2f8c..8fe9818 100644
--- a/Include/pydebug.h
+++ b/Include/pydebug.h
@@ -20,6 +20,7 @@ PyAPI_DATA(int) Py_DontWriteBytecodeFlag;
PyAPI_DATA(int) Py_NoUserSiteDirectory;
PyAPI_DATA(int) Py_UnbufferedStdioFlag;
PyAPI_DATA(int) Py_HashRandomizationFlag;
+PyAPI_DATA(int) Py_IsolatedFlag;
/* this is a wrapper around getenv() that pays attention to
Py_IgnoreEnvironmentFlag. It should be used for getting variables like
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
index e385123..e44fb5f 100644
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -53,6 +53,7 @@ typedef struct {
PyObject *myerrno;
PyObject *strerror;
PyObject *filename;
+ PyObject *filename2;
#ifdef MS_WINDOWS
PyObject *winerror;
#endif
@@ -75,6 +76,9 @@ typedef PyOSErrorObject PyWindowsErrorObject;
PyAPI_FUNC(void) PyErr_SetNone(PyObject *);
PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *);
+#endif
PyAPI_FUNC(void) PyErr_SetString(
PyObject *exception,
const char *string /* decoded from utf-8 */
@@ -198,9 +202,6 @@ PyAPI_DATA(PyObject *) PyExc_IOError;
#ifdef MS_WINDOWS
PyAPI_DATA(PyObject *) PyExc_WindowsError;
#endif
-#ifdef __VMS
-PyAPI_DATA(PyObject *) PyExc_VMSError;
-#endif
PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst;
@@ -225,6 +226,8 @@ PyAPI_FUNC(PyObject *) PyErr_NoMemory(void);
PyAPI_FUNC(PyObject *) PyErr_SetFromErrno(PyObject *);
PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilenameObject(
PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilenameObjects(
+ PyObject *, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyErr_SetFromErrnoWithFilename(
PyObject *exc,
const char *filename /* decoded from the filesystem encoding */
@@ -253,6 +256,8 @@ PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithUnicodeFilename(
PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErr(int);
PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilenameObject(
PyObject *,int, PyObject *);
+PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilenameObjects(
+ PyObject *,int, PyObject *, PyObject *);
PyAPI_FUNC(PyObject *) PyErr_SetExcFromWindowsErrWithFilename(
PyObject *exc,
int ierr,
@@ -284,6 +289,28 @@ PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(
const char *name, const char *doc, PyObject *base, PyObject *dict);
PyAPI_FUNC(void) PyErr_WriteUnraisable(PyObject *);
+/* In exceptions.c */
+#ifndef Py_LIMITED_API
+/* Helper that attempts to replace the current exception with one of the
+ * same type but with a prefix added to the exception text. The resulting
+ * exception description looks like:
+ *
+ * prefix (exc_type: original_exc_str)
+ *
+ * Only some exceptions can be safely replaced. If the function determines
+ * it isn't safe to perform the replacement, it will leave the original
+ * unmodified exception in place.
+ *
+ * Returns a borrowed reference to the new exception (if any), NULL if the
+ * existing exception was left in place.
+ */
+PyAPI_FUNC(PyObject *) _PyErr_TrySetFromCause(
+ const char *prefix_format, /* ASCII-encoded string */
+ ...
+ );
+#endif
+
+
/* In sigcheck.c or signalmodule.c */
PyAPI_FUNC(int) PyErr_CheckSignals(void);
PyAPI_FUNC(void) PyErr_SetInterrupt(void);
@@ -301,9 +328,20 @@ PyAPI_FUNC(void) PyErr_SyntaxLocationEx(
const char *filename, /* decoded from the filesystem encoding */
int lineno,
int col_offset);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) PyErr_SyntaxLocationObject(
+ PyObject *filename,
+ int lineno,
+ int col_offset);
+#endif
PyAPI_FUNC(PyObject *) PyErr_ProgramText(
const char *filename, /* decoded from the filesystem encoding */
int lineno);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject(
+ PyObject *filename,
+ int lineno);
+#endif
/* The following functions are used to create and modify unicode
exceptions from C */
diff --git a/Include/pyhash.h b/Include/pyhash.h
new file mode 100644
index 0000000..a7ca937
--- /dev/null
+++ b/Include/pyhash.h
@@ -0,0 +1,149 @@
+#ifndef Py_HASH_H
+
+#define Py_HASH_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Helpers for hash functions */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double);
+PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*);
+PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t);
+#endif
+
+/* Prime multiplier used in string and various other hashes. */
+#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */
+
+/* Parameters used for the numeric hash implementation. See notes for
+ _Py_HashDouble in Objects/object.c. Numeric hashes are based on
+ reduction modulo the prime 2**_PyHASH_BITS - 1. */
+
+#if SIZEOF_VOID_P >= 8
+# define _PyHASH_BITS 61
+#else
+# define _PyHASH_BITS 31
+#endif
+
+#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1)
+#define _PyHASH_INF 314159
+#define _PyHASH_NAN 0
+#define _PyHASH_IMAG _PyHASH_MULTIPLIER
+
+
+/* hash secret
+ *
+ * memory layout on 64 bit systems
+ * cccccccc cccccccc cccccccc uc -- unsigned char[24]
+ * pppppppp ssssssss ........ fnv -- two Py_hash_t
+ * k0k0k0k0 k1k1k1k1 ........ siphash -- two PY_UINT64_T
+ * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t
+ * ........ ........ eeeeeeee pyexpat XML hash salt
+ *
+ * memory layout on 32 bit systems
+ * cccccccc cccccccc cccccccc uc
+ * ppppssss ........ ........ fnv -- two Py_hash_t
+ * k0k0k0k0 k1k1k1k1 ........ siphash -- two PY_UINT64_T (*)
+ * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t
+ * ........ ........ eeee.... pyexpat XML hash salt
+ *
+ * (*) The siphash member may not be available on 32 bit platforms without
+ * an unsigned int64 data type.
+ */
+#ifndef Py_LIMITED_API
+typedef union {
+ /* ensure 24 bytes */
+ unsigned char uc[24];
+ /* two Py_hash_t for FNV */
+ struct {
+ Py_hash_t prefix;
+ Py_hash_t suffix;
+ } fnv;
+#ifdef PY_UINT64_T
+ /* two uint64 for SipHash24 */
+ struct {
+ PY_UINT64_T k0;
+ PY_UINT64_T k1;
+ } siphash;
+#endif
+ /* a different (!) Py_hash_t for small string optimization */
+ struct {
+ unsigned char padding[16];
+ Py_hash_t suffix;
+ } djbx33a;
+ struct {
+ unsigned char padding[16];
+ Py_hash_t hashsalt;
+ } expat;
+} _Py_HashSecret_t;
+PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret;
+#endif
+
+#ifdef Py_DEBUG
+PyAPI_DATA(int) _Py_HashSecret_Initialized;
+#endif
+
+
+/* hash function definition */
+#ifndef Py_LIMITED_API
+typedef struct {
+ Py_hash_t (*const hash)(const void *, Py_ssize_t);
+ const char *name;
+ const int hash_bits;
+ const int seed_bits;
+} PyHash_FuncDef;
+
+PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void);
+#endif
+
+
+/* cutoff for small string DJBX33A optimization in range [1, cutoff).
+ *
+ * About 50% of the strings in a typical Python application are smaller than
+ * 6 to 7 chars. However DJBX33A is vulnerable to hash collision attacks.
+ * NEVER use DJBX33A for long strings!
+ *
+ * A Py_HASH_CUTOFF of 0 disables small string optimization. 32 bit platforms
+ * should use a smaller cutoff because it is easier to create colliding
+ * strings. A cutoff of 7 on 64bit platforms and 5 on 32bit platforms should
+ * provide a decent safety margin.
+ */
+#ifndef Py_HASH_CUTOFF
+# define Py_HASH_CUTOFF 0
+#elif (Py_HASH_CUTOFF > 7 || Py_HASH_CUTOFF < 0)
+# error Py_HASH_CUTOFF must in range 0...7.
+#endif /* Py_HASH_CUTOFF */
+
+
+/* hash algorithm selection
+ *
+ * The values for Py_HASH_SIPHASH24 and Py_HASH_FNV are hard-coded in the
+ * configure script.
+ *
+ * - FNV is available on all platforms and architectures.
+ * - SIPHASH24 only works on plaforms that provide PY_UINT64_T and doesn't
+ * require aligned memory for integers.
+ * - With EXTERNAL embedders can provide an alternative implementation with::
+ *
+ * PyHash_FuncDef PyHash_Func = {...};
+ *
+ * XXX: Figure out __declspec() for extern PyHash_FuncDef.
+ */
+#define Py_HASH_EXTERNAL 0
+#define Py_HASH_SIPHASH24 1
+#define Py_HASH_FNV 2
+
+#ifndef Py_HASH_ALGORITHM
+# if (defined(PY_UINT64_T) && defined(PY_UINT32_T) \
+ && !defined(HAVE_ALIGNED_REQUIRED))
+# define Py_HASH_ALGORITHM Py_HASH_SIPHASH24
+# else
+# define Py_HASH_ALGORITHM Py_HASH_FNV
+# endif /* uint64_t && uint32_t && aligned */
+#endif /* Py_HASH_ALGORITHM */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !Py_HASH_H */
diff --git a/Include/pymacro.h b/Include/pymacro.h
index 793f67d..7997c55 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -69,4 +69,10 @@
/* Check if pointer "p" is aligned to "a"-bytes boundary. */
#define _Py_IS_ALIGNED(p, a) (!((Py_uintptr_t)(p) & (Py_uintptr_t)((a) - 1)))
+#ifdef __GNUC__
+#define Py_UNUSED(name) _unused_ ## name __attribute__((unused))
+#else
+#define Py_UNUSED(name) _unused_ ## name
+#endif
+
#endif /* Py_PYMACRO_H */
diff --git a/Include/pymem.h b/Include/pymem.h
index 10b5bea..2372b86 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -11,6 +11,13 @@
extern "C" {
#endif
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
+PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
+#endif
+
+
/* BEWARE:
Each interface exports both functions and macros. Extension modules should
@@ -49,21 +56,16 @@ extern "C" {
performed on failure (no exception is set, no warning is printed, etc).
*/
-PyAPI_FUNC(void *) PyMem_Malloc(size_t);
-PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t);
-PyAPI_FUNC(void) PyMem_Free(void *);
+PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
+PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
+PyAPI_FUNC(void) PyMem_Free(void *ptr);
-/* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
- no longer supported. They used to call PyErr_NoMemory() on failure. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
+PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
+#endif
/* Macros. */
-#ifdef PYMALLOC_DEBUG
-/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC _PyMem_DebugMalloc
-#define PyMem_REALLOC _PyMem_DebugRealloc
-#define PyMem_FREE _PyMem_DebugFree
-
-#else /* ! PYMALLOC_DEBUG */
/* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
for malloc(0), which would be treated as an error. Some platforms
@@ -71,13 +73,9 @@ PyAPI_FUNC(void) PyMem_Free(void *);
pymalloc. To solve these problems, allocate an extra byte. */
/* Returns NULL to indicate error if a negative size or size larger than
Py_ssize_t can represent is supplied. Helps prevents security holes. */
-#define PyMem_MALLOC(n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
- : malloc((n) ? (n) : 1))
-#define PyMem_REALLOC(p, n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \
- : realloc((p), (n) ? (n) : 1))
-#define PyMem_FREE free
-
-#endif /* PYMALLOC_DEBUG */
+#define PyMem_MALLOC(n) PyMem_Malloc(n)
+#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n)
+#define PyMem_FREE(p) PyMem_Free(p)
/*
* Type-oriented memory interface
@@ -115,6 +113,69 @@ PyAPI_FUNC(void) PyMem_Free(void *);
#define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE
+#ifndef Py_LIMITED_API
+typedef enum {
+ /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
+ PYMEM_DOMAIN_RAW,
+
+ /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
+ PYMEM_DOMAIN_MEM,
+
+ /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
+ PYMEM_DOMAIN_OBJ
+} PyMemAllocatorDomain;
+
+typedef struct {
+ /* user context passed as the first argument to the 3 functions */
+ void *ctx;
+
+ /* allocate a memory block */
+ void* (*malloc) (void *ctx, size_t size);
+
+ /* allocate or resize a memory block */
+ void* (*realloc) (void *ctx, void *ptr, size_t new_size);
+
+ /* release a memory block */
+ void (*free) (void *ctx, void *ptr);
+} PyMemAllocator;
+
+/* Get the memory block allocator of the specified domain. */
+PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
+ PyMemAllocator *allocator);
+
+/* Set the memory block allocator of the specified domain.
+
+ The new allocator must return a distinct non-NULL pointer when requesting
+ zero bytes.
+
+ For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
+ is not held when the allocator is called.
+
+ If the new allocator is not a hook (don't call the previous allocator), the
+ PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
+ on top on the new allocator. */
+PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
+ PyMemAllocator *allocator);
+
+/* Setup hooks to detect bugs in the following Python memory allocator
+ functions:
+
+ - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
+ - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
+ - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()
+
+ Newly allocated memory is filled with the byte 0xCB, freed memory is filled
+ with the byte 0xDB. Additionnal checks:
+
+ - detect API violations, ex: PyObject_Free() called on a buffer allocated
+ by PyMem_Malloc()
+ - detect write before the start of the buffer (buffer underflow)
+ - detect write after the end of the buffer (buffer overflow)
+
+ The function does nothing if Python is not compiled is debug mode. */
+PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/pyport.h b/Include/pyport.h
index a5edea9..c706213 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -144,23 +144,6 @@ Used in: PY_LONG_LONG
#endif
#endif
-/* Prime multiplier used in string and various other hashes. */
-#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */
-
-/* Parameters used for the numeric hash implementation. See notes for
- _Py_HashDouble in Objects/object.c. Numeric hashes are based on
- reduction modulo the prime 2**_PyHASH_BITS - 1. */
-
-#if SIZEOF_VOID_P >= 8
-#define _PyHASH_BITS 61
-#else
-#define _PyHASH_BITS 31
-#endif
-#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1)
-#define _PyHASH_INF 314159
-#define _PyHASH_NAN 0
-#define _PyHASH_IMAG _PyHASH_MULTIPLIER
-
/* uintptr_t is the C9X name for an unsigned integral type such that a
* legitimate void* can be cast to uintptr_t and then back to void* again
* without loss of information. Similarly for intptr_t, wrt a signed
@@ -199,10 +182,19 @@ typedef Py_intptr_t Py_ssize_t;
#endif
/* Py_hash_t is the same size as a pointer. */
+#define SIZEOF_PY_HASH_T SIZEOF_SIZE_T
typedef Py_ssize_t Py_hash_t;
/* Py_uhash_t is the unsigned equivalent needed to calculate numeric hash. */
+#define SIZEOF_PY_UHASH_T SIZEOF_SIZE_T
typedef size_t Py_uhash_t;
+/* Only used for compatibility with code that may not be PY_SSIZE_T_CLEAN. */
+#ifdef PY_SSIZE_T_CLEAN
+typedef Py_ssize_t Py_ssize_clean_t;
+#else
+typedef int Py_ssize_clean_t;
+#endif
+
/* Largest possible value of size_t.
SIZE_MAX is part of C99, so it might be defined on some
platforms. If it is not defined, (size_t)-1 is a portable
@@ -219,10 +211,6 @@ typedef size_t Py_uhash_t;
/* Smallest negative value of type Py_ssize_t. */
#define PY_SSIZE_T_MIN (-PY_SSIZE_T_MAX-1)
-#if SIZEOF_PID_T > SIZEOF_LONG
-# error "Python doesn't support sizeof(pid_t) > sizeof(long)"
-#endif
-
/* PY_FORMAT_SIZE_T is a platform-specific modifier for use in a printf
* format to convert an argument with the width of a size_t or Py_ssize_t.
* C99 introduced "z" for this purpose, but not all platforms support that;
@@ -267,7 +255,7 @@ typedef size_t Py_uhash_t;
*/
#ifdef HAVE_LONG_LONG
# ifndef PY_FORMAT_LONG_LONG
-# if defined(MS_WIN64) || defined(MS_WINDOWS)
+# ifdef MS_WINDOWS
# define PY_FORMAT_LONG_LONG "I64"
# else
# error "This platform's pyconfig.h needs to define PY_FORMAT_LONG_LONG"
@@ -392,17 +380,20 @@ typedef size_t Py_uhash_t;
#endif
#ifdef HAVE_SYS_STAT_H
-#if defined(PYOS_OS2) && defined(PYCC_GCC)
-#include <sys/types.h>
-#endif
#include <sys/stat.h>
#elif defined(HAVE_STAT_H)
#include <stat.h>
#endif
-#if defined(PYCC_VACPP)
+#ifndef S_IFMT
/* VisualAge C/C++ Failed to Define MountType Field in sys/stat.h */
-#define S_IFMT (S_IFDIR|S_IFCHR|S_IFREG)
+#define S_IFMT 0170000
+#endif
+
+#ifndef S_IFLNK
+/* Windows doesn't define S_IFLNK but posixmodule.c maps
+ * IO_REPARSE_TAG_SYMLINK to S_IFLNK */
+# define S_IFLNK 0120000
#endif
#ifndef S_ISREG
@@ -413,6 +404,9 @@ typedef size_t Py_uhash_t;
#define S_ISDIR(x) (((x) & S_IFMT) == S_IFDIR)
#endif
+#ifndef S_ISCHR
+#define S_ISCHR(x) (((x) & S_IFMT) == S_IFCHR)
+#endif
#ifdef __cplusplus
/* Move this down here since some C++ #include's don't like to be included
@@ -835,15 +829,6 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
#endif
/*
- * Add PyArg_ParseTuple format where available.
- */
-#ifdef HAVE_ATTRIBUTE_FORMAT_PARSETUPLE
-#define Py_FORMAT_PARSETUPLE(func,p1,p2) __attribute__((format(func,p1,p2)))
-#else
-#define Py_FORMAT_PARSETUPLE(func,p1,p2)
-#endif
-
-/*
* Specify alignment on compilers that support it.
*/
#if defined(__GNUC__) && __GNUC__ >= 3
@@ -881,4 +866,18 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
#endif
#endif
+/*
+ * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is
+ * detected by configure and defined in pyconfig.h. The code in pyconfig.h
+ * also takes care of Apple's universal builds.
+ */
+
+#ifdef WORDS_BIGENDIAN
+#define PY_BIG_ENDIAN 1
+#define PY_LITTLE_ENDIAN 0
+#else
+#define PY_BIG_ENDIAN 0
+#define PY_LITTLE_ENDIAN 1
+#endif
+
#endif /* Py_PYPORT_H */
diff --git a/Include/pystate.h b/Include/pystate.h
index 2017b02..4992c22 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -33,7 +33,6 @@ typedef struct _is {
int codecs_initialized;
int fscodec_initialized;
-
#ifdef HAVE_DLOPEN
int dlopenflags;
#endif
@@ -41,6 +40,7 @@ typedef struct _is {
int tscdump;
#endif
+ PyObject *builtins_copy;
} PyInterpreterState;
#endif
@@ -69,6 +69,7 @@ typedef struct _ts PyThreadState;
typedef struct _ts {
/* See Python/ceval.c for comments explaining most fields */
+ struct _ts *prev;
struct _ts *next;
PyInterpreterState *interp;
@@ -99,16 +100,6 @@ typedef struct _ts {
PyObject *dict; /* Stores per-thread state */
- /* XXX doesn't mean anything anymore (the comment below is obsolete)
- => deprecate or remove? */
- /* tick_counter is incremented whenever the check_interval ticker
- * reaches zero. The purpose is to give a useful measure of the number
- * of interpreted bytecode instructions in a given thread. This
- * extremely lightweight statistic collector may be of interest to
- * profilers (like psyco.jit()), although nothing in the core uses it.
- */
- int tick_counter;
-
int gilstate_counter;
PyObject *async_exc; /* Asynchronous exception to raise */
@@ -117,6 +108,32 @@ typedef struct _ts {
int trash_delete_nesting;
PyObject *trash_delete_later;
+ /* Called when a thread state is deleted normally, but not when it
+ * is destroyed after fork().
+ * Pain: to prevent rare but fatal shutdown errors (issue 18808),
+ * Thread.join() must wait for the join'ed thread's tstate to be unlinked
+ * from the tstate chain. That happens at the end of a thread's life,
+ * in pystate.c.
+ * The obvious way doesn't quite work: create a lock which the tstate
+ * unlinking code releases, and have Thread.join() wait to acquire that
+ * lock. The problem is that we _are_ at the end of the thread's life:
+ * if the thread holds the last reference to the lock, decref'ing the
+ * lock will delete the lock, and that may trigger arbitrary Python code
+ * if there's a weakref, with a callback, to the lock. But by this time
+ * _PyThreadState_Current is already NULL, so only the simplest of C code
+ * can be allowed to run (in particular it must not be possible to
+ * release the GIL).
+ * So instead of holding the lock directly, the tstate holds a weakref to
+ * the lock: that's the value of on_delete_data below. Decref'ing a
+ * weakref is harmless.
+ * on_delete points to _threadmodule.c's static release_sentinel() function.
+ * After the tstate is unlinked, release_sentinel is called with the
+ * weakref-to-lock (on_delete_data) argument, and release_sentinel releases
+ * the indirectly held lock.
+ */
+ void (*on_delete)(void *);
+ void *on_delete_data;
+
/* XXX signal handlers should also be here */
} PyThreadState;
@@ -133,12 +150,16 @@ PyAPI_FUNC(int) PyState_AddModule(PyObject*, struct PyModuleDef*);
PyAPI_FUNC(int) PyState_RemoveModule(struct PyModuleDef*);
#endif
PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyState_ClearModules(void);
+#endif
PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *);
PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *);
PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *);
PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *);
PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
#ifdef WITH_THREAD
PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void);
PyAPI_FUNC(void) _PyGILState_Reinit(void);
@@ -212,6 +233,13 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE);
*/
PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void);
+/* Helper/diagnostic function - return 1 if the current thread
+ * currently holds the GIL, 0 otherwise
+ */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) PyGILState_Check(void);
+#endif
+
#endif /* #ifdef WITH_THREAD */
/* The implementation of sys._current_frames() Returns a dict mapping
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index e8a582d..2fc5578 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -28,6 +28,14 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
+#ifndef Py_LIMITED_API
+/* Only used by applications that embed the interpreter and need to
+ * override the standard encoding determination mechanism
+ */
+PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
+ const char *errors);
+#endif
+
PyAPI_FUNC(void) Py_Initialize(void);
PyAPI_FUNC(void) Py_InitializeEx(int);
#ifndef Py_LIMITED_API
@@ -55,6 +63,10 @@ PyAPI_FUNC(int) PyRun_InteractiveOneFlags(
FILE *fp,
const char *filename, /* decoded from the filesystem encoding */
PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveOneObject(
+ FILE *fp,
+ PyObject *filename,
+ PyCompilerFlags *flags);
PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(
FILE *fp,
const char *filename, /* decoded from the filesystem encoding */
@@ -66,6 +78,12 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
int start,
PyCompilerFlags *flags,
PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject(
+ const char *s,
+ PyObject *filename,
+ int start,
+ PyCompilerFlags *flags,
+ PyArena *arena);
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
FILE *fp,
const char *filename, /* decoded from the filesystem encoding */
@@ -76,6 +94,16 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
PyCompilerFlags *flags,
int *errcode,
PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject(
+ FILE *fp,
+ PyObject *filename,
+ const char* enc,
+ int start,
+ char *ps1,
+ char *ps2,
+ PyCompilerFlags *flags,
+ int *errcode,
+ PyArena *arena);
#endif
#ifndef PyParser_SimpleParseString
@@ -117,11 +145,22 @@ PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
int start,
PyCompilerFlags *flags,
int optimize);
+PyAPI_FUNC(PyObject *) Py_CompileStringObject(
+ const char *str,
+ PyObject *filename, int start,
+ PyCompilerFlags *flags,
+ int optimize);
#endif
PyAPI_FUNC(struct symtable *) Py_SymtableString(
const char *str,
const char *filename, /* decoded from the filesystem encoding */
int start);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
+ const char *str,
+ PyObject *filename,
+ int start);
+#endif
PyAPI_FUNC(void) PyErr_Print(void);
PyAPI_FUNC(void) PyErr_PrintEx(int);
@@ -197,7 +236,7 @@ PyAPI_FUNC(void) _PyImport_Init(void);
PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
PyAPI_FUNC(void) _PyImportHooks_Init(void);
PyAPI_FUNC(int) _PyFrame_Init(void);
-PyAPI_FUNC(void) _PyFloat_Init(void);
+PyAPI_FUNC(int) _PyFloat_Init(void);
PyAPI_FUNC(int) PyByteArray_Init(void);
PyAPI_FUNC(void) _PyRandom_Init(void);
#endif
@@ -217,19 +256,21 @@ PyAPI_FUNC(void) PyBytes_Fini(void);
PyAPI_FUNC(void) PyByteArray_Fini(void);
PyAPI_FUNC(void) PyFloat_Fini(void);
PyAPI_FUNC(void) PyOS_FiniInterrupts(void);
+PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
PyAPI_FUNC(void) _PyGC_Fini(void);
PyAPI_FUNC(void) PySlice_Fini(void);
PyAPI_FUNC(void) _PyType_Fini(void);
+PyAPI_FUNC(void) _PyRandom_Fini(void);
PyAPI_DATA(PyThreadState *) _Py_Finalizing;
#endif
/* Stuff with no proper home (yet) */
#ifndef Py_LIMITED_API
-PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, char *);
+PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *);
#endif
PyAPI_DATA(int) (*PyOS_InputHook)(void);
-PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, char *);
+PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *);
#ifndef Py_LIMITED_API
PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState;
#endif
diff --git a/Include/pytime.h b/Include/pytime.h
index 52902f5..b0fc6d0 100644
--- a/Include/pytime.h
+++ b/Include/pytime.h
@@ -53,10 +53,19 @@ do { \
(tv_end.tv_usec - tv_start.tv_usec) * 0.000001)
#ifndef Py_LIMITED_API
+
+typedef enum {
+ /* Round towards zero. */
+ _PyTime_ROUND_DOWN=0,
+ /* Round away from zero. */
+ _PyTime_ROUND_UP
+} _PyTime_round_t;
+
/* Convert a number of seconds, int or float, to time_t. */
PyAPI_FUNC(int) _PyTime_ObjectToTime_t(
PyObject *obj,
- time_t *sec);
+ time_t *sec,
+ _PyTime_round_t);
/* Convert a time_t to a PyLong. */
PyAPI_FUNC(PyObject *) _PyLong_FromTime_t(
@@ -72,7 +81,8 @@ PyAPI_FUNC(time_t) _PyLong_AsTime_t(
PyAPI_FUNC(int) _PyTime_ObjectToTimeval(
PyObject *obj,
time_t *sec,
- long *usec);
+ long *usec,
+ _PyTime_round_t);
/* Convert a number of seconds, int or float, to a timespec structure.
nsec is in the range [0; 999999999] and rounded towards zero.
@@ -80,7 +90,8 @@ PyAPI_FUNC(int) _PyTime_ObjectToTimeval(
PyAPI_FUNC(int) _PyTime_ObjectToTimespec(
PyObject *obj,
time_t *sec,
- long *nsec);
+ long *nsec,
+ _PyTime_round_t);
#endif
/* Dummy to force linking. */
diff --git a/Include/setobject.h b/Include/setobject.h
index a14874b..ae3f556 100644
--- a/Include/setobject.h
+++ b/Include/setobject.h
@@ -23,8 +23,8 @@ no meaning otherwise.
typedef struct {
/* Cached hash code of the key. */
- Py_hash_t hash;
PyObject *key;
+ Py_hash_t hash;
} setentry;
@@ -51,9 +51,9 @@ struct _setobject {
*/
setentry *table;
setentry *(*lookup)(PySetObject *so, PyObject *key, Py_hash_t hash);
+ Py_hash_t hash; /* only used by frozenset objects */
setentry smalltable[PySet_MINSIZE];
- Py_hash_t hash; /* only used by frozenset objects */
PyObject *weakreflist; /* List of weak references */
};
#endif /* Py_LIMITED_API */
@@ -61,6 +61,10 @@ struct _setobject {
PyAPI_DATA(PyTypeObject) PySet_Type;
PyAPI_DATA(PyTypeObject) PyFrozenSet_Type;
PyAPI_DATA(PyTypeObject) PySetIter_Type;
+#ifndef Py_LIMITED_API
+PyAPI_DATA(PyObject *) _PySet_Dummy;
+#endif
+
/* Invariants for frozensets:
* data is immutable.
@@ -101,7 +105,6 @@ PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set);
PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
PyAPI_FUNC(int) PySet_ClearFreeList(void);
-PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out);
#endif
#ifdef __cplusplus
diff --git a/Include/sliceobject.h b/Include/sliceobject.h
index 8bec179..f7ee90c 100644
--- a/Include/sliceobject.h
+++ b/Include/sliceobject.h
@@ -34,6 +34,9 @@ PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop,
PyObject* step);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _PySlice_FromIndices(Py_ssize_t start, Py_ssize_t stop);
+PyAPI_FUNC(int) _PySlice_GetLongIndices(PySliceObject *self, PyObject *length,
+ PyObject **start_ptr, PyObject **stop_ptr,
+ PyObject **step_ptr);
#endif
PyAPI_FUNC(int) PySlice_GetIndices(PyObject *r, Py_ssize_t length,
Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step);
diff --git a/Include/structseq.h b/Include/structseq.h
index 30c52ac..af22716 100644
--- a/Include/structseq.h
+++ b/Include/structseq.h
@@ -24,6 +24,8 @@ extern char* PyStructSequence_UnnamedField;
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type,
PyStructSequence_Desc *desc);
+PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type,
+ PyStructSequence_Desc *desc);
#endif
PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc);
diff --git a/Include/symtable.h b/Include/symtable.h
index 6ed3a2b..1cfd884 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -16,7 +16,7 @@ typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
struct _symtable_entry;
struct symtable {
- const char *st_filename; /* name of file being compiled,
+ PyObject *st_filename; /* name of file being compiled,
decoded from the filesystem encoding */
struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* symbol table entry for module */
@@ -41,6 +41,7 @@ typedef struct _symtable_entry {
PyObject *ste_name; /* string: name of current block */
PyObject *ste_varnames; /* list of function parameters */
PyObject *ste_children; /* list of child blocks */
+ PyObject *ste_directives;/* locations of global and nonlocal statements */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
int ste_nested; /* true if block is nested */
@@ -52,6 +53,9 @@ typedef struct _symtable_entry {
unsigned ste_varkeywords : 1; /* true if block has varkeywords */
unsigned ste_returns_value : 1; /* true if namespace uses return with
an argument */
+ unsigned ste_needs_class_closure : 1; /* for class scopes, true if a
+ closure over __class__
+ should be created */
int ste_lineno; /* first line of block */
int ste_col_offset; /* offset of first line of block */
int ste_opt_lineno; /* lineno of last exec or import * */
@@ -70,6 +74,10 @@ PyAPI_FUNC(struct symtable *) PySymtable_Build(
mod_ty mod,
const char *filename, /* decoded from the filesystem encoding */
PyFutureFeatures *future);
+PyAPI_FUNC(struct symtable *) PySymtable_BuildObject(
+ mod_ty mod,
+ PyObject *filename,
+ PyFutureFeatures *future);
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
diff --git a/Include/sysmodule.h b/Include/sysmodule.h
index 0cabf6f..79e52a3 100644
--- a/Include/sysmodule.h
+++ b/Include/sysmodule.h
@@ -8,7 +8,12 @@ extern "C" {
#endif
PyAPI_FUNC(PyObject *) PySys_GetObject(const char *);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
+#endif
PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *);
+PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
+
PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **);
PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int);
PyAPI_FUNC(void) PySys_SetPath(const wchar_t *);
diff --git a/Include/token.h b/Include/token.h
index f7f6504..905022b 100644
--- a/Include/token.h
+++ b/Include/token.h
@@ -75,7 +75,7 @@ extern "C" {
#define ISEOF(x) ((x) == ENDMARKER)
-PyAPI_DATA(char *) _PyParser_TokenNames[]; /* Token names */
+PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */
PyAPI_FUNC(int) PyToken_OneChar(int);
PyAPI_FUNC(int) PyToken_TwoChars(int, int);
PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int);
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index a8f5b5d..faa53d6 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -180,9 +180,9 @@ typedef unsigned char Py_UCS1;
} while (0)
/* macros to work with surrogates */
-#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDFFF)
-#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF)
-#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF)
+#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF)
+#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF)
+#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= (ch) && (ch) <= 0xDFFF)
/* Join two surrogate characters and return a single Py_UCS4 value. */
#define Py_UNICODE_JOIN_SURROGATES(high, low) \
(((((Py_UCS4)(high) & 0x03FF) << 10) | \
@@ -343,6 +343,9 @@ typedef struct {
the data pointer is filled out. The bit is redundant, and helps
to minimize the test in PyUnicode_IS_READY(). */
unsigned int ready:1;
+ /* Padding to ensure that PyUnicode_DATA() is always aligned to
+ 4 bytes (see issue #19537 on m68k). */
+ unsigned int :24;
} state;
wchar_t *wstr; /* wchar_t representation (null-terminated) */
} PyASCIIObject;
@@ -859,7 +862,7 @@ PyAPI_FUNC(int) PyUnicode_Resize(
*/
PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
- register PyObject *obj, /* Object */
+ PyObject *obj, /* Object */
const char *encoding, /* encoding */
const char *errors /* error handling */
);
@@ -878,7 +881,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromEncodedObject(
*/
PyAPI_FUNC(PyObject*) PyUnicode_FromObject(
- register PyObject *obj /* Object */
+ PyObject *obj /* Object */
);
PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(
@@ -898,22 +901,28 @@ typedef struct {
Py_UCS4 maxchar;
Py_ssize_t size;
Py_ssize_t pos;
- /* minimum length of the buffer when overallocation is enabled,
- see _PyUnicodeWriter_Init() */
+
+ /* minimum number of allocated characters (default: 0) */
Py_ssize_t min_length;
+
+ /* minimum character (default: 127, ASCII) */
+ Py_UCS4 min_char;
+
+ /* If non-zero, overallocate the buffer by 25% (default: 0). */
unsigned char overallocate;
+
/* If readonly is 1, buffer is a shared string (cannot be modified)
and size is set to 0. */
unsigned char readonly;
} _PyUnicodeWriter ;
/* Initialize a Unicode writer.
-
- If min_length is greater than zero, _PyUnicodeWriter_Prepare()
- overallocates the buffer and min_length is the minimum length in characters
- of the buffer. */
+ *
+ * By default, the minimum buffer size is 0 character and overallocation is
+ * disabled. Set min_length, min_char and overallocate attributes to control
+ * the allocation of the buffer. */
PyAPI_FUNC(void)
-_PyUnicodeWriter_Init(_PyUnicodeWriter *writer, Py_ssize_t min_length);
+_PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
/* Prepare the buffer to write 'length' characters
with the specified maximum character.
@@ -933,12 +942,52 @@ PyAPI_FUNC(int)
_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
Py_ssize_t length, Py_UCS4 maxchar);
+/* Append a Unicode character.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer,
+ Py_UCS4 ch
+ );
+
+/* Append a Unicode string.
+ Return 0 on success, raise an exception and return -1 on error. */
PyAPI_FUNC(int)
-_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str);
+_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer,
+ PyObject *str /* Unicode string */
+ );
+/* Append a substring of a Unicode string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer,
+ PyObject *str, /* Unicode string */
+ Py_ssize_t start,
+ Py_ssize_t end
+ );
+
+/* Append a ASCII-encoded byte string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
+ const char *str, /* ASCII-encoded byte string */
+ Py_ssize_t len /* number of bytes, or -1 if unknown */
+ );
+
+/* Append a latin1-encoded byte string.
+ Return 0 on success, raise an exception and return -1 on error. */
+PyAPI_FUNC(int)
+_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
+ const char *str, /* latin1-encoded byte string */
+ Py_ssize_t len /* length in bytes */
+ );
+
+/* Get the value of the writer as an Unicode string. Clear the
+ buffer of the writer. Raise an exception and return NULL
+ on error. */
PyAPI_FUNC(PyObject *)
_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer);
+/* Deallocate memory of a writer (clear its internal buffer). */
PyAPI_FUNC(void)
_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer);
#endif
@@ -977,7 +1026,7 @@ PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void);
The buffer is copied into the new object. */
PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
- register const wchar_t *w, /* wchar_t buffer */
+ const wchar_t *w, /* wchar_t buffer */
Py_ssize_t size /* size of buffer */
);
@@ -995,7 +1044,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromWideChar(
PyAPI_FUNC(Py_ssize_t) PyUnicode_AsWideChar(
PyObject *unicode, /* Unicode object */
- register wchar_t *w, /* wchar_t buffer */
+ wchar_t *w, /* wchar_t buffer */
Py_ssize_t size /* size of buffer */
);
@@ -1726,7 +1775,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocale(
/* Encode a Unicode object to the current locale encoding. The encoder is
strict is *surrogateescape* is equal to zero, otherwise the
"surrogateescape" error handler is used. Return a bytes object. The string
- cannot contain embedded null characters.. */
+ cannot contain embedded null characters. */
PyAPI_FUNC(PyObject*) PyUnicode_EncodeLocale(
PyObject *unicode,
@@ -1950,13 +1999,21 @@ PyAPI_FUNC(PyObject *) PyUnicode_Replace(
);
/* Compare two strings and return -1, 0, 1 for less than, equal,
- greater than resp. */
+ greater than resp.
+ Raise an exception and return -1 on error. */
PyAPI_FUNC(int) PyUnicode_Compare(
PyObject *left, /* Left string */
PyObject *right /* Right string */
);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyUnicode_CompareWithId(
+ PyObject *left, /* Left string */
+ _Py_Identifier *right /* Right identifier */
+ );
+#endif
+
PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString(
PyObject *left,
const char *right /* ASCII-encoded string */
diff --git a/Include/warnings.h b/Include/warnings.h
index b7db681..effb9fad 100644
--- a/Include/warnings.h
+++ b/Include/warnings.h
@@ -17,6 +17,15 @@ PyAPI_FUNC(int) PyErr_WarnFormat(
Py_ssize_t stack_level,
const char *format, /* ASCII-encoded string */
...);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) PyErr_WarnExplicitObject(
+ PyObject *category,
+ PyObject *message,
+ PyObject *filename,
+ int lineno,
+ PyObject *module,
+ PyObject *registry);
+#endif
PyAPI_FUNC(int) PyErr_WarnExplicit(
PyObject *category,
const char *message, /* UTF-8 encoded string */
@@ -25,6 +34,14 @@ PyAPI_FUNC(int) PyErr_WarnExplicit(
const char *module, /* UTF-8 encoded string */
PyObject *registry);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int)
+PyErr_WarnExplicitFormat(PyObject *category,
+ const char *filename, int lineno,
+ const char *module, PyObject *registry,
+ const char *format, ...);
+#endif
+
/* DEPRECATED: Use PyErr_WarnEx() instead. */
#ifndef Py_LIMITED_API
#define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1)
diff --git a/LICENSE b/LICENSE
index f9ca2c9..583f9f6 100644
--- a/LICENSE
+++ b/LICENSE
@@ -74,8 +74,8 @@ analyze, test, perform and/or display publicly, prepare derivative works,
distribute, and otherwise use Python alone or in any derivative version,
provided, however, that PSF's License Agreement and PSF's notice of copyright,
i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
-2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are retained
-in Python alone or in any derivative version prepared by Licensee.
+2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are
+retained in Python alone or in any derivative version prepared by Licensee.
3. In the event Licensee prepares a derivative work that is based on
or incorporates Python or any part thereof, and wants to make
diff --git a/Lib/_bootlocale.py b/Lib/_bootlocale.py
new file mode 100644
index 0000000..4bccac1
--- /dev/null
+++ b/Lib/_bootlocale.py
@@ -0,0 +1,34 @@
+"""A minimal subset of the locale module used at interpreter startup
+(imported by the _io module), in order to reduce startup time.
+
+Don't import directly from third-party code; use the `locale` module instead!
+"""
+
+import sys
+import _locale
+
+if sys.platform.startswith("win"):
+ def getpreferredencoding(do_setlocale=True):
+ return _locale._getdefaultlocale()[1]
+else:
+ try:
+ _locale.CODESET
+ except AttributeError:
+ def getpreferredencoding(do_setlocale=True):
+ # This path for legacy systems needs the more complex
+ # getdefaultlocale() function, import the full locale module.
+ import locale
+ return locale.getpreferredencoding(do_setlocale)
+ else:
+ def getpreferredencoding(do_setlocale=True):
+ assert not do_setlocale
+ result = _locale.nl_langinfo(_locale.CODESET)
+ if not result and sys.platform == 'darwin':
+ # nl_langinfo can return an empty string
+ # when the setting has an invalid value.
+ # Default to UTF-8 in that case because
+ # UTF-8 is the default charset on OSX and
+ # returning nothing will crash the
+ # interpreter.
+ result = 'UTF-8'
+ return result
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
new file mode 100644
index 0000000..faa1ff2
--- /dev/null
+++ b/Lib/_collections_abc.py
@@ -0,0 +1,734 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
+
+Unit tests are in test_collections.
+"""
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+__all__ = ["Hashable", "Iterable", "Iterator",
+ "Sized", "Container", "Callable",
+ "Set", "MutableSet",
+ "Mapping", "MutableMapping",
+ "MappingView", "KeysView", "ItemsView", "ValuesView",
+ "Sequence", "MutableSequence",
+ "ByteString",
+ ]
+
+# This module has been renamed from collections.abc to _collections_abc to
+# speed up interpreter startup. Some of the types such as MutableMapping are
+# required early but collections module imports a lot of other modules.
+# See issue #19218
+__name__ = "collections.abc"
+
+# Private list of types that we want to register with the various ABCs
+# so that they will pass tests like:
+# it = iter(somebytearray)
+# assert isinstance(it, Iterable)
+# Note: in other implementations, these types many not be distinct
+# and they make have their own implementation specific types that
+# are not included on this list.
+bytes_iterator = type(iter(b''))
+bytearray_iterator = type(iter(bytearray()))
+#callable_iterator = ???
+dict_keyiterator = type(iter({}.keys()))
+dict_valueiterator = type(iter({}.values()))
+dict_itemiterator = type(iter({}.items()))
+list_iterator = type(iter([]))
+list_reverseiterator = type(iter(reversed([])))
+range_iterator = type(iter(range(0)))
+set_iterator = type(iter(set()))
+str_iterator = type(iter(""))
+tuple_iterator = type(iter(()))
+zip_iterator = type(iter(zip()))
+## views ##
+dict_keys = type({}.keys())
+dict_values = type({}.values())
+dict_items = type({}.items())
+## misc ##
+mappingproxy = type(type.__dict__)
+
+
+### ONE-TRICK PONIES ###
+
+class Hashable(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __hash__(self):
+ return 0
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Hashable:
+ for B in C.__mro__:
+ if "__hash__" in B.__dict__:
+ if B.__dict__["__hash__"]:
+ return True
+ break
+ return NotImplemented
+
+
+class Iterable(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __iter__(self):
+ while False:
+ yield None
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Iterable:
+ if any("__iter__" in B.__dict__ for B in C.__mro__):
+ return True
+ return NotImplemented
+
+
+class Iterator(Iterable):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __next__(self):
+ 'Return the next item from the iterator. When exhausted, raise StopIteration'
+ raise StopIteration
+
+ def __iter__(self):
+ return self
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Iterator:
+ if (any("__next__" in B.__dict__ for B in C.__mro__) and
+ any("__iter__" in B.__dict__ for B in C.__mro__)):
+ return True
+ return NotImplemented
+
+Iterator.register(bytes_iterator)
+Iterator.register(bytearray_iterator)
+#Iterator.register(callable_iterator)
+Iterator.register(dict_keyiterator)
+Iterator.register(dict_valueiterator)
+Iterator.register(dict_itemiterator)
+Iterator.register(list_iterator)
+Iterator.register(list_reverseiterator)
+Iterator.register(range_iterator)
+Iterator.register(set_iterator)
+Iterator.register(str_iterator)
+Iterator.register(tuple_iterator)
+Iterator.register(zip_iterator)
+
+class Sized(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __len__(self):
+ return 0
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Sized:
+ if any("__len__" in B.__dict__ for B in C.__mro__):
+ return True
+ return NotImplemented
+
+
+class Container(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __contains__(self, x):
+ return False
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Container:
+ if any("__contains__" in B.__dict__ for B in C.__mro__):
+ return True
+ return NotImplemented
+
+
+class Callable(metaclass=ABCMeta):
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __call__(self, *args, **kwds):
+ return False
+
+ @classmethod
+ def __subclasshook__(cls, C):
+ if cls is Callable:
+ if any("__call__" in B.__dict__ for B in C.__mro__):
+ return True
+ return NotImplemented
+
+
+### SETS ###
+
+
+class Set(Sized, Iterable, Container):
+
+ """A set is a finite, iterable container.
+
+ This class provides concrete generic implementations of all
+ methods except for __contains__, __iter__ and __len__.
+
+ To override the comparisons (presumably for speed, as the
+ semantics are fixed), all you have to do is redefine __le__ and
+ then the other operations will automatically follow suit.
+ """
+
+ __slots__ = ()
+
+ def __le__(self, other):
+ if not isinstance(other, Set):
+ return NotImplemented
+ if len(self) > len(other):
+ return False
+ for elem in self:
+ if elem not in other:
+ return False
+ return True
+
+ def __lt__(self, other):
+ if not isinstance(other, Set):
+ return NotImplemented
+ return len(self) < len(other) and self.__le__(other)
+
+ def __gt__(self, other):
+ if not isinstance(other, Set):
+ return NotImplemented
+ return other.__lt__(self)
+
+ def __ge__(self, other):
+ if not isinstance(other, Set):
+ return NotImplemented
+ return other.__le__(self)
+
+ def __eq__(self, other):
+ if not isinstance(other, Set):
+ return NotImplemented
+ return len(self) == len(other) and self.__le__(other)
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ @classmethod
+ def _from_iterable(cls, it):
+ '''Construct an instance of the class from any iterable input.
+
+ Must override this method if the class constructor signature
+ does not accept an iterable for an input.
+ '''
+ return cls(it)
+
+ def __and__(self, other):
+ if not isinstance(other, Iterable):
+ return NotImplemented
+ return self._from_iterable(value for value in other if value in self)
+
+ def isdisjoint(self, other):
+ 'Return True if two sets have a null intersection.'
+ for value in other:
+ if value in self:
+ return False
+ return True
+
+ def __or__(self, other):
+ if not isinstance(other, Iterable):
+ return NotImplemented
+ chain = (e for s in (self, other) for e in s)
+ return self._from_iterable(chain)
+
+ def __sub__(self, other):
+ if not isinstance(other, Set):
+ if not isinstance(other, Iterable):
+ return NotImplemented
+ other = self._from_iterable(other)
+ return self._from_iterable(value for value in self
+ if value not in other)
+
+ def __xor__(self, other):
+ if not isinstance(other, Set):
+ if not isinstance(other, Iterable):
+ return NotImplemented
+ other = self._from_iterable(other)
+ return (self - other) | (other - self)
+
+ def _hash(self):
+ """Compute the hash value of a set.
+
+ Note that we don't define __hash__: not all sets are hashable.
+ But if you define a hashable set type, its __hash__ should
+ call this function.
+
+ This must be compatible __eq__.
+
+ All sets ought to compare equal if they contain the same
+ elements, regardless of how they are implemented, and
+ regardless of the order of the elements; so there's not much
+ freedom for __eq__ or __hash__. We match the algorithm used
+ by the built-in frozenset type.
+ """
+ MAX = sys.maxsize
+ MASK = 2 * MAX + 1
+ n = len(self)
+ h = 1927868237 * (n + 1)
+ h &= MASK
+ for x in self:
+ hx = hash(x)
+ h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
+ h &= MASK
+ h = h * 69069 + 907133923
+ h &= MASK
+ if h > MAX:
+ h -= MASK + 1
+ if h == -1:
+ h = 590923713
+ return h
+
+Set.register(frozenset)
+
+
+class MutableSet(Set):
+ """A mutable set is a finite, iterable container.
+
+ This class provides concrete generic implementations of all
+ methods except for __contains__, __iter__, __len__,
+ add(), and discard().
+
+ To override the comparisons (presumably for speed, as the
+ semantics are fixed), all you have to do is redefine __le__ and
+ then the other operations will automatically follow suit.
+ """
+
+ __slots__ = ()
+
+ @abstractmethod
+ def add(self, value):
+ """Add an element."""
+ raise NotImplementedError
+
+ @abstractmethod
+ def discard(self, value):
+ """Remove an element. Do not raise an exception if absent."""
+ raise NotImplementedError
+
+ def remove(self, value):
+ """Remove an element. If not a member, raise a KeyError."""
+ if value not in self:
+ raise KeyError(value)
+ self.discard(value)
+
+ def pop(self):
+ """Return the popped value. Raise KeyError if empty."""
+ it = iter(self)
+ try:
+ value = next(it)
+ except StopIteration:
+ raise KeyError
+ self.discard(value)
+ return value
+
+ def clear(self):
+ """This is slow (creates N new iterators!) but effective."""
+ try:
+ while True:
+ self.pop()
+ except KeyError:
+ pass
+
+ def __ior__(self, it):
+ for value in it:
+ self.add(value)
+ return self
+
+ def __iand__(self, it):
+ for value in (self - it):
+ self.discard(value)
+ return self
+
+ def __ixor__(self, it):
+ if it is self:
+ self.clear()
+ else:
+ if not isinstance(it, Set):
+ it = self._from_iterable(it)
+ for value in it:
+ if value in self:
+ self.discard(value)
+ else:
+ self.add(value)
+ return self
+
+ def __isub__(self, it):
+ if it is self:
+ self.clear()
+ else:
+ for value in it:
+ self.discard(value)
+ return self
+
+MutableSet.register(set)
+
+
+### MAPPINGS ###
+
+
+class Mapping(Sized, Iterable, Container):
+
+ __slots__ = ()
+
+ """A Mapping is a generic container for associating key/value
+ pairs.
+
+ This class provides concrete generic implementations of all
+ methods except for __getitem__, __iter__, and __len__.
+
+ """
+
+ @abstractmethod
+ def __getitem__(self, key):
+ raise KeyError
+
+ def get(self, key, default=None):
+ 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.'
+ try:
+ return self[key]
+ except KeyError:
+ return default
+
+ def __contains__(self, key):
+ try:
+ self[key]
+ except KeyError:
+ return False
+ else:
+ return True
+
+ def keys(self):
+ "D.keys() -> a set-like object providing a view on D's keys"
+ return KeysView(self)
+
+ def items(self):
+ "D.items() -> a set-like object providing a view on D's items"
+ return ItemsView(self)
+
+ def values(self):
+ "D.values() -> an object providing a view on D's values"
+ return ValuesView(self)
+
+ def __eq__(self, other):
+ if not isinstance(other, Mapping):
+ return NotImplemented
+ return dict(self.items()) == dict(other.items())
+
+ def __ne__(self, other):
+ return not (self == other)
+
+Mapping.register(mappingproxy)
+
+
+class MappingView(Sized):
+
+ def __init__(self, mapping):
+ self._mapping = mapping
+
+ def __len__(self):
+ return len(self._mapping)
+
+ def __repr__(self):
+ return '{0.__class__.__name__}({0._mapping!r})'.format(self)
+
+
+class KeysView(MappingView, Set):
+
+ @classmethod
+ def _from_iterable(self, it):
+ return set(it)
+
+ def __contains__(self, key):
+ return key in self._mapping
+
+ def __iter__(self):
+ yield from self._mapping
+
+KeysView.register(dict_keys)
+
+
+class ItemsView(MappingView, Set):
+
+ @classmethod
+ def _from_iterable(self, it):
+ return set(it)
+
+ def __contains__(self, item):
+ key, value = item
+ try:
+ v = self._mapping[key]
+ except KeyError:
+ return False
+ else:
+ return v == value
+
+ def __iter__(self):
+ for key in self._mapping:
+ yield (key, self._mapping[key])
+
+ItemsView.register(dict_items)
+
+
+class ValuesView(MappingView):
+
+ def __contains__(self, value):
+ for key in self._mapping:
+ if value == self._mapping[key]:
+ return True
+ return False
+
+ def __iter__(self):
+ for key in self._mapping:
+ yield self._mapping[key]
+
+ValuesView.register(dict_values)
+
+
+class MutableMapping(Mapping):
+
+ __slots__ = ()
+
+ """A MutableMapping is a generic container for associating
+ key/value pairs.
+
+ This class provides concrete generic implementations of all
+ methods except for __getitem__, __setitem__, __delitem__,
+ __iter__, and __len__.
+
+ """
+
+ @abstractmethod
+ def __setitem__(self, key, value):
+ raise KeyError
+
+ @abstractmethod
+ def __delitem__(self, key):
+ raise KeyError
+
+ __marker = object()
+
+ def pop(self, key, default=__marker):
+ '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+ If key is not found, d is returned if given, otherwise KeyError is raised.
+ '''
+ try:
+ value = self[key]
+ except KeyError:
+ if default is self.__marker:
+ raise
+ return default
+ else:
+ del self[key]
+ return value
+
+ def popitem(self):
+ '''D.popitem() -> (k, v), remove and return some (key, value) pair
+ as a 2-tuple; but raise KeyError if D is empty.
+ '''
+ try:
+ key = next(iter(self))
+ except StopIteration:
+ raise KeyError
+ value = self[key]
+ del self[key]
+ return key, value
+
+ def clear(self):
+ 'D.clear() -> None. Remove all items from D.'
+ try:
+ while True:
+ self.popitem()
+ except KeyError:
+ pass
+
+ def update(*args, **kwds):
+ ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
+ If E present and has a .keys() method, does: for k in E: D[k] = E[k]
+ If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
+ In either case, this is followed by: for k, v in F.items(): D[k] = v
+ '''
+ if len(args) > 2:
+ raise TypeError("update() takes at most 2 positional "
+ "arguments ({} given)".format(len(args)))
+ elif not args:
+ raise TypeError("update() takes at least 1 argument (0 given)")
+ self = args[0]
+ other = args[1] if len(args) >= 2 else ()
+
+ if isinstance(other, Mapping):
+ for key in other:
+ self[key] = other[key]
+ elif hasattr(other, "keys"):
+ for key in other.keys():
+ self[key] = other[key]
+ else:
+ for key, value in other:
+ self[key] = value
+ for key, value in kwds.items():
+ self[key] = value
+
+ def setdefault(self, key, default=None):
+ 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D'
+ try:
+ return self[key]
+ except KeyError:
+ self[key] = default
+ return default
+
+MutableMapping.register(dict)
+
+
+### SEQUENCES ###
+
+
+class Sequence(Sized, Iterable, Container):
+
+ """All the operations on a read-only sequence.
+
+ Concrete subclasses must override __new__ or __init__,
+ __getitem__, and __len__.
+ """
+
+ __slots__ = ()
+
+ @abstractmethod
+ def __getitem__(self, index):
+ raise IndexError
+
+ def __iter__(self):
+ i = 0
+ try:
+ while True:
+ v = self[i]
+ yield v
+ i += 1
+ except IndexError:
+ return
+
+ def __contains__(self, value):
+ for v in self:
+ if v == value:
+ return True
+ return False
+
+ def __reversed__(self):
+ for i in reversed(range(len(self))):
+ yield self[i]
+
+ def index(self, value):
+ '''S.index(value) -> integer -- return first index of value.
+ Raises ValueError if the value is not present.
+ '''
+ for i, v in enumerate(self):
+ if v == value:
+ return i
+ raise ValueError
+
+ def count(self, value):
+ 'S.count(value) -> integer -- return number of occurrences of value'
+ return sum(1 for v in self if v == value)
+
+Sequence.register(tuple)
+Sequence.register(str)
+Sequence.register(range)
+Sequence.register(memoryview)
+
+
+class ByteString(Sequence):
+
+ """This unifies bytes and bytearray.
+
+ XXX Should add all their methods.
+ """
+
+ __slots__ = ()
+
+ByteString.register(bytes)
+ByteString.register(bytearray)
+
+
+class MutableSequence(Sequence):
+
+ __slots__ = ()
+
+ """All the operations on a read-write sequence.
+
+ Concrete subclasses must provide __new__ or __init__,
+ __getitem__, __setitem__, __delitem__, __len__, and insert().
+
+ """
+
+ @abstractmethod
+ def __setitem__(self, index, value):
+ raise IndexError
+
+ @abstractmethod
+ def __delitem__(self, index):
+ raise IndexError
+
+ @abstractmethod
+ def insert(self, index, value):
+ 'S.insert(index, value) -- insert value before index'
+ raise IndexError
+
+ def append(self, value):
+ 'S.append(value) -- append value to the end of the sequence'
+ self.insert(len(self), value)
+
+ def clear(self):
+ 'S.clear() -> None -- remove all items from S'
+ try:
+ while True:
+ self.pop()
+ except IndexError:
+ pass
+
+ def reverse(self):
+ 'S.reverse() -- reverse *IN PLACE*'
+ n = len(self)
+ for i in range(n//2):
+ self[i], self[n-i-1] = self[n-i-1], self[i]
+
+ def extend(self, values):
+ 'S.extend(iterable) -- extend sequence by appending elements from the iterable'
+ for v in values:
+ self.append(v)
+
+ def pop(self, index=-1):
+ '''S.pop([index]) -> item -- remove and return item at index (default last).
+ Raise IndexError if list is empty or index is out of range.
+ '''
+ v = self[index]
+ del self[index]
+ return v
+
+ def remove(self, value):
+ '''S.remove(value) -- remove first occurrence of value.
+ Raise ValueError if the value is not present.
+ '''
+ del self[self.index(value)]
+
+ def __iadd__(self, values):
+ self.extend(values)
+ return self
+
+MutableSequence.register(list)
+MutableSequence.register(bytearray) # Multiply inheriting, see ByteString
diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
index 13b1f26..b67cfb9 100644
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -81,6 +81,10 @@ def stack_size(size=None):
raise error("setting thread stack size not supported")
return 0
+def _set_sentinel():
+ """Dummy implementation of _thread._set_sentinel()."""
+ return LockType()
+
class LockType(object):
"""Class implementing dummy implementation of _thread.LockType.
diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py
index 50b2d17..40fc78b 100644
--- a/Lib/_osx_support.py
+++ b/Lib/_osx_support.py
@@ -38,7 +38,7 @@ def _find_executable(executable, path=None):
paths = path.split(os.pathsep)
base, ext = os.path.splitext(executable)
- if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'):
+ if (sys.platform == 'win32') and (ext != '.exe'):
executable = executable + '.exe'
if not os.path.isfile(executable):
@@ -94,7 +94,7 @@ def _get_system_version():
_SYSTEM_VERSION = ''
try:
f = open('/System/Library/CoreServices/SystemVersion.plist')
- except IOError:
+ except OSError:
# We're on a plain darwin box, fall back to the default
# behaviour.
pass
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index a0c4b25..b04d23a 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -34,7 +34,7 @@ BlockingIOError = BlockingIOError
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
newline=None, closefd=True, opener=None):
- r"""Open file and return a stream. Raise IOError upon failure.
+ r"""Open file and return a stream. Raise OSError upon failure.
file is either a text or byte string giving the name (and the path
if the file isn't in the current working directory) of the file to
@@ -62,8 +62,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
'b' binary mode
't' text mode (default)
'+' open a disk file for updating (reading and writing)
- 'U' universal newline mode (for backwards compatibility; unneeded
- for new code)
+ 'U' universal newline mode (deprecated)
========= ===============================================================
The default mode is 'rt' (open for reading text). For binary random
@@ -79,6 +78,10 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
returned as strings, the bytes having been first decoded using a
platform-dependent encoding or using the specified encoding if given.
+ 'U' mode is deprecated and will raise an exception in future versions
+ of Python. It has no effect in Python 3. Use newline to control
+ universal newlines mode.
+
buffering is an optional integer used to set the buffering policy.
Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
line buffering (only usable in text mode), and an integer > 1 to indicate
@@ -129,6 +132,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
be kept open when the file is closed. This does not work when a file name is
given and must be True in that case.
+ The newly created file is non-inheritable.
+
A custom opener can be used by passing a callable as *opener*. The
underlying file descriptor for the file object is then obtained by calling
*opener* with (*file*, *flags*). *opener* must return an open file
@@ -172,6 +177,9 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
if "U" in modes:
if creating or writing or appending:
raise ValueError("can't use U and writing mode at once")
+ import warnings
+ warnings.warn("'U' mode is deprecated",
+ DeprecationWarning, 2)
reading = True
if text and binary:
raise ValueError("can't have text and binary mode at once")
@@ -200,7 +208,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
buffering = DEFAULT_BUFFER_SIZE
try:
bs = os.fstat(raw.fileno()).st_blksize
- except (os.error, AttributeError):
+ except (OSError, AttributeError):
pass
else:
if bs > 1:
@@ -254,7 +262,7 @@ class OpenWrapper:
try:
UnsupportedOperation = io.UnsupportedOperation
except AttributeError:
- class UnsupportedOperation(ValueError, IOError):
+ class UnsupportedOperation(ValueError, OSError):
pass
@@ -278,7 +286,7 @@ class IOBase(metaclass=abc.ABCMeta):
readinto) needed. Text I/O classes work with str data.
Note that calling any method (even inquiries) on a closed stream is
- undefined. Implementations may raise IOError in this case.
+ undefined. Implementations may raise OSError in this case.
IOBase (and its subclasses) support the iterator protocol, meaning
that an IOBase object can be iterated over yielding the lines in a
@@ -294,7 +302,7 @@ class IOBase(metaclass=abc.ABCMeta):
### Internal ###
def _unsupported(self, name):
- """Internal: raise an IOError exception for unsupported operations."""
+ """Internal: raise an OSError exception for unsupported operations."""
raise UnsupportedOperation("%s.%s() not supported" %
(self.__class__.__name__, name))
@@ -441,7 +449,7 @@ class IOBase(metaclass=abc.ABCMeta):
def fileno(self):
"""Returns underlying file descriptor (an int) if one exists.
- An IOError is raised if the IO object does not use a file descriptor.
+ An OSError is raised if the IO object does not use a file descriptor.
"""
self._unsupported("fileno")
@@ -455,11 +463,11 @@ class IOBase(metaclass=abc.ABCMeta):
### Readline[s] and writelines ###
- def readline(self, limit=-1):
+ def readline(self, size=-1):
r"""Read and return a line of bytes from the stream.
- If limit is specified, at most limit bytes will be read.
- Limit should be an int.
+ If size is specified, at most size bytes will be read.
+ Size should be an int.
The line terminator is always b'\n' for binary files; for text
files, the newlines argument to open can be used to select the line
@@ -472,18 +480,18 @@ class IOBase(metaclass=abc.ABCMeta):
if not readahead:
return 1
n = (readahead.find(b"\n") + 1) or len(readahead)
- if limit >= 0:
- n = min(n, limit)
+ if size >= 0:
+ n = min(n, size)
return n
else:
def nreadahead():
return 1
- if limit is None:
- limit = -1
- elif not isinstance(limit, int):
- raise TypeError("limit must be an integer")
+ if size is None:
+ size = -1
+ elif not isinstance(size, int):
+ raise TypeError("size must be an integer")
res = bytearray()
- while limit < 0 or len(res) < limit:
+ while size < 0 or len(res) < size:
b = self.read(nreadahead())
if not b:
break
@@ -542,17 +550,17 @@ class RawIOBase(IOBase):
# primitive operation, but that would lead to nasty recursion in case
# a subclass doesn't implement either.)
- def read(self, n=-1):
- """Read and return up to n bytes, where n is an int.
+ def read(self, size=-1):
+ """Read and return up to size bytes, where size is an int.
Returns an empty bytes object on EOF, or None if the object is
set not to block and has no data to read.
"""
- if n is None:
- n = -1
- if n < 0:
+ if size is None:
+ size = -1
+ if size < 0:
return self.readall()
- b = bytearray(n.__index__())
+ b = bytearray(size.__index__())
n = self.readinto(b)
if n is None:
return None
@@ -610,8 +618,8 @@ class BufferedIOBase(IOBase):
implementation, but wrap one.
"""
- def read(self, n=None):
- """Read and return up to n bytes, where n is an int.
+ def read(self, size=None):
+ """Read and return up to size bytes, where size is an int.
If the argument is omitted, None, or negative, reads and
returns all data until EOF.
@@ -630,9 +638,9 @@ class BufferedIOBase(IOBase):
"""
self._unsupported("read")
- def read1(self, n=None):
- """Read up to n bytes with at most one read() system call,
- where n is an int.
+ def read1(self, size=None):
+ """Read up to size bytes with at most one read() system call,
+ where size is an int.
"""
self._unsupported("read1")
@@ -699,13 +707,13 @@ class _BufferedIOMixin(BufferedIOBase):
def seek(self, pos, whence=0):
new_position = self.raw.seek(pos, whence)
if new_position < 0:
- raise IOError("seek() returned an invalid position")
+ raise OSError("seek() returned an invalid position")
return new_position
def tell(self):
pos = self.raw.tell()
if pos < 0:
- raise IOError("tell() returned an invalid position")
+ raise OSError("tell() returned an invalid position")
return pos
def truncate(self, pos=None):
@@ -820,24 +828,24 @@ class BytesIO(BufferedIOBase):
"""
return memoryview(self._buffer)
- def read(self, n=None):
+ def read(self, size=None):
if self.closed:
raise ValueError("read from closed file")
- if n is None:
- n = -1
- if n < 0:
- n = len(self._buffer)
+ if size is None:
+ size = -1
+ if size < 0:
+ size = len(self._buffer)
if len(self._buffer) <= self._pos:
return b""
- newpos = min(len(self._buffer), self._pos + n)
+ newpos = min(len(self._buffer), self._pos + size)
b = self._buffer[self._pos : newpos]
self._pos = newpos
return bytes(b)
- def read1(self, n):
+ def read1(self, size):
"""This is the same as read.
"""
- return self.read(n)
+ return self.read(size)
def write(self, b):
if self.closed:
@@ -927,7 +935,7 @@ class BufferedReader(_BufferedIOMixin):
"""Create a new buffered reader using the given readable raw IO object.
"""
if not raw.readable():
- raise IOError('"raw" argument must be readable.')
+ raise OSError('"raw" argument must be readable.')
_BufferedIOMixin.__init__(self, raw)
if buffer_size <= 0:
@@ -940,18 +948,18 @@ class BufferedReader(_BufferedIOMixin):
self._read_buf = b""
self._read_pos = 0
- def read(self, n=None):
- """Read n bytes.
+ def read(self, size=None):
+ """Read size bytes.
- Returns exactly n bytes of data unless the underlying raw IO
+ Returns exactly size bytes of data unless the underlying raw IO
stream reaches EOF or if the call would block in non-blocking
- mode. If n is negative, read until EOF or until read() would
+ mode. If size is negative, read until EOF or until read() would
block.
"""
- if n is not None and n < -1:
+ if size is not None and size < -1:
raise ValueError("invalid number of bytes to read")
with self._read_lock:
- return self._read_unlocked(n)
+ return self._read_unlocked(size)
def _read_unlocked(self, n=None):
nodata_val = b""
@@ -1011,7 +1019,7 @@ class BufferedReader(_BufferedIOMixin):
self._read_pos = 0
return out[:n] if out else nodata_val
- def peek(self, n=0):
+ def peek(self, size=0):
"""Returns buffered bytes without advancing the position.
The argument indicates a desired minimal number of bytes; we
@@ -1019,7 +1027,7 @@ class BufferedReader(_BufferedIOMixin):
than self.buffer_size.
"""
with self._read_lock:
- return self._peek_unlocked(n)
+ return self._peek_unlocked(size)
def _peek_unlocked(self, n=0):
want = min(n, self.buffer_size)
@@ -1037,18 +1045,18 @@ class BufferedReader(_BufferedIOMixin):
self._read_pos = 0
return self._read_buf[self._read_pos:]
- def read1(self, n):
- """Reads up to n bytes, with at most one read() system call."""
- # Returns up to n bytes. If at least one byte is buffered, we
+ def read1(self, size):
+ """Reads up to size bytes, with at most one read() system call."""
+ # Returns up to size bytes. If at least one byte is buffered, we
# only return buffered bytes. Otherwise, we do one raw read.
- if n < 0:
+ if size < 0:
raise ValueError("number of bytes to read must be positive")
- if n == 0:
+ if size == 0:
return b""
with self._read_lock:
self._peek_unlocked(1)
return self._read_unlocked(
- min(n, len(self._read_buf) - self._read_pos))
+ min(size, len(self._read_buf) - self._read_pos))
def tell(self):
return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
@@ -1074,7 +1082,7 @@ class BufferedWriter(_BufferedIOMixin):
def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
if not raw.writable():
- raise IOError('"raw" argument must be writable.')
+ raise OSError('"raw" argument must be writable.')
_BufferedIOMixin.__init__(self, raw)
if buffer_size <= 0:
@@ -1138,7 +1146,7 @@ class BufferedWriter(_BufferedIOMixin):
errno.EAGAIN,
"write could not complete without blocking", 0)
if n > len(self._write_buf) or n < 0:
- raise IOError("write() returned incorrect number of bytes")
+ raise OSError("write() returned incorrect number of bytes")
del self._write_buf[:n]
def tell(self):
@@ -1174,18 +1182,18 @@ class BufferedRWPair(BufferedIOBase):
The arguments are two RawIO instances.
"""
if not reader.readable():
- raise IOError('"reader" argument must be readable.')
+ raise OSError('"reader" argument must be readable.')
if not writer.writable():
- raise IOError('"writer" argument must be writable.')
+ raise OSError('"writer" argument must be writable.')
self.reader = BufferedReader(reader, buffer_size)
self.writer = BufferedWriter(writer, buffer_size)
- def read(self, n=None):
- if n is None:
- n = -1
- return self.reader.read(n)
+ def read(self, size=None):
+ if size is None:
+ size = -1
+ return self.reader.read(size)
def readinto(self, b):
return self.reader.readinto(b)
@@ -1193,11 +1201,11 @@ class BufferedRWPair(BufferedIOBase):
def write(self, b):
return self.writer.write(b)
- def peek(self, n=0):
- return self.reader.peek(n)
+ def peek(self, size=0):
+ return self.reader.peek(size)
- def read1(self, n):
- return self.reader.read1(n)
+ def read1(self, size):
+ return self.reader.read1(size)
def readable(self):
return self.reader.readable()
@@ -1248,7 +1256,7 @@ class BufferedRandom(BufferedWriter, BufferedReader):
with self._read_lock:
self._reset_read_buf()
if pos < 0:
- raise IOError("seek() returned invalid position")
+ raise OSError("seek() returned invalid position")
return pos
def tell(self):
@@ -1263,23 +1271,23 @@ class BufferedRandom(BufferedWriter, BufferedReader):
# Use seek to flush the read buffer.
return BufferedWriter.truncate(self, pos)
- def read(self, n=None):
- if n is None:
- n = -1
+ def read(self, size=None):
+ if size is None:
+ size = -1
self.flush()
- return BufferedReader.read(self, n)
+ return BufferedReader.read(self, size)
def readinto(self, b):
self.flush()
return BufferedReader.readinto(self, b)
- def peek(self, n=0):
+ def peek(self, size=0):
self.flush()
- return BufferedReader.peek(self, n)
+ return BufferedReader.peek(self, size)
- def read1(self, n):
+ def read1(self, size):
self.flush()
- return BufferedReader.read1(self, n)
+ return BufferedReader.read1(self, size)
def write(self, b):
if self._read_buf:
@@ -1299,11 +1307,11 @@ class TextIOBase(IOBase):
are immutable. There is no public constructor.
"""
- def read(self, n=-1):
- """Read at most n characters from stream, where n is an int.
+ def read(self, size=-1):
+ """Read at most size characters from stream, where size is an int.
- Read from underlying buffer until we have n characters or we hit EOF.
- If n is negative or omitted, read until EOF.
+ Read from underlying buffer until we have size characters or we hit EOF.
+ If size is negative or omitted, read until EOF.
Returns a string.
"""
@@ -1732,7 +1740,7 @@ class TextIOWrapper(TextIOBase):
if not self._seekable:
raise UnsupportedOperation("underlying stream is not seekable")
if not self._telling:
- raise IOError("telling position disabled by next() call")
+ raise OSError("telling position disabled by next() call")
self.flush()
position = self.buffer.tell()
decoder = self._decoder
@@ -1819,7 +1827,7 @@ class TextIOWrapper(TextIOBase):
chars_decoded += len(decoder.decode(b'', final=True))
need_eof = 1
if chars_decoded < chars_to_skip:
- raise IOError("can't reconstruct logical file position")
+ raise OSError("can't reconstruct logical file position")
# The returned cookie corresponds to the last safe start point.
return self._pack_cookie(
@@ -1896,7 +1904,7 @@ class TextIOWrapper(TextIOBase):
# Skip chars_to_skip of the decoded characters.
if len(self._decoded_chars) < chars_to_skip:
- raise IOError("can't restore logical file position")
+ raise OSError("can't restore logical file position")
self._decoded_chars_used = chars_to_skip
# Finally, reset the encoder (merely useful for proper BOM handling)
@@ -1912,16 +1920,16 @@ class TextIOWrapper(TextIOBase):
encoder.reset()
return cookie
- def read(self, n=None):
+ def read(self, size=None):
self._checkReadable()
- if n is None:
- n = -1
+ if size is None:
+ size = -1
decoder = self._decoder or self._get_decoder()
try:
- n.__index__
+ size.__index__
except AttributeError as err:
raise TypeError("an integer is required") from err
- if n < 0:
+ if size < 0:
# Read everything.
result = (self._get_decoded_chars() +
decoder.decode(self.buffer.read(), final=True))
@@ -1929,12 +1937,12 @@ class TextIOWrapper(TextIOBase):
self._snapshot = None
return result
else:
- # Keep reading chunks until we have n characters to return.
+ # Keep reading chunks until we have size characters to return.
eof = False
- result = self._get_decoded_chars(n)
- while len(result) < n and not eof:
+ result = self._get_decoded_chars(size)
+ while len(result) < size and not eof:
eof = not self._read_chunk()
- result += self._get_decoded_chars(n - len(result))
+ result += self._get_decoded_chars(size - len(result))
return result
def __next__(self):
@@ -1946,13 +1954,13 @@ class TextIOWrapper(TextIOBase):
raise StopIteration
return line
- def readline(self, limit=None):
+ def readline(self, size=None):
if self.closed:
raise ValueError("read from closed file")
- if limit is None:
- limit = -1
- elif not isinstance(limit, int):
- raise TypeError("limit must be an integer")
+ if size is None:
+ size = -1
+ elif not isinstance(size, int):
+ raise TypeError("size must be an integer")
# Grab all the decoded text (we will rewind any extra bits later).
line = self._get_decoded_chars()
@@ -2011,8 +2019,8 @@ class TextIOWrapper(TextIOBase):
endpos = pos + len(self._readnl)
break
- if limit >= 0 and len(line) >= limit:
- endpos = limit # reached length limit
+ if size >= 0 and len(line) >= size:
+ endpos = size # reached length size
break
# No line ending seen yet - get more data'
@@ -2027,8 +2035,8 @@ class TextIOWrapper(TextIOBase):
self._snapshot = None
return line
- if limit >= 0 and endpos > limit:
- endpos = limit # don't exceed limit
+ if size >= 0 and endpos > size:
+ endpos = size # don't exceed size
# Rewind _decoded_chars to just after the line ending we found.
self._rewind_decoded_chars(len(line) - endpos)
@@ -2059,7 +2067,6 @@ class StringIO(TextIOWrapper):
if not isinstance(initial_value, str):
raise TypeError("initial_value must be str or None, not {0}"
.format(type(initial_value).__name__))
- initial_value = str(initial_value)
self.write(initial_value)
self.seek(0)
diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py
new file mode 100644
index 0000000..1f21358
--- /dev/null
+++ b/Lib/_sitebuiltins.py
@@ -0,0 +1,99 @@
+"""
+The objects used by the site module to add custom builtins.
+"""
+
+# Those objects are almost immortal and they keep a reference to their module
+# globals. Defining them in the site module would keep too many references
+# alive.
+# Note this means this module should also avoid keep things alive in its
+# globals.
+
+import sys
+
+class Quitter(object):
+ def __init__(self, name, eof):
+ self.name = name
+ self.eof = eof
+ def __repr__(self):
+ return 'Use %s() or %s to exit' % (self.name, self.eof)
+ def __call__(self, code=None):
+ # Shells like IDLE catch the SystemExit, but listen when their
+ # stdin wrapper is closed.
+ try:
+ sys.stdin.close()
+ except:
+ pass
+ raise SystemExit(code)
+
+
+class _Printer(object):
+ """interactive prompt objects for printing the license text, a list of
+ contributors and the copyright notice."""
+
+ MAXLINES = 23
+
+ def __init__(self, name, data, files=(), dirs=()):
+ import os
+ self.__name = name
+ self.__data = data
+ self.__lines = None
+ self.__filenames = [os.path.join(dir, filename)
+ for dir in dirs
+ for filename in files]
+
+ def __setup(self):
+ if self.__lines:
+ return
+ data = None
+ for filename in self.__filenames:
+ try:
+ with open(filename, "r") as fp:
+ data = fp.read()
+ break
+ except OSError:
+ pass
+ if not data:
+ data = self.__data
+ self.__lines = data.split('\n')
+ self.__linecnt = len(self.__lines)
+
+ def __repr__(self):
+ self.__setup()
+ if len(self.__lines) <= self.MAXLINES:
+ return "\n".join(self.__lines)
+ else:
+ return "Type %s() to see the full %s text" % ((self.__name,)*2)
+
+ def __call__(self):
+ self.__setup()
+ prompt = 'Hit Return for more, or q (and Return) to quit: '
+ lineno = 0
+ while 1:
+ try:
+ for i in range(lineno, lineno + self.MAXLINES):
+ print(self.__lines[i])
+ except IndexError:
+ break
+ else:
+ lineno += self.MAXLINES
+ key = None
+ while key is None:
+ key = input(prompt)
+ if key not in ('', 'q'):
+ key = None
+ if key == 'q':
+ break
+
+
+class _Helper(object):
+ """Define the builtin 'help'.
+ This is a wrapper around pydoc.help (with a twist).
+
+ """
+
+ def __repr__(self):
+ return "Type help() for interactive help, " \
+ "or help(object) for help about object."
+ def __call__(self, *args, **kwds):
+ import pydoc
+ return pydoc.help(*args, **kwds)
diff --git a/Lib/_strptime.py b/Lib/_strptime.py
index 9058a69..53bd34b 100644
--- a/Lib/_strptime.py
+++ b/Lib/_strptime.py
@@ -14,14 +14,14 @@ import time
import locale
import calendar
from re import compile as re_compile
-from re import IGNORECASE, ASCII
+from re import IGNORECASE
from re import escape as re_escape
from datetime import (date as datetime_date,
timedelta as datetime_timedelta,
timezone as datetime_timezone)
try:
from _thread import allocate_lock as _thread_allocate_lock
-except:
+except ImportError:
from _dummy_thread import allocate_lock as _thread_allocate_lock
__all__ = []
diff --git a/Lib/abc.py b/Lib/abc.py
index 09778e8..0358a46 100644
--- a/Lib/abc.py
+++ b/Lib/abc.py
@@ -5,6 +5,7 @@
from _weakrefset import WeakSet
+
def abstractmethod(funcobj):
"""A decorator indicating abstract methods.
@@ -124,6 +125,8 @@ class ABCMeta(type):
# A global counter that is incremented each time a class is
# registered as a virtual subclass of anything. It forces the
# negative cache to be cleared before its next use.
+ # Note: this counter is private. Use `abc.get_cache_token()` for
+ # external code.
_abc_invalidation_counter = 0
def __new__(mcls, name, bases, namespace):
@@ -226,3 +229,20 @@ class ABCMeta(type):
# No dice; update negative cache
cls._abc_negative_cache.add(subclass)
return False
+
+
+class ABC(metaclass=ABCMeta):
+ """Helper class that provides a standard way to create an ABC using
+ inheritance.
+ """
+ pass
+
+
+def get_cache_token():
+ """Returns the current ABC cache token.
+
+ The token is an opaque object (supporting equality testing) identifying the
+ current version of the ABC cache for virtual subclasses. The token changes
+ with every call to ``register()`` on any ABC.
+ """
+ return ABCMeta._abc_invalidation_counter
diff --git a/Lib/aifc.py b/Lib/aifc.py
index db0924a..9e64de9 100644
--- a/Lib/aifc.py
+++ b/Lib/aifc.py
@@ -69,7 +69,7 @@ This returns an instance of a class with the following public methods:
getcomptype() -- returns compression type ('NONE' for AIFF files)
getcompname() -- returns human-readable version of
compression type ('not compressed' for AIFF files)
- getparams() -- returns a tuple consisting of all of the
+ getparams() -- returns a namedtuple consisting of all of the
above in the above order
getmarkers() -- get the list of marks in the audio file or None
if there are no marks
@@ -252,6 +252,11 @@ def _write_float(f, x):
_write_ulong(f, lomant)
from chunk import Chunk
+from collections import namedtuple
+
+_aifc_params = namedtuple('_aifc_params',
+ 'nchannels sampwidth framerate nframes comptype compname')
+
class Aifc_read:
# Variables used in this class:
@@ -334,6 +339,12 @@ class Aifc_read:
# else, assume it is an open file object already
self.initfp(f)
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
#
# User visible methods.
#
@@ -372,9 +383,9 @@ class Aifc_read:
## return self._version
def getparams(self):
- return self.getnchannels(), self.getsampwidth(), \
- self.getframerate(), self.getnframes(), \
- self.getcomptype(), self.getcompname()
+ return _aifc_params(self.getnchannels(), self.getsampwidth(),
+ self.getframerate(), self.getnframes(),
+ self.getcomptype(), self.getcompname())
def getmarkers(self):
if len(self._markers) == 0:
@@ -551,6 +562,12 @@ class Aifc_write:
def __del__(self):
self.close()
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
#
# User visible methods.
#
@@ -644,8 +661,8 @@ class Aifc_write:
def getparams(self):
if not self._nchannels or not self._sampwidth or not self._framerate:
raise Error('not all parameters set')
- return self._nchannels, self._sampwidth, self._framerate, \
- self._nframes, self._comptype, self._compname
+ return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
+ self._nframes, self._comptype, self._compname)
def setmark(self, id, pos, name):
if id <= 0:
@@ -675,6 +692,8 @@ class Aifc_write:
return self._nframeswritten
def writeframesraw(self, data):
+ if not isinstance(data, (bytes, bytearray)):
+ data = memoryview(data).cast('B')
self._ensure_header_written(len(data))
nframes = len(data) // (self._sampwidth * self._nchannels)
if self._convert:
@@ -878,8 +897,7 @@ if __name__ == '__main__':
if not sys.argv[1:]:
sys.argv.append('/usr/demos/data/audio/bach.aiff')
fn = sys.argv[1]
- f = open(fn, 'r')
- try:
+ with open(fn, 'r') as f:
print("Reading", fn)
print("nchannels =", f.getnchannels())
print("nframes =", f.getnframes())
@@ -890,16 +908,11 @@ if __name__ == '__main__':
if sys.argv[2:]:
gn = sys.argv[2]
print("Writing", gn)
- g = open(gn, 'w')
- try:
+ with open(gn, 'w') as g:
g.setparams(f.getparams())
while 1:
data = f.readframes(1024)
if not data:
break
g.writeframes(data)
- finally:
- g.close()
print("Done.")
- finally:
- f.close()
diff --git a/Lib/argparse.py b/Lib/argparse.py
index bc2ba13..5ad7e13 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -608,8 +608,7 @@ class HelpFormatter(object):
pass
else:
self._indent()
- for subaction in get_subactions():
- yield subaction
+ yield from get_subactions()
self._dedent()
def _split_lines(self, text, width):
@@ -1040,7 +1039,8 @@ class _VersionAction(Action):
version = parser.version
formatter = parser._get_formatter()
formatter.add_text(version)
- parser.exit(message=formatter.format_help())
+ parser._print_message(formatter.format_help(), _sys.stdout)
+ parser.exit()
class _SubParsersAction(Action):
@@ -1143,11 +1143,17 @@ class FileType(object):
same values as the builtin open() function.
- bufsize -- The file's desired buffer size. Accepts the same values as
the builtin open() function.
+ - encoding -- The file's encoding. Accepts the same values as the
+ builtin open() function.
+ - errors -- A string indicating how encoding and decoding errors are to
+ be handled. Accepts the same value as the builtin open() function.
"""
- def __init__(self, mode='r', bufsize=-1):
+ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None):
self._mode = mode
self._bufsize = bufsize
+ self._encoding = encoding
+ self._errors = errors
def __call__(self, string):
# the special argument "-" means sys.std{in,out}
@@ -1162,14 +1168,18 @@ class FileType(object):
# all other arguments are used as file names
try:
- return open(string, self._mode, self._bufsize)
- except IOError as e:
+ return open(string, self._mode, self._bufsize, self._encoding,
+ self._errors)
+ except OSError as e:
message = _("can't open '%s': %s")
raise ArgumentTypeError(message % (string, e))
def __repr__(self):
args = self._mode, self._bufsize
- args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+ kwargs = [('encoding', self._encoding), ('errors', self._errors)]
+ args_str = ', '.join([repr(arg) for arg in args if arg != -1] +
+ ['%s=%r' % (kw, arg) for kw, arg in kwargs
+ if arg is not None])
return '%s(%s)' % (type(self).__name__, args_str)
# ===========================
@@ -2003,17 +2013,14 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
# replace arguments referencing files with the file content
else:
try:
- args_file = open(arg_string[1:])
- try:
+ with open(arg_string[1:]) as args_file:
arg_strings = []
for arg_line in args_file.read().splitlines():
for arg in self.convert_arg_line_to_args(arg_line):
arg_strings.append(arg)
arg_strings = self._read_args_from_files(arg_strings)
new_arg_strings.extend(arg_strings)
- finally:
- args_file.close()
- except IOError:
+ except OSError:
err = _sys.exc_info()[1]
self.error(str(err))
diff --git a/Lib/ast.py b/Lib/ast.py
index 13f59f9..02c3b28 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -42,7 +42,6 @@ def literal_eval(node_or_string):
Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
sets, booleans, and None.
"""
- _safe_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, str):
node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, Expression):
@@ -61,9 +60,8 @@ def literal_eval(node_or_string):
elif isinstance(node, Dict):
return dict((_convert(k), _convert(v)) for k, v
in zip(node.keys, node.values))
- elif isinstance(node, Name):
- if node.id in _safe_names:
- return _safe_names[node.id]
+ elif isinstance(node, NameConstant):
+ return node.value
elif isinstance(node, UnaryOp) and \
isinstance(node.op, (UAdd, USub)) and \
isinstance(node.operand, (Num, UnaryOp, BinOp)):
diff --git a/Lib/asynchat.py b/Lib/asynchat.py
index eea3418..f1a5731 100644
--- a/Lib/asynchat.py
+++ b/Lib/asynchat.py
@@ -45,7 +45,6 @@ command will be accumulated (using your own 'collect_incoming_data'
method) up to the terminator, and then control will be returned to
you - by calling your self.found_terminator() method.
"""
-import socket
import asyncore
from collections import deque
@@ -56,8 +55,8 @@ class async_chat (asyncore.dispatcher):
# these are overridable defaults
- ac_in_buffer_size = 4096
- ac_out_buffer_size = 4096
+ ac_in_buffer_size = 65536
+ ac_out_buffer_size = 65536
# we don't want to enable the use of encoding by default, because that is a
# sign of an application bug that we don't want to pass silently
@@ -111,7 +110,7 @@ class async_chat (asyncore.dispatcher):
try:
data = self.recv (self.ac_in_buffer_size)
- except socket.error as why:
+ except OSError as why:
self.handle_error()
return
@@ -240,7 +239,7 @@ class async_chat (asyncore.dispatcher):
# send the data
try:
num_sent = self.send(data)
- except socket.error:
+ except OSError:
self.handle_error()
return
diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py
new file mode 100644
index 0000000..3df2f80
--- /dev/null
+++ b/Lib/asyncio/__init__.py
@@ -0,0 +1,45 @@
+"""The asyncio package, tracking PEP 3156."""
+
+import sys
+
+# The selectors module is in the stdlib in Python 3.4 but not in 3.3.
+# Do this first, so the other submodules can use "from . import selectors".
+# Prefer asyncio/selectors.py over the stdlib one, as ours may be newer.
+try:
+ from . import selectors
+except ImportError:
+ import selectors # Will also be exported.
+
+if sys.platform == 'win32':
+ # Similar thing for _overlapped.
+ try:
+ from . import _overlapped
+ except ImportError:
+ import _overlapped # Will also be exported.
+
+# This relies on each of the submodules having an __all__ variable.
+from .events import *
+from .futures import *
+from .locks import *
+from .protocols import *
+from .queues import *
+from .streams import *
+from .subprocess import *
+from .tasks import *
+from .transports import *
+
+if sys.platform == 'win32': # pragma: no cover
+ from .windows_events import *
+else:
+ from .unix_events import * # pragma: no cover
+
+
+__all__ = (events.__all__ +
+ futures.__all__ +
+ locks.__all__ +
+ protocols.__all__ +
+ queues.__all__ +
+ streams.__all__ +
+ subprocess.__all__ +
+ tasks.__all__ +
+ transports.__all__)
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
new file mode 100644
index 0000000..d2bdc07
--- /dev/null
+++ b/Lib/asyncio/base_events.py
@@ -0,0 +1,828 @@
+"""Base implementation of event loop.
+
+The event loop can be broken up into a multiplexer (the part
+responsible for notifying us of IO events) and the event loop proper,
+which wraps a multiplexer with functionality for scheduling callbacks,
+immediately or at a given time in the future.
+
+Whenever a public API takes a callback, subsequent positional
+arguments will be passed to the callback if/when it is called. This
+avoids the proliferation of trivial lambdas implementing closures.
+Keyword arguments for the callback are not supported; this is a
+conscious design decision, leaving the door open for keyword arguments
+to modify the meaning of the API call itself.
+"""
+
+
+import collections
+import concurrent.futures
+import heapq
+import logging
+import socket
+import subprocess
+import time
+import os
+import sys
+
+from . import events
+from . import futures
+from . import tasks
+from .log import logger
+
+
+__all__ = ['BaseEventLoop', 'Server']
+
+
+# Argument for default thread pool executor creation.
+_MAX_WORKERS = 5
+
+
+class _StopError(BaseException):
+ """Raised to stop the event loop."""
+
+
+def _check_resolved_address(sock, address):
+ # Ensure that the address is already resolved to avoid the trap of hanging
+ # the entire event loop when the address requires doing a DNS lookup.
+ family = sock.family
+ if family == socket.AF_INET:
+ host, port = address
+ elif family == socket.AF_INET6:
+ host, port = address[:2]
+ else:
+ return
+
+ type_mask = 0
+ if hasattr(socket, 'SOCK_NONBLOCK'):
+ type_mask |= socket.SOCK_NONBLOCK
+ if hasattr(socket, 'SOCK_CLOEXEC'):
+ type_mask |= socket.SOCK_CLOEXEC
+ # Use getaddrinfo(AI_NUMERICHOST) to ensure that the address is
+ # already resolved.
+ try:
+ socket.getaddrinfo(host, port,
+ family=family,
+ type=(sock.type & ~type_mask),
+ proto=sock.proto,
+ flags=socket.AI_NUMERICHOST)
+ except socket.gaierror as err:
+ raise ValueError("address must be resolved (IP address), got %r: %s"
+ % (address, err))
+
+def _raise_stop_error(*args):
+ raise _StopError
+
+
+class Server(events.AbstractServer):
+
+ def __init__(self, loop, sockets):
+ self.loop = loop
+ self.sockets = sockets
+ self.active_count = 0
+ self.waiters = []
+
+ def attach(self, transport):
+ assert self.sockets is not None
+ self.active_count += 1
+
+ def detach(self, transport):
+ assert self.active_count > 0
+ self.active_count -= 1
+ if self.active_count == 0 and self.sockets is None:
+ self._wakeup()
+
+ def close(self):
+ sockets = self.sockets
+ if sockets is not None:
+ self.sockets = None
+ for sock in sockets:
+ self.loop._stop_serving(sock)
+ if self.active_count == 0:
+ self._wakeup()
+
+ def _wakeup(self):
+ waiters = self.waiters
+ self.waiters = None
+ for waiter in waiters:
+ if not waiter.done():
+ waiter.set_result(waiter)
+
+ @tasks.coroutine
+ def wait_closed(self):
+ if self.sockets is None or self.waiters is None:
+ return
+ waiter = futures.Future(loop=self.loop)
+ self.waiters.append(waiter)
+ yield from waiter
+
+
+class BaseEventLoop(events.AbstractEventLoop):
+
+ def __init__(self):
+ self._ready = collections.deque()
+ self._scheduled = []
+ self._default_executor = None
+ self._internal_fds = 0
+ self._running = False
+ self._clock_resolution = time.get_clock_info('monotonic').resolution
+ self._exception_handler = None
+ self._debug = False
+
+ def _make_socket_transport(self, sock, protocol, waiter=None, *,
+ extra=None, server=None):
+ """Create socket transport."""
+ raise NotImplementedError
+
+ def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter, *,
+ server_side=False, server_hostname=None,
+ extra=None, server=None):
+ """Create SSL transport."""
+ raise NotImplementedError
+
+ def _make_datagram_transport(self, sock, protocol,
+ address=None, extra=None):
+ """Create datagram transport."""
+ raise NotImplementedError
+
+ def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
+ extra=None):
+ """Create read pipe transport."""
+ raise NotImplementedError
+
+ def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
+ extra=None):
+ """Create write pipe transport."""
+ raise NotImplementedError
+
+ @tasks.coroutine
+ def _make_subprocess_transport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=None, **kwargs):
+ """Create subprocess transport."""
+ raise NotImplementedError
+
+ def _read_from_self(self):
+ """XXX"""
+ raise NotImplementedError
+
+ def _write_to_self(self):
+ """XXX"""
+ raise NotImplementedError
+
+ def _process_events(self, event_list):
+ """Process selector events."""
+ raise NotImplementedError
+
+ def run_forever(self):
+ """Run until stop() is called."""
+ if self._running:
+ raise RuntimeError('Event loop is running.')
+ self._running = True
+ try:
+ while True:
+ try:
+ self._run_once()
+ except _StopError:
+ break
+ finally:
+ self._running = False
+
+ def run_until_complete(self, future):
+ """Run until the Future is done.
+
+ If the argument is a coroutine, it is wrapped in a Task.
+
+ XXX TBD: It would be disastrous to call run_until_complete()
+ with the same coroutine twice -- it would wrap it in two
+ different Tasks and that can't be good.
+
+ Return the Future's result, or raise its exception.
+ """
+ future = tasks.async(future, loop=self)
+ future.add_done_callback(_raise_stop_error)
+ self.run_forever()
+ future.remove_done_callback(_raise_stop_error)
+ if not future.done():
+ raise RuntimeError('Event loop stopped before Future completed.')
+
+ return future.result()
+
+ def stop(self):
+ """Stop running the event loop.
+
+ Every callback scheduled before stop() is called will run.
+ Callback scheduled after stop() is called won't. However,
+ those callbacks will run if run() is called again later.
+ """
+ self.call_soon(_raise_stop_error)
+
+ def close(self):
+ """Close the event loop.
+
+ This clears the queues and shuts down the executor,
+ but does not wait for the executor to finish.
+ """
+ self._ready.clear()
+ self._scheduled.clear()
+ executor = self._default_executor
+ if executor is not None:
+ self._default_executor = None
+ executor.shutdown(wait=False)
+
+ def is_running(self):
+ """Returns running status of event loop."""
+ return self._running
+
+ def time(self):
+ """Return the time according to the event loop's clock."""
+ return time.monotonic()
+
+ def call_later(self, delay, callback, *args):
+ """Arrange for a callback to be called at a given time.
+
+ Return a Handle: an opaque object with a cancel() method that
+ can be used to cancel the call.
+
+ The delay can be an int or float, expressed in seconds. It is
+ always a relative time.
+
+ Each callback will be called exactly once. If two callbacks
+ are scheduled for exactly the same time, it undefined which
+ will be called first.
+
+ Any positional arguments after the callback will be passed to
+ the callback when it is called.
+ """
+ return self.call_at(self.time() + delay, callback, *args)
+
+ def call_at(self, when, callback, *args):
+ """Like call_later(), but uses an absolute time."""
+ if tasks.iscoroutinefunction(callback):
+ raise TypeError("coroutines cannot be used with call_at()")
+ if self._debug:
+ self._assert_is_current_event_loop()
+ timer = events.TimerHandle(when, callback, args, self)
+ heapq.heappush(self._scheduled, timer)
+ return timer
+
+ def call_soon(self, callback, *args):
+ """Arrange for a callback to be called as soon as possible.
+
+ This operates as a FIFO queue, callbacks are called in the
+ order in which they are registered. Each callback will be
+ called exactly once.
+
+ Any positional arguments after the callback will be passed to
+ the callback when it is called.
+ """
+ return self._call_soon(callback, args, check_loop=True)
+
+ def _call_soon(self, callback, args, check_loop):
+ if tasks.iscoroutinefunction(callback):
+ raise TypeError("coroutines cannot be used with call_soon()")
+ if self._debug and check_loop:
+ self._assert_is_current_event_loop()
+ handle = events.Handle(callback, args, self)
+ self._ready.append(handle)
+ return handle
+
+ def _assert_is_current_event_loop(self):
+ """Asserts that this event loop is the current event loop.
+
+ Non-threadsafe methods of this class make this assumption and will
+ likely behave incorrectly when the assumption is violated.
+
+ Should only be called when (self._debug == True). The caller is
+ responsible for checking this condition for performance reasons.
+ """
+ if events.get_event_loop() is not self:
+ raise RuntimeError(
+ "non-threadsafe operation invoked on an event loop other "
+ "than the current one")
+
+ def call_soon_threadsafe(self, callback, *args):
+ """XXX"""
+ handle = self._call_soon(callback, args, check_loop=False)
+ self._write_to_self()
+ return handle
+
+ def run_in_executor(self, executor, callback, *args):
+ if tasks.iscoroutinefunction(callback):
+ raise TypeError("coroutines cannot be used with run_in_executor()")
+ if isinstance(callback, events.Handle):
+ assert not args
+ assert not isinstance(callback, events.TimerHandle)
+ if callback._cancelled:
+ f = futures.Future(loop=self)
+ f.set_result(None)
+ return f
+ callback, args = callback._callback, callback._args
+ if executor is None:
+ executor = self._default_executor
+ if executor is None:
+ executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS)
+ self._default_executor = executor
+ return futures.wrap_future(executor.submit(callback, *args), loop=self)
+
+ def set_default_executor(self, executor):
+ self._default_executor = executor
+
+ def getaddrinfo(self, host, port, *,
+ family=0, type=0, proto=0, flags=0):
+ return self.run_in_executor(None, socket.getaddrinfo,
+ host, port, family, type, proto, flags)
+
+ def getnameinfo(self, sockaddr, flags=0):
+ return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags)
+
+ @tasks.coroutine
+ def create_connection(self, protocol_factory, host=None, port=None, *,
+ ssl=None, family=0, proto=0, flags=0, sock=None,
+ local_addr=None, server_hostname=None):
+ """XXX"""
+ if server_hostname is not None and not ssl:
+ raise ValueError('server_hostname is only meaningful with ssl')
+
+ if server_hostname is None and ssl:
+ # Use host as default for server_hostname. It is an error
+ # if host is empty or not set, e.g. when an
+ # already-connected socket was passed or when only a port
+ # is given. To avoid this error, you can pass
+ # server_hostname='' -- this will bypass the hostname
+ # check. (This also means that if host is a numeric
+ # IP/IPv6 address, we will attempt to verify that exact
+ # address; this will probably fail, but it is possible to
+ # create a certificate for a specific IP address, so we
+ # don't judge it here.)
+ if not host:
+ raise ValueError('You must set server_hostname '
+ 'when using ssl without a host')
+ server_hostname = host
+
+ if host is not None or port is not None:
+ if sock is not None:
+ raise ValueError(
+ 'host/port and sock can not be specified at the same time')
+
+ f1 = self.getaddrinfo(
+ host, port, family=family,
+ type=socket.SOCK_STREAM, proto=proto, flags=flags)
+ fs = [f1]
+ if local_addr is not None:
+ f2 = self.getaddrinfo(
+ *local_addr, family=family,
+ type=socket.SOCK_STREAM, proto=proto, flags=flags)
+ fs.append(f2)
+ else:
+ f2 = None
+
+ yield from tasks.wait(fs, loop=self)
+
+ infos = f1.result()
+ if not infos:
+ raise OSError('getaddrinfo() returned empty list')
+ if f2 is not None:
+ laddr_infos = f2.result()
+ if not laddr_infos:
+ raise OSError('getaddrinfo() returned empty list')
+
+ exceptions = []
+ for family, type, proto, cname, address in infos:
+ try:
+ sock = socket.socket(family=family, type=type, proto=proto)
+ sock.setblocking(False)
+ if f2 is not None:
+ for _, _, _, _, laddr in laddr_infos:
+ try:
+ sock.bind(laddr)
+ break
+ except OSError as exc:
+ exc = OSError(
+ exc.errno, 'error while '
+ 'attempting to bind on address '
+ '{!r}: {}'.format(
+ laddr, exc.strerror.lower()))
+ exceptions.append(exc)
+ else:
+ sock.close()
+ sock = None
+ continue
+ yield from self.sock_connect(sock, address)
+ except OSError as exc:
+ if sock is not None:
+ sock.close()
+ exceptions.append(exc)
+ else:
+ break
+ else:
+ if len(exceptions) == 1:
+ raise exceptions[0]
+ else:
+ # If they all have the same str(), raise one.
+ model = str(exceptions[0])
+ if all(str(exc) == model for exc in exceptions):
+ raise exceptions[0]
+ # Raise a combined exception so the user can see all
+ # the various error messages.
+ raise OSError('Multiple exceptions: {}'.format(
+ ', '.join(str(exc) for exc in exceptions)))
+
+ elif sock is None:
+ raise ValueError(
+ 'host and port was not specified and no sock specified')
+
+ sock.setblocking(False)
+
+ transport, protocol = yield from self._create_connection_transport(
+ sock, protocol_factory, ssl, server_hostname)
+ return transport, protocol
+
+ @tasks.coroutine
+ def _create_connection_transport(self, sock, protocol_factory, ssl,
+ server_hostname):
+ protocol = protocol_factory()
+ waiter = futures.Future(loop=self)
+ if ssl:
+ sslcontext = None if isinstance(ssl, bool) else ssl
+ transport = self._make_ssl_transport(
+ sock, protocol, sslcontext, waiter,
+ server_side=False, server_hostname=server_hostname)
+ else:
+ transport = self._make_socket_transport(sock, protocol, waiter)
+
+ yield from waiter
+ return transport, protocol
+
+ @tasks.coroutine
+ def create_datagram_endpoint(self, protocol_factory,
+ local_addr=None, remote_addr=None, *,
+ family=0, proto=0, flags=0):
+ """Create datagram connection."""
+ if not (local_addr or remote_addr):
+ if family == 0:
+ raise ValueError('unexpected address family')
+ addr_pairs_info = (((family, proto), (None, None)),)
+ else:
+ # join addresss by (family, protocol)
+ addr_infos = collections.OrderedDict()
+ for idx, addr in ((0, local_addr), (1, remote_addr)):
+ if addr is not None:
+ assert isinstance(addr, tuple) and len(addr) == 2, (
+ '2-tuple is expected')
+
+ infos = yield from self.getaddrinfo(
+ *addr, family=family, type=socket.SOCK_DGRAM,
+ proto=proto, flags=flags)
+ if not infos:
+ raise OSError('getaddrinfo() returned empty list')
+
+ for fam, _, pro, _, address in infos:
+ key = (fam, pro)
+ if key not in addr_infos:
+ addr_infos[key] = [None, None]
+ addr_infos[key][idx] = address
+
+ # each addr has to have info for each (family, proto) pair
+ addr_pairs_info = [
+ (key, addr_pair) for key, addr_pair in addr_infos.items()
+ if not ((local_addr and addr_pair[0] is None) or
+ (remote_addr and addr_pair[1] is None))]
+
+ if not addr_pairs_info:
+ raise ValueError('can not get address information')
+
+ exceptions = []
+
+ for ((family, proto),
+ (local_address, remote_address)) in addr_pairs_info:
+ sock = None
+ r_addr = None
+ try:
+ sock = socket.socket(
+ family=family, type=socket.SOCK_DGRAM, proto=proto)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.setblocking(False)
+
+ if local_addr:
+ sock.bind(local_address)
+ if remote_addr:
+ yield from self.sock_connect(sock, remote_address)
+ r_addr = remote_address
+ except OSError as exc:
+ if sock is not None:
+ sock.close()
+ exceptions.append(exc)
+ else:
+ break
+ else:
+ raise exceptions[0]
+
+ protocol = protocol_factory()
+ transport = self._make_datagram_transport(sock, protocol, r_addr)
+ return transport, protocol
+
+ @tasks.coroutine
+ def create_server(self, protocol_factory, host=None, port=None,
+ *,
+ family=socket.AF_UNSPEC,
+ flags=socket.AI_PASSIVE,
+ sock=None,
+ backlog=100,
+ ssl=None,
+ reuse_address=None):
+ """XXX"""
+ if isinstance(ssl, bool):
+ raise TypeError('ssl argument must be an SSLContext or None')
+ if host is not None or port is not None:
+ if sock is not None:
+ raise ValueError(
+ 'host/port and sock can not be specified at the same time')
+
+ AF_INET6 = getattr(socket, 'AF_INET6', 0)
+ if reuse_address is None:
+ reuse_address = os.name == 'posix' and sys.platform != 'cygwin'
+ sockets = []
+ if host == '':
+ host = None
+
+ infos = yield from self.getaddrinfo(
+ host, port, family=family,
+ type=socket.SOCK_STREAM, proto=0, flags=flags)
+ if not infos:
+ raise OSError('getaddrinfo() returned empty list')
+
+ completed = False
+ try:
+ for res in infos:
+ af, socktype, proto, canonname, sa = res
+ try:
+ sock = socket.socket(af, socktype, proto)
+ except socket.error:
+ # Assume it's a bad family/type/protocol combination.
+ continue
+ sockets.append(sock)
+ if reuse_address:
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
+ True)
+ # Disable IPv4/IPv6 dual stack support (enabled by
+ # default on Linux) which makes a single socket
+ # listen on both address families.
+ if af == AF_INET6 and hasattr(socket, 'IPPROTO_IPV6'):
+ sock.setsockopt(socket.IPPROTO_IPV6,
+ socket.IPV6_V6ONLY,
+ True)
+ try:
+ sock.bind(sa)
+ except OSError as err:
+ raise OSError(err.errno, 'error while attempting '
+ 'to bind on address %r: %s'
+ % (sa, err.strerror.lower()))
+ completed = True
+ finally:
+ if not completed:
+ for sock in sockets:
+ sock.close()
+ else:
+ if sock is None:
+ raise ValueError(
+ 'host and port was not specified and no sock specified')
+ sockets = [sock]
+
+ server = Server(self, sockets)
+ for sock in sockets:
+ sock.listen(backlog)
+ sock.setblocking(False)
+ self._start_serving(protocol_factory, sock, ssl, server)
+ return server
+
+ @tasks.coroutine
+ def connect_read_pipe(self, protocol_factory, pipe):
+ protocol = protocol_factory()
+ waiter = futures.Future(loop=self)
+ transport = self._make_read_pipe_transport(pipe, protocol, waiter)
+ yield from waiter
+ return transport, protocol
+
+ @tasks.coroutine
+ def connect_write_pipe(self, protocol_factory, pipe):
+ protocol = protocol_factory()
+ waiter = futures.Future(loop=self)
+ transport = self._make_write_pipe_transport(pipe, protocol, waiter)
+ yield from waiter
+ return transport, protocol
+
+ @tasks.coroutine
+ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=False, shell=True, bufsize=0,
+ **kwargs):
+ if not isinstance(cmd, (bytes, str)):
+ raise ValueError("cmd must be a string")
+ if universal_newlines:
+ raise ValueError("universal_newlines must be False")
+ if not shell:
+ raise ValueError("shell must be True")
+ if bufsize != 0:
+ raise ValueError("bufsize must be 0")
+ protocol = protocol_factory()
+ transport = yield from self._make_subprocess_transport(
+ protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
+ return transport, protocol
+
+ @tasks.coroutine
+ def subprocess_exec(self, protocol_factory, program, *args,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, universal_newlines=False,
+ shell=False, bufsize=0, **kwargs):
+ if universal_newlines:
+ raise ValueError("universal_newlines must be False")
+ if shell:
+ raise ValueError("shell must be False")
+ if bufsize != 0:
+ raise ValueError("bufsize must be 0")
+ popen_args = (program,) + args
+ for arg in popen_args:
+ if not isinstance(arg, (str, bytes)):
+ raise TypeError("program arguments must be "
+ "a bytes or text string, not %s"
+ % type(arg).__name__)
+ protocol = protocol_factory()
+ transport = yield from self._make_subprocess_transport(
+ protocol, popen_args, False, stdin, stdout, stderr,
+ bufsize, **kwargs)
+ return transport, protocol
+
+ def set_exception_handler(self, handler):
+ """Set handler as the new event loop exception handler.
+
+ If handler is None, the default exception handler will
+ be set.
+
+ If handler is a callable object, it should have a
+ matching signature to '(loop, context)', where 'loop'
+ will be a reference to the active event loop, 'context'
+ will be a dict object (see `call_exception_handler()`
+ documentation for details about context).
+ """
+ if handler is not None and not callable(handler):
+ raise TypeError('A callable object or None is expected, '
+ 'got {!r}'.format(handler))
+ self._exception_handler = handler
+
+ def default_exception_handler(self, context):
+ """Default exception handler.
+
+ This is called when an exception occurs and no exception
+ handler is set, and can be called by a custom exception
+ handler that wants to defer to the default behavior.
+
+ context parameter has the same meaning as in
+ `call_exception_handler()`.
+ """
+ message = context.get('message')
+ if not message:
+ message = 'Unhandled exception in event loop'
+
+ exception = context.get('exception')
+ if exception is not None:
+ exc_info = (type(exception), exception, exception.__traceback__)
+ else:
+ exc_info = False
+
+ log_lines = [message]
+ for key in sorted(context):
+ if key in {'message', 'exception'}:
+ continue
+ log_lines.append('{}: {!r}'.format(key, context[key]))
+
+ logger.error('\n'.join(log_lines), exc_info=exc_info)
+
+ def call_exception_handler(self, context):
+ """Call the current event loop exception handler.
+
+ context is a dict object containing the following keys
+ (new keys maybe introduced later):
+ - 'message': Error message;
+ - 'exception' (optional): Exception object;
+ - 'future' (optional): Future instance;
+ - 'handle' (optional): Handle instance;
+ - 'protocol' (optional): Protocol instance;
+ - 'transport' (optional): Transport instance;
+ - 'socket' (optional): Socket instance.
+
+ Note: this method should not be overloaded in subclassed
+ event loops. For any custom exception handling, use
+ `set_exception_handler()` method.
+ """
+ if self._exception_handler is None:
+ try:
+ self.default_exception_handler(context)
+ except Exception:
+ # Second protection layer for unexpected errors
+ # in the default implementation, as well as for subclassed
+ # event loops with overloaded "default_exception_handler".
+ logger.error('Exception in default exception handler',
+ exc_info=True)
+ else:
+ try:
+ self._exception_handler(self, context)
+ except Exception as exc:
+ # Exception in the user set custom exception handler.
+ try:
+ # Let's try default handler.
+ self.default_exception_handler({
+ 'message': 'Unhandled error in exception handler',
+ 'exception': exc,
+ 'context': context,
+ })
+ except Exception:
+ # Guard 'default_exception_handler' in case it's
+ # overloaded.
+ logger.error('Exception in default exception handler '
+ 'while handling an unexpected error '
+ 'in custom exception handler',
+ exc_info=True)
+
+ def _add_callback(self, handle):
+ """Add a Handle to ready or scheduled."""
+ assert isinstance(handle, events.Handle), 'A Handle is required here'
+ if handle._cancelled:
+ return
+ if isinstance(handle, events.TimerHandle):
+ heapq.heappush(self._scheduled, handle)
+ else:
+ self._ready.append(handle)
+
+ def _add_callback_signalsafe(self, handle):
+ """Like _add_callback() but called from a signal handler."""
+ self._add_callback(handle)
+ self._write_to_self()
+
+ def _run_once(self):
+ """Run one full iteration of the event loop.
+
+ This calls all currently ready callbacks, polls for I/O,
+ schedules the resulting callbacks, and finally schedules
+ 'call_later' callbacks.
+ """
+ # Remove delayed calls that were cancelled from head of queue.
+ while self._scheduled and self._scheduled[0]._cancelled:
+ heapq.heappop(self._scheduled)
+
+ timeout = None
+ if self._ready:
+ timeout = 0
+ elif self._scheduled:
+ # Compute the desired timeout.
+ when = self._scheduled[0]._when
+ deadline = max(0, when - self.time())
+ if timeout is None:
+ timeout = deadline
+ else:
+ timeout = min(timeout, deadline)
+
+ # TODO: Instrumentation only in debug mode?
+ if logger.isEnabledFor(logging.INFO):
+ t0 = self.time()
+ event_list = self._selector.select(timeout)
+ t1 = self.time()
+ if t1-t0 >= 1:
+ level = logging.INFO
+ else:
+ level = logging.DEBUG
+ if timeout is not None:
+ logger.log(level, 'poll %.3f took %.3f seconds',
+ timeout, t1-t0)
+ else:
+ logger.log(level, 'poll took %.3f seconds', t1-t0)
+ else:
+ event_list = self._selector.select(timeout)
+ self._process_events(event_list)
+
+ # Handle 'later' callbacks that are ready.
+ end_time = self.time() + self._clock_resolution
+ while self._scheduled:
+ handle = self._scheduled[0]
+ if handle._when >= end_time:
+ break
+ handle = heapq.heappop(self._scheduled)
+ self._ready.append(handle)
+
+ # This is the only place where callbacks are actually *called*.
+ # All other places just add them to ready.
+ # Note: We run all currently scheduled callbacks, but not any
+ # callbacks scheduled by callbacks run this time around --
+ # they will be run the next time (after another I/O poll).
+ # Use an idiom that is threadsafe without using locks.
+ ntodo = len(self._ready)
+ for i in range(ntodo):
+ handle = self._ready.popleft()
+ if not handle._cancelled:
+ handle._run()
+ handle = None # Needed to break cycles when an exception occurs.
+
+ def get_debug(self):
+ return self._debug
+
+ def set_debug(self, enabled):
+ self._debug = enabled
diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py
new file mode 100644
index 0000000..b78f816
--- /dev/null
+++ b/Lib/asyncio/base_subprocess.py
@@ -0,0 +1,159 @@
+import collections
+import subprocess
+
+from . import protocols
+from . import tasks
+from . import transports
+
+
+class BaseSubprocessTransport(transports.SubprocessTransport):
+
+ def __init__(self, loop, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=None, **kwargs):
+ super().__init__(extra)
+ self._protocol = protocol
+ self._loop = loop
+
+ self._pipes = {}
+ if stdin == subprocess.PIPE:
+ self._pipes[0] = None
+ if stdout == subprocess.PIPE:
+ self._pipes[1] = None
+ if stderr == subprocess.PIPE:
+ self._pipes[2] = None
+ self._pending_calls = collections.deque()
+ self._finished = False
+ self._returncode = None
+ self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
+ stderr=stderr, bufsize=bufsize, **kwargs)
+ self._extra['subprocess'] = self._proc
+
+ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
+ raise NotImplementedError
+
+ def _make_write_subprocess_pipe_proto(self, fd):
+ raise NotImplementedError
+
+ def _make_read_subprocess_pipe_proto(self, fd):
+ raise NotImplementedError
+
+ def close(self):
+ for proto in self._pipes.values():
+ proto.pipe.close()
+ if self._returncode is None:
+ self.terminate()
+
+ def get_pid(self):
+ return self._proc.pid
+
+ def get_returncode(self):
+ return self._returncode
+
+ def get_pipe_transport(self, fd):
+ if fd in self._pipes:
+ return self._pipes[fd].pipe
+ else:
+ return None
+
+ def send_signal(self, signal):
+ self._proc.send_signal(signal)
+
+ def terminate(self):
+ self._proc.terminate()
+
+ def kill(self):
+ self._proc.kill()
+
+ @tasks.coroutine
+ def _post_init(self):
+ proc = self._proc
+ loop = self._loop
+ if proc.stdin is not None:
+ _, pipe = yield from loop.connect_write_pipe(
+ lambda: WriteSubprocessPipeProto(self, 0),
+ proc.stdin)
+ self._pipes[0] = pipe
+ if proc.stdout is not None:
+ _, pipe = yield from loop.connect_read_pipe(
+ lambda: ReadSubprocessPipeProto(self, 1),
+ proc.stdout)
+ self._pipes[1] = pipe
+ if proc.stderr is not None:
+ _, pipe = yield from loop.connect_read_pipe(
+ lambda: ReadSubprocessPipeProto(self, 2),
+ proc.stderr)
+ self._pipes[2] = pipe
+
+ assert self._pending_calls is not None
+
+ self._loop.call_soon(self._protocol.connection_made, self)
+ for callback, data in self._pending_calls:
+ self._loop.call_soon(callback, *data)
+ self._pending_calls = None
+
+ def _call(self, cb, *data):
+ if self._pending_calls is not None:
+ self._pending_calls.append((cb, data))
+ else:
+ self._loop.call_soon(cb, *data)
+
+ def _pipe_connection_lost(self, fd, exc):
+ self._call(self._protocol.pipe_connection_lost, fd, exc)
+ self._try_finish()
+
+ def _pipe_data_received(self, fd, data):
+ self._call(self._protocol.pipe_data_received, fd, data)
+
+ def _process_exited(self, returncode):
+ assert returncode is not None, returncode
+ assert self._returncode is None, self._returncode
+ self._returncode = returncode
+ self._call(self._protocol.process_exited)
+ self._try_finish()
+
+ def _try_finish(self):
+ assert not self._finished
+ if self._returncode is None:
+ return
+ if all(p is not None and p.disconnected
+ for p in self._pipes.values()):
+ self._finished = True
+ self._loop.call_soon(self._call_connection_lost, None)
+
+ def _call_connection_lost(self, exc):
+ try:
+ self._protocol.connection_lost(exc)
+ finally:
+ self._proc = None
+ self._protocol = None
+ self._loop = None
+
+
+class WriteSubprocessPipeProto(protocols.BaseProtocol):
+
+ def __init__(self, proc, fd):
+ self.proc = proc
+ self.fd = fd
+ self.pipe = None
+ self.disconnected = False
+
+ def connection_made(self, transport):
+ self.pipe = transport
+
+ def connection_lost(self, exc):
+ self.disconnected = True
+ self.proc._pipe_connection_lost(self.fd, exc)
+
+ def pause_writing(self):
+ self.proc._protocol.pause_writing()
+
+ def resume_writing(self):
+ self.proc._protocol.resume_writing()
+
+
+class ReadSubprocessPipeProto(WriteSubprocessPipeProto,
+ protocols.Protocol):
+
+ def data_received(self, data):
+ self.proc._pipe_data_received(self.fd, data)
diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py
new file mode 100644
index 0000000..f9e1232
--- /dev/null
+++ b/Lib/asyncio/constants.py
@@ -0,0 +1,7 @@
+"""Constants."""
+
+# After the connection is lost, log warnings after this many write()s.
+LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5
+
+# Seconds to wait before retrying accept().
+ACCEPT_RETRY_DELAY = 1
diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py
new file mode 100644
index 0000000..57af68a
--- /dev/null
+++ b/Lib/asyncio/events.py
@@ -0,0 +1,485 @@
+"""Event loop and event loop policy."""
+
+__all__ = ['AbstractEventLoopPolicy',
+ 'AbstractEventLoop', 'AbstractServer',
+ 'Handle', 'TimerHandle',
+ 'get_event_loop_policy', 'set_event_loop_policy',
+ 'get_event_loop', 'set_event_loop', 'new_event_loop',
+ 'get_child_watcher', 'set_child_watcher',
+ ]
+
+import subprocess
+import threading
+import socket
+
+
+class Handle:
+ """Object returned by callback registration methods."""
+
+ __slots__ = ['_callback', '_args', '_cancelled', '_loop']
+
+ def __init__(self, callback, args, loop):
+ assert not isinstance(callback, Handle), 'A Handle is not a callback'
+ self._loop = loop
+ self._callback = callback
+ self._args = args
+ self._cancelled = False
+
+ def __repr__(self):
+ res = 'Handle({}, {})'.format(self._callback, self._args)
+ if self._cancelled:
+ res += '<cancelled>'
+ return res
+
+ def cancel(self):
+ self._cancelled = True
+
+ def _run(self):
+ try:
+ self._callback(*self._args)
+ except Exception as exc:
+ msg = 'Exception in callback {}{!r}'.format(self._callback,
+ self._args)
+ self._loop.call_exception_handler({
+ 'message': msg,
+ 'exception': exc,
+ 'handle': self,
+ })
+ self = None # Needed to break cycles when an exception occurs.
+
+
+class TimerHandle(Handle):
+ """Object returned by timed callback registration methods."""
+
+ __slots__ = ['_when']
+
+ def __init__(self, when, callback, args, loop):
+ assert when is not None
+ super().__init__(callback, args, loop)
+
+ self._when = when
+
+ def __repr__(self):
+ res = 'TimerHandle({}, {}, {})'.format(self._when,
+ self._callback,
+ self._args)
+ if self._cancelled:
+ res += '<cancelled>'
+
+ return res
+
+ def __hash__(self):
+ return hash(self._when)
+
+ def __lt__(self, other):
+ return self._when < other._when
+
+ def __le__(self, other):
+ if self._when < other._when:
+ return True
+ return self.__eq__(other)
+
+ def __gt__(self, other):
+ return self._when > other._when
+
+ def __ge__(self, other):
+ if self._when > other._when:
+ return True
+ return self.__eq__(other)
+
+ def __eq__(self, other):
+ if isinstance(other, TimerHandle):
+ return (self._when == other._when and
+ self._callback == other._callback and
+ self._args == other._args and
+ self._cancelled == other._cancelled)
+ return NotImplemented
+
+ def __ne__(self, other):
+ equal = self.__eq__(other)
+ return NotImplemented if equal is NotImplemented else not equal
+
+
+class AbstractServer:
+ """Abstract server returned by create_server()."""
+
+ def close(self):
+ """Stop serving. This leaves existing connections open."""
+ return NotImplemented
+
+ def wait_closed(self):
+ """Coroutine to wait until service is closed."""
+ return NotImplemented
+
+
+class AbstractEventLoop:
+ """Abstract event loop."""
+
+ # Running and stopping the event loop.
+
+ def run_forever(self):
+ """Run the event loop until stop() is called."""
+ raise NotImplementedError
+
+ def run_until_complete(self, future):
+ """Run the event loop until a Future is done.
+
+ Return the Future's result, or raise its exception.
+ """
+ raise NotImplementedError
+
+ def stop(self):
+ """Stop the event loop as soon as reasonable.
+
+ Exactly how soon that is may depend on the implementation, but
+ no more I/O callbacks should be scheduled.
+ """
+ raise NotImplementedError
+
+ def is_running(self):
+ """Return whether the event loop is currently running."""
+ raise NotImplementedError
+
+ def close(self):
+ """Close the loop.
+
+ The loop should not be running.
+
+ This is idempotent and irreversible.
+
+ No other methods should be called after this one.
+ """
+ raise NotImplementedError
+
+ # Methods scheduling callbacks. All these return Handles.
+
+ def call_soon(self, callback, *args):
+ return self.call_later(0, callback, *args)
+
+ def call_later(self, delay, callback, *args):
+ raise NotImplementedError
+
+ def call_at(self, when, callback, *args):
+ raise NotImplementedError
+
+ def time(self):
+ raise NotImplementedError
+
+ # Methods for interacting with threads.
+
+ def call_soon_threadsafe(self, callback, *args):
+ raise NotImplementedError
+
+ def run_in_executor(self, executor, callback, *args):
+ raise NotImplementedError
+
+ def set_default_executor(self, executor):
+ raise NotImplementedError
+
+ # Network I/O methods returning Futures.
+
+ def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0):
+ raise NotImplementedError
+
+ def getnameinfo(self, sockaddr, flags=0):
+ raise NotImplementedError
+
+ def create_connection(self, protocol_factory, host=None, port=None, *,
+ ssl=None, family=0, proto=0, flags=0, sock=None,
+ local_addr=None, server_hostname=None):
+ raise NotImplementedError
+
+ def create_server(self, protocol_factory, host=None, port=None, *,
+ family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE,
+ sock=None, backlog=100, ssl=None, reuse_address=None):
+ """A coroutine which creates a TCP server bound to host and port.
+
+ The return value is a Server object which can be used to stop
+ the service.
+
+ If host is an empty string or None all interfaces are assumed
+ and a list of multiple sockets will be returned (most likely
+ one for IPv4 and another one for IPv6).
+
+ family can be set to either AF_INET or AF_INET6 to force the
+ socket to use IPv4 or IPv6. If not set it will be determined
+ from host (defaults to AF_UNSPEC).
+
+ flags is a bitmask for getaddrinfo().
+
+ sock can optionally be specified in order to use a preexisting
+ socket object.
+
+ backlog is the maximum number of queued connections passed to
+ listen() (defaults to 100).
+
+ ssl can be set to an SSLContext to enable SSL over the
+ accepted connections.
+
+ reuse_address tells the kernel to reuse a local socket in
+ TIME_WAIT state, without waiting for its natural timeout to
+ expire. If not specified will automatically be set to True on
+ UNIX.
+ """
+ raise NotImplementedError
+
+ def create_unix_connection(self, protocol_factory, path, *,
+ ssl=None, sock=None,
+ server_hostname=None):
+ raise NotImplementedError
+
+ def create_unix_server(self, protocol_factory, path, *,
+ sock=None, backlog=100, ssl=None):
+ """A coroutine which creates a UNIX Domain Socket server.
+
+ The return value is a Server object, which can be used to stop
+ the service.
+
+ path is a str, representing a file systsem path to bind the
+ server socket to.
+
+ sock can optionally be specified in order to use a preexisting
+ socket object.
+
+ backlog is the maximum number of queued connections passed to
+ listen() (defaults to 100).
+
+ ssl can be set to an SSLContext to enable SSL over the
+ accepted connections.
+ """
+ raise NotImplementedError
+
+ def create_datagram_endpoint(self, protocol_factory,
+ local_addr=None, remote_addr=None, *,
+ family=0, proto=0, flags=0):
+ raise NotImplementedError
+
+ # Pipes and subprocesses.
+
+ def connect_read_pipe(self, protocol_factory, pipe):
+ """Register read pipe in event loop.
+
+ protocol_factory should instantiate object with Protocol interface.
+ pipe is file-like object already switched to nonblocking.
+ Return pair (transport, protocol), where transport support
+ ReadTransport interface."""
+ # The reason to accept file-like object instead of just file descriptor
+ # is: we need to own pipe and close it at transport finishing
+ # Can got complicated errors if pass f.fileno(),
+ # close fd in pipe transport then close f and vise versa.
+ raise NotImplementedError
+
+ def connect_write_pipe(self, protocol_factory, pipe):
+ """Register write pipe in event loop.
+
+ protocol_factory should instantiate object with BaseProtocol interface.
+ Pipe is file-like object already switched to nonblocking.
+ Return pair (transport, protocol), where transport support
+ WriteTransport interface."""
+ # The reason to accept file-like object instead of just file descriptor
+ # is: we need to own pipe and close it at transport finishing
+ # Can got complicated errors if pass f.fileno(),
+ # close fd in pipe transport then close f and vise versa.
+ raise NotImplementedError
+
+ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ **kwargs):
+ raise NotImplementedError
+
+ def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ **kwargs):
+ raise NotImplementedError
+
+ # Ready-based callback registration methods.
+ # The add_*() methods return None.
+ # The remove_*() methods return True if something was removed,
+ # False if there was nothing to delete.
+
+ def add_reader(self, fd, callback, *args):
+ raise NotImplementedError
+
+ def remove_reader(self, fd):
+ raise NotImplementedError
+
+ def add_writer(self, fd, callback, *args):
+ raise NotImplementedError
+
+ def remove_writer(self, fd):
+ raise NotImplementedError
+
+ # Completion based I/O methods returning Futures.
+
+ def sock_recv(self, sock, nbytes):
+ raise NotImplementedError
+
+ def sock_sendall(self, sock, data):
+ raise NotImplementedError
+
+ def sock_connect(self, sock, address):
+ raise NotImplementedError
+
+ def sock_accept(self, sock):
+ raise NotImplementedError
+
+ # Signal handling.
+
+ def add_signal_handler(self, sig, callback, *args):
+ raise NotImplementedError
+
+ def remove_signal_handler(self, sig):
+ raise NotImplementedError
+
+ # Error handlers.
+
+ def set_exception_handler(self, handler):
+ raise NotImplementedError
+
+ def default_exception_handler(self, context):
+ raise NotImplementedError
+
+ def call_exception_handler(self, context):
+ raise NotImplementedError
+
+ # Debug flag management.
+
+ def get_debug(self):
+ raise NotImplementedError
+
+ def set_debug(self, enabled):
+ raise NotImplementedError
+
+
+class AbstractEventLoopPolicy:
+ """Abstract policy for accessing the event loop."""
+
+ def get_event_loop(self):
+ """XXX"""
+ raise NotImplementedError
+
+ def set_event_loop(self, loop):
+ """XXX"""
+ raise NotImplementedError
+
+ def new_event_loop(self):
+ """XXX"""
+ raise NotImplementedError
+
+ # Child processes handling (Unix only).
+
+ def get_child_watcher(self):
+ """XXX"""
+ raise NotImplementedError
+
+ def set_child_watcher(self, watcher):
+ """XXX"""
+ raise NotImplementedError
+
+
+class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
+ """Default policy implementation for accessing the event loop.
+
+ In this policy, each thread has its own event loop. However, we
+ only automatically create an event loop by default for the main
+ thread; other threads by default have no event loop.
+
+ Other policies may have different rules (e.g. a single global
+ event loop, or automatically creating an event loop per thread, or
+ using some other notion of context to which an event loop is
+ associated).
+ """
+
+ _loop_factory = None
+
+ class _Local(threading.local):
+ _loop = None
+ _set_called = False
+
+ def __init__(self):
+ self._local = self._Local()
+
+ def get_event_loop(self):
+ """Get the event loop.
+
+ This may be None or an instance of EventLoop.
+ """
+ if (self._local._loop is None and
+ not self._local._set_called and
+ isinstance(threading.current_thread(), threading._MainThread)):
+ self.set_event_loop(self.new_event_loop())
+ assert self._local._loop is not None, \
+ ('There is no current event loop in thread %r.' %
+ threading.current_thread().name)
+ return self._local._loop
+
+ def set_event_loop(self, loop):
+ """Set the event loop."""
+ self._local._set_called = True
+ assert loop is None or isinstance(loop, AbstractEventLoop)
+ self._local._loop = loop
+
+ def new_event_loop(self):
+ """Create a new event loop.
+
+ You must call set_event_loop() to make this the current event
+ loop.
+ """
+ return self._loop_factory()
+
+
+# Event loop policy. The policy itself is always global, even if the
+# policy's rules say that there is an event loop per thread (or other
+# notion of context). The default policy is installed by the first
+# call to get_event_loop_policy().
+_event_loop_policy = None
+
+# Lock for protecting the on-the-fly creation of the event loop policy.
+_lock = threading.Lock()
+
+
+def _init_event_loop_policy():
+ global _event_loop_policy
+ with _lock:
+ if _event_loop_policy is None: # pragma: no branch
+ from . import DefaultEventLoopPolicy
+ _event_loop_policy = DefaultEventLoopPolicy()
+
+
+def get_event_loop_policy():
+ """XXX"""
+ if _event_loop_policy is None:
+ _init_event_loop_policy()
+ return _event_loop_policy
+
+
+def set_event_loop_policy(policy):
+ """XXX"""
+ global _event_loop_policy
+ assert policy is None or isinstance(policy, AbstractEventLoopPolicy)
+ _event_loop_policy = policy
+
+
+def get_event_loop():
+ """XXX"""
+ return get_event_loop_policy().get_event_loop()
+
+
+def set_event_loop(loop):
+ """XXX"""
+ get_event_loop_policy().set_event_loop(loop)
+
+
+def new_event_loop():
+ """XXX"""
+ return get_event_loop_policy().new_event_loop()
+
+
+def get_child_watcher():
+ """XXX"""
+ return get_event_loop_policy().get_child_watcher()
+
+
+def set_child_watcher(watcher):
+ """XXX"""
+ return get_event_loop_policy().set_child_watcher(watcher)
diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py
new file mode 100644
index 0000000..91ea170
--- /dev/null
+++ b/Lib/asyncio/futures.py
@@ -0,0 +1,371 @@
+"""A Future class similar to the one in PEP 3148."""
+
+__all__ = ['CancelledError', 'TimeoutError',
+ 'InvalidStateError',
+ 'Future', 'wrap_future',
+ ]
+
+import concurrent.futures._base
+import logging
+import sys
+import traceback
+
+from . import events
+
+# States for Future.
+_PENDING = 'PENDING'
+_CANCELLED = 'CANCELLED'
+_FINISHED = 'FINISHED'
+
+_PY34 = sys.version_info >= (3, 4)
+
+# TODO: Do we really want to depend on concurrent.futures internals?
+Error = concurrent.futures._base.Error
+CancelledError = concurrent.futures.CancelledError
+TimeoutError = concurrent.futures.TimeoutError
+
+STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging
+
+
+class InvalidStateError(Error):
+ """The operation is not allowed in this state."""
+ # TODO: Show the future, its state, the method, and the required state.
+
+
+class _TracebackLogger:
+ """Helper to log a traceback upon destruction if not cleared.
+
+ This solves a nasty problem with Futures and Tasks that have an
+ exception set: if nobody asks for the exception, the exception is
+ never logged. This violates the Zen of Python: 'Errors should
+ never pass silently. Unless explicitly silenced.'
+
+ However, we don't want to log the exception as soon as
+ set_exception() is called: if the calling code is written
+ properly, it will get the exception and handle it properly. But
+ we *do* want to log it if result() or exception() was never called
+ -- otherwise developers waste a lot of time wondering why their
+ buggy code fails silently.
+
+ An earlier attempt added a __del__() method to the Future class
+ itself, but this backfired because the presence of __del__()
+ prevents garbage collection from breaking cycles. A way out of
+ this catch-22 is to avoid having a __del__() method on the Future
+ class itself, but instead to have a reference to a helper object
+ with a __del__() method that logs the traceback, where we ensure
+ that the helper object doesn't participate in cycles, and only the
+ Future has a reference to it.
+
+ The helper object is added when set_exception() is called. When
+ the Future is collected, and the helper is present, the helper
+ object is also collected, and its __del__() method will log the
+ traceback. When the Future's result() or exception() method is
+ called (and a helper object is present), it removes the the helper
+ object, after calling its clear() method to prevent it from
+ logging.
+
+ One downside is that we do a fair amount of work to extract the
+ traceback from the exception, even when it is never logged. It
+ would seem cheaper to just store the exception object, but that
+ references the traceback, which references stack frames, which may
+ reference the Future, which references the _TracebackLogger, and
+ then the _TracebackLogger would be included in a cycle, which is
+ what we're trying to avoid! As an optimization, we don't
+ immediately format the exception; we only do the work when
+ activate() is called, which call is delayed until after all the
+ Future's callbacks have run. Since usually a Future has at least
+ one callback (typically set by 'yield from') and usually that
+ callback extracts the callback, thereby removing the need to
+ format the exception.
+
+ PS. I don't claim credit for this solution. I first heard of it
+ in a discussion about closing files when they are collected.
+ """
+
+ __slots__ = ['exc', 'tb', 'loop']
+
+ def __init__(self, exc, loop):
+ self.loop = loop
+ self.exc = exc
+ self.tb = None
+
+ def activate(self):
+ exc = self.exc
+ if exc is not None:
+ self.exc = None
+ self.tb = traceback.format_exception(exc.__class__, exc,
+ exc.__traceback__)
+
+ def clear(self):
+ self.exc = None
+ self.tb = None
+
+ def __del__(self):
+ if self.tb:
+ msg = 'Future/Task exception was never retrieved:\n{tb}'
+ context = {
+ 'message': msg.format(tb=''.join(self.tb)),
+ }
+ self.loop.call_exception_handler(context)
+
+
+class Future:
+ """This class is *almost* compatible with concurrent.futures.Future.
+
+ Differences:
+
+ - result() and exception() do not take a timeout argument and
+ raise an exception when the future isn't done yet.
+
+ - Callbacks registered with add_done_callback() are always called
+ via the event loop's call_soon_threadsafe().
+
+ - This class is not compatible with the wait() and as_completed()
+ methods in the concurrent.futures package.
+
+ (In Python 3.4 or later we may be able to unify the implementations.)
+ """
+
+ # Class variables serving as defaults for instance variables.
+ _state = _PENDING
+ _result = None
+ _exception = None
+ _loop = None
+
+ _blocking = False # proper use of future (yield vs yield from)
+
+ _log_traceback = False # Used for Python 3.4 and later
+ _tb_logger = None # Used for Python 3.3 only
+
+ def __init__(self, *, loop=None):
+ """Initialize the future.
+
+ The optional event_loop argument allows to explicitly set the event
+ loop object used by the future. If it's not provided, the future uses
+ the default event loop.
+ """
+ if loop is None:
+ self._loop = events.get_event_loop()
+ else:
+ self._loop = loop
+ self._callbacks = []
+
+ def __repr__(self):
+ res = self.__class__.__name__
+ if self._state == _FINISHED:
+ if self._exception is not None:
+ res += '<exception={!r}>'.format(self._exception)
+ else:
+ res += '<result={!r}>'.format(self._result)
+ elif self._callbacks:
+ size = len(self._callbacks)
+ if size > 2:
+ res += '<{}, [{}, <{} more>, {}]>'.format(
+ self._state, self._callbacks[0],
+ size-2, self._callbacks[-1])
+ else:
+ res += '<{}, {}>'.format(self._state, self._callbacks)
+ else:
+ res += '<{}>'.format(self._state)
+ return res
+
+ if _PY34:
+ def __del__(self):
+ if not self._log_traceback:
+ # set_exception() was not called, or result() or exception()
+ # has consumed the exception
+ return
+ exc = self._exception
+ context = {
+ 'message': 'Future/Task exception was never retrieved',
+ 'exception': exc,
+ 'future': self,
+ }
+ self._loop.call_exception_handler(context)
+
+ def cancel(self):
+ """Cancel the future and schedule callbacks.
+
+ If the future is already done or cancelled, return False. Otherwise,
+ change the future's state to cancelled, schedule the callbacks and
+ return True.
+ """
+ if self._state != _PENDING:
+ return False
+ self._state = _CANCELLED
+ self._schedule_callbacks()
+ return True
+
+ def _schedule_callbacks(self):
+ """Internal: Ask the event loop to call all callbacks.
+
+ The callbacks are scheduled to be called as soon as possible. Also
+ clears the callback list.
+ """
+ callbacks = self._callbacks[:]
+ if not callbacks:
+ return
+
+ self._callbacks[:] = []
+ for callback in callbacks:
+ self._loop.call_soon(callback, self)
+
+ def cancelled(self):
+ """Return True if the future was cancelled."""
+ return self._state == _CANCELLED
+
+ # Don't implement running(); see http://bugs.python.org/issue18699
+
+ def done(self):
+ """Return True if the future is done.
+
+ Done means either that a result / exception are available, or that the
+ future was cancelled.
+ """
+ return self._state != _PENDING
+
+ def result(self):
+ """Return the result this future represents.
+
+ If the future has been cancelled, raises CancelledError. If the
+ future's result isn't yet available, raises InvalidStateError. If
+ the future is done and has an exception set, this exception is raised.
+ """
+ if self._state == _CANCELLED:
+ raise CancelledError
+ if self._state != _FINISHED:
+ raise InvalidStateError('Result is not ready.')
+ self._log_traceback = False
+ if self._tb_logger is not None:
+ self._tb_logger.clear()
+ self._tb_logger = None
+ if self._exception is not None:
+ raise self._exception
+ return self._result
+
+ def exception(self):
+ """Return the exception that was set on this future.
+
+ The exception (or None if no exception was set) is returned only if
+ the future is done. If the future has been cancelled, raises
+ CancelledError. If the future isn't done yet, raises
+ InvalidStateError.
+ """
+ if self._state == _CANCELLED:
+ raise CancelledError
+ if self._state != _FINISHED:
+ raise InvalidStateError('Exception is not set.')
+ self._log_traceback = False
+ if self._tb_logger is not None:
+ self._tb_logger.clear()
+ self._tb_logger = None
+ return self._exception
+
+ def add_done_callback(self, fn):
+ """Add a callback to be run when the future becomes done.
+
+ The callback is called with a single argument - the future object. If
+ the future is already done when this is called, the callback is
+ scheduled with call_soon.
+ """
+ if self._state != _PENDING:
+ self._loop.call_soon(fn, self)
+ else:
+ self._callbacks.append(fn)
+
+ # New method not in PEP 3148.
+
+ def remove_done_callback(self, fn):
+ """Remove all instances of a callback from the "call when done" list.
+
+ Returns the number of callbacks removed.
+ """
+ filtered_callbacks = [f for f in self._callbacks if f != fn]
+ removed_count = len(self._callbacks) - len(filtered_callbacks)
+ if removed_count:
+ self._callbacks[:] = filtered_callbacks
+ return removed_count
+
+ # So-called internal methods (note: no set_running_or_notify_cancel()).
+
+ def set_result(self, result):
+ """Mark the future done and set its result.
+
+ If the future is already done when this method is called, raises
+ InvalidStateError.
+ """
+ if self._state != _PENDING:
+ raise InvalidStateError('{}: {!r}'.format(self._state, self))
+ self._result = result
+ self._state = _FINISHED
+ self._schedule_callbacks()
+
+ def set_exception(self, exception):
+ """Mark the future done and set an exception.
+
+ If the future is already done when this method is called, raises
+ InvalidStateError.
+ """
+ if self._state != _PENDING:
+ raise InvalidStateError('{}: {!r}'.format(self._state, self))
+ if isinstance(exception, type):
+ exception = exception()
+ self._exception = exception
+ self._state = _FINISHED
+ self._schedule_callbacks()
+ if _PY34:
+ self._log_traceback = True
+ else:
+ self._tb_logger = _TracebackLogger(exception, self._loop)
+ # Arrange for the logger to be activated after all callbacks
+ # have had a chance to call result() or exception().
+ self._loop.call_soon(self._tb_logger.activate)
+
+ # Truly internal methods.
+
+ def _copy_state(self, other):
+ """Internal helper to copy state from another Future.
+
+ The other Future may be a concurrent.futures.Future.
+ """
+ assert other.done()
+ if self.cancelled():
+ return
+ assert not self.done()
+ if other.cancelled():
+ self.cancel()
+ else:
+ exception = other.exception()
+ if exception is not None:
+ self.set_exception(exception)
+ else:
+ result = other.result()
+ self.set_result(result)
+
+ def __iter__(self):
+ if not self.done():
+ self._blocking = True
+ yield self # This tells Task to wait for completion.
+ assert self.done(), "yield from wasn't used with future"
+ return self.result() # May raise too.
+
+
+def wrap_future(fut, *, loop=None):
+ """Wrap concurrent.futures.Future object."""
+ if isinstance(fut, Future):
+ return fut
+ assert isinstance(fut, concurrent.futures.Future), \
+ 'concurrent.futures.Future is expected, got {!r}'.format(fut)
+ if loop is None:
+ loop = events.get_event_loop()
+ new_future = Future(loop=loop)
+
+ def _check_cancel_other(f):
+ if f.cancelled():
+ fut.cancel()
+
+ new_future.add_done_callback(_check_cancel_other)
+ fut.add_done_callback(
+ lambda future: loop.call_soon_threadsafe(
+ new_future._copy_state, fut))
+ return new_future
diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
new file mode 100644
index 0000000..29c4434
--- /dev/null
+++ b/Lib/asyncio/locks.py
@@ -0,0 +1,466 @@
+"""Synchronization primitives."""
+
+__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore']
+
+import collections
+
+from . import events
+from . import futures
+from . import tasks
+
+
+class _ContextManager:
+ """Context manager.
+
+ This enables the following idiom for acquiring and releasing a
+ lock around a block:
+
+ with (yield from lock):
+ <block>
+
+ while failing loudly when accidentally using:
+
+ with lock:
+ <block>
+ """
+
+ def __init__(self, lock):
+ self._lock = lock
+
+ def __enter__(self):
+ # We have no use for the "as ..." clause in the with
+ # statement for locks.
+ return None
+
+ def __exit__(self, *args):
+ try:
+ self._lock.release()
+ finally:
+ self._lock = None # Crudely prevent reuse.
+
+
+class Lock:
+ """Primitive lock objects.
+
+ A primitive lock is a synchronization primitive that is not owned
+ by a particular coroutine when locked. A primitive lock is in one
+ of two states, 'locked' or 'unlocked'.
+
+ It is created in the unlocked state. It has two basic methods,
+ acquire() and release(). When the state is unlocked, acquire()
+ changes the state to locked and returns immediately. When the
+ state is locked, acquire() blocks until a call to release() in
+ another coroutine changes it to unlocked, then the acquire() call
+ resets it to locked and returns. The release() method should only
+ be called in the locked state; it changes the state to unlocked
+ and returns immediately. If an attempt is made to release an
+ unlocked lock, a RuntimeError will be raised.
+
+ When more than one coroutine is blocked in acquire() waiting for
+ the state to turn to unlocked, only one coroutine proceeds when a
+ release() call resets the state to unlocked; first coroutine which
+ is blocked in acquire() is being processed.
+
+ acquire() is a coroutine and should be called with 'yield from'.
+
+ Locks also support the context manager protocol. '(yield from lock)'
+ should be used as context manager expression.
+
+ Usage:
+
+ lock = Lock()
+ ...
+ yield from lock
+ try:
+ ...
+ finally:
+ lock.release()
+
+ Context manager usage:
+
+ lock = Lock()
+ ...
+ with (yield from lock):
+ ...
+
+ Lock objects can be tested for locking state:
+
+ if not lock.locked():
+ yield from lock
+ else:
+ # lock is acquired
+ ...
+
+ """
+
+ def __init__(self, *, loop=None):
+ self._waiters = collections.deque()
+ self._locked = False
+ if loop is not None:
+ self._loop = loop
+ else:
+ self._loop = events.get_event_loop()
+
+ def __repr__(self):
+ res = super().__repr__()
+ extra = 'locked' if self._locked else 'unlocked'
+ if self._waiters:
+ extra = '{},waiters:{}'.format(extra, len(self._waiters))
+ return '<{} [{}]>'.format(res[1:-1], extra)
+
+ def locked(self):
+ """Return True if lock is acquired."""
+ return self._locked
+
+ @tasks.coroutine
+ def acquire(self):
+ """Acquire a lock.
+
+ This method blocks until the lock is unlocked, then sets it to
+ locked and returns True.
+ """
+ if not self._waiters and not self._locked:
+ self._locked = True
+ return True
+
+ fut = futures.Future(loop=self._loop)
+ self._waiters.append(fut)
+ try:
+ yield from fut
+ self._locked = True
+ return True
+ finally:
+ self._waiters.remove(fut)
+
+ def release(self):
+ """Release a lock.
+
+ When the lock is locked, reset it to unlocked, and return.
+ If any other coroutines are blocked waiting for the lock to become
+ unlocked, allow exactly one of them to proceed.
+
+ When invoked on an unlocked lock, a RuntimeError is raised.
+
+ There is no return value.
+ """
+ if self._locked:
+ self._locked = False
+ # Wake up the first waiter who isn't cancelled.
+ for fut in self._waiters:
+ if not fut.done():
+ fut.set_result(True)
+ break
+ else:
+ raise RuntimeError('Lock is not acquired.')
+
+ def __enter__(self):
+ raise RuntimeError(
+ '"yield from" should be used as context manager expression')
+
+ def __exit__(self, *args):
+ # This must exist because __enter__ exists, even though that
+ # always raises; that's how the with-statement works.
+ pass
+
+ def __iter__(self):
+ # This is not a coroutine. It is meant to enable the idiom:
+ #
+ # with (yield from lock):
+ # <block>
+ #
+ # as an alternative to:
+ #
+ # yield from lock.acquire()
+ # try:
+ # <block>
+ # finally:
+ # lock.release()
+ yield from self.acquire()
+ return _ContextManager(self)
+
+
+class Event:
+ """Asynchronous equivalent to threading.Event.
+
+ Class implementing event objects. An event manages a flag that can be set
+ to true with the set() method and reset to false with the clear() method.
+ The wait() method blocks until the flag is true. The flag is initially
+ false.
+ """
+
+ def __init__(self, *, loop=None):
+ self._waiters = collections.deque()
+ self._value = False
+ if loop is not None:
+ self._loop = loop
+ else:
+ self._loop = events.get_event_loop()
+
+ def __repr__(self):
+ res = super().__repr__()
+ extra = 'set' if self._value else 'unset'
+ if self._waiters:
+ extra = '{},waiters:{}'.format(extra, len(self._waiters))
+ return '<{} [{}]>'.format(res[1:-1], extra)
+
+ def is_set(self):
+ """Return True if and only if the internal flag is true."""
+ return self._value
+
+ def set(self):
+ """Set the internal flag to true. All coroutines waiting for it to
+ become true are awakened. Coroutine that call wait() once the flag is
+ true will not block at all.
+ """
+ if not self._value:
+ self._value = True
+
+ for fut in self._waiters:
+ if not fut.done():
+ fut.set_result(True)
+
+ def clear(self):
+ """Reset the internal flag to false. Subsequently, coroutines calling
+ wait() will block until set() is called to set the internal flag
+ to true again."""
+ self._value = False
+
+ @tasks.coroutine
+ def wait(self):
+ """Block until the internal flag is true.
+
+ If the internal flag is true on entry, return True
+ immediately. Otherwise, block until another coroutine calls
+ set() to set the flag to true, then return True.
+ """
+ if self._value:
+ return True
+
+ fut = futures.Future(loop=self._loop)
+ self._waiters.append(fut)
+ try:
+ yield from fut
+ return True
+ finally:
+ self._waiters.remove(fut)
+
+
+class Condition:
+ """Asynchronous equivalent to threading.Condition.
+
+ This class implements condition variable objects. A condition variable
+ allows one or more coroutines to wait until they are notified by another
+ coroutine.
+
+ A new Lock object is created and used as the underlying lock.
+ """
+
+ def __init__(self, *, loop=None):
+ if loop is not None:
+ self._loop = loop
+ else:
+ self._loop = events.get_event_loop()
+
+ # Lock as an attribute as in threading.Condition.
+ lock = Lock(loop=self._loop)
+ self._lock = lock
+ # Export the lock's locked(), acquire() and release() methods.
+ self.locked = lock.locked
+ self.acquire = lock.acquire
+ self.release = lock.release
+
+ self._waiters = collections.deque()
+
+ def __repr__(self):
+ res = super().__repr__()
+ extra = 'locked' if self.locked() else 'unlocked'
+ if self._waiters:
+ extra = '{},waiters:{}'.format(extra, len(self._waiters))
+ return '<{} [{}]>'.format(res[1:-1], extra)
+
+ @tasks.coroutine
+ def wait(self):
+ """Wait until notified.
+
+ If the calling coroutine has not acquired the lock when this
+ method is called, a RuntimeError is raised.
+
+ This method releases the underlying lock, and then blocks
+ until it is awakened by a notify() or notify_all() call for
+ the same condition variable in another coroutine. Once
+ awakened, it re-acquires the lock and returns True.
+ """
+ if not self.locked():
+ raise RuntimeError('cannot wait on un-acquired lock')
+
+ self.release()
+ try:
+ fut = futures.Future(loop=self._loop)
+ self._waiters.append(fut)
+ try:
+ yield from fut
+ return True
+ finally:
+ self._waiters.remove(fut)
+
+ finally:
+ yield from self.acquire()
+
+ @tasks.coroutine
+ def wait_for(self, predicate):
+ """Wait until a predicate becomes true.
+
+ The predicate should be a callable which result will be
+ interpreted as a boolean value. The final predicate value is
+ the return value.
+ """
+ result = predicate()
+ while not result:
+ yield from self.wait()
+ result = predicate()
+ return result
+
+ def notify(self, n=1):
+ """By default, wake up one coroutine waiting on this condition, if any.
+ If the calling coroutine has not acquired the lock when this method
+ is called, a RuntimeError is raised.
+
+ This method wakes up at most n of the coroutines waiting for the
+ condition variable; it is a no-op if no coroutines are waiting.
+
+ Note: an awakened coroutine does not actually return from its
+ wait() call until it can reacquire the lock. Since notify() does
+ not release the lock, its caller should.
+ """
+ if not self.locked():
+ raise RuntimeError('cannot notify on un-acquired lock')
+
+ idx = 0
+ for fut in self._waiters:
+ if idx >= n:
+ break
+
+ if not fut.done():
+ idx += 1
+ fut.set_result(False)
+
+ def notify_all(self):
+ """Wake up all threads waiting on this condition. This method acts
+ like notify(), but wakes up all waiting threads instead of one. If the
+ calling thread has not acquired the lock when this method is called,
+ a RuntimeError is raised.
+ """
+ self.notify(len(self._waiters))
+
+ def __enter__(self):
+ raise RuntimeError(
+ '"yield from" should be used as context manager expression')
+
+ def __exit__(self, *args):
+ pass
+
+ def __iter__(self):
+ # See comment in Lock.__iter__().
+ yield from self.acquire()
+ return _ContextManager(self)
+
+
+class Semaphore:
+ """A Semaphore implementation.
+
+ A semaphore manages an internal counter which is decremented by each
+ acquire() call and incremented by each release() call. The counter
+ can never go below zero; when acquire() finds that it is zero, it blocks,
+ waiting until some other thread calls release().
+
+ Semaphores also support the context manager protocol.
+
+ The optional argument gives the initial value for the internal
+ counter; it defaults to 1. If the value given is less than 0,
+ ValueError is raised.
+ """
+
+ def __init__(self, value=1, *, loop=None):
+ if value < 0:
+ raise ValueError("Semaphore initial value must be >= 0")
+ self._value = value
+ self._waiters = collections.deque()
+ if loop is not None:
+ self._loop = loop
+ else:
+ self._loop = events.get_event_loop()
+
+ def __repr__(self):
+ res = super().__repr__()
+ extra = 'locked' if self.locked() else 'unlocked,value:{}'.format(
+ self._value)
+ if self._waiters:
+ extra = '{},waiters:{}'.format(extra, len(self._waiters))
+ return '<{} [{}]>'.format(res[1:-1], extra)
+
+ def locked(self):
+ """Returns True if semaphore can not be acquired immediately."""
+ return self._value == 0
+
+ @tasks.coroutine
+ def acquire(self):
+ """Acquire a semaphore.
+
+ If the internal counter is larger than zero on entry,
+ decrement it by one and return True immediately. If it is
+ zero on entry, block, waiting until some other coroutine has
+ called release() to make it larger than 0, and then return
+ True.
+ """
+ if not self._waiters and self._value > 0:
+ self._value -= 1
+ return True
+
+ fut = futures.Future(loop=self._loop)
+ self._waiters.append(fut)
+ try:
+ yield from fut
+ self._value -= 1
+ return True
+ finally:
+ self._waiters.remove(fut)
+
+ def release(self):
+ """Release a semaphore, incrementing the internal counter by one.
+ When it was zero on entry and another coroutine is waiting for it to
+ become larger than zero again, wake up that coroutine.
+ """
+ self._value += 1
+ for waiter in self._waiters:
+ if not waiter.done():
+ waiter.set_result(True)
+ break
+
+ def __enter__(self):
+ raise RuntimeError(
+ '"yield from" should be used as context manager expression')
+
+ def __exit__(self, *args):
+ pass
+
+ def __iter__(self):
+ # See comment in Lock.__iter__().
+ yield from self.acquire()
+ return _ContextManager(self)
+
+
+class BoundedSemaphore(Semaphore):
+ """A bounded semaphore implementation.
+
+ This raises ValueError in release() if it would increase the value
+ above the initial value.
+ """
+
+ def __init__(self, value=1, *, loop=None):
+ self._bound_value = value
+ super().__init__(value, loop=loop)
+
+ def release(self):
+ if self._value >= self._bound_value:
+ raise ValueError('BoundedSemaphore released too many times')
+ super().release()
diff --git a/Lib/asyncio/log.py b/Lib/asyncio/log.py
new file mode 100644
index 0000000..23a7074
--- /dev/null
+++ b/Lib/asyncio/log.py
@@ -0,0 +1,7 @@
+"""Logging configuration."""
+
+import logging
+
+
+# Name the logger after the package.
+logger = logging.getLogger(__package__)
diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py
new file mode 100644
index 0000000..d99e8ce
--- /dev/null
+++ b/Lib/asyncio/proactor_events.py
@@ -0,0 +1,455 @@
+"""Event loop using a proactor and related classes.
+
+A proactor is a "notify-on-completion" multiplexer. Currently a
+proactor is only implemented on Windows with IOCP.
+"""
+
+__all__ = ['BaseProactorEventLoop']
+
+import socket
+
+from . import base_events
+from . import constants
+from . import futures
+from . import transports
+from .log import logger
+
+
+class _ProactorBasePipeTransport(transports._FlowControlMixin,
+ transports.BaseTransport):
+ """Base class for pipe and socket transports."""
+
+ def __init__(self, loop, sock, protocol, waiter=None,
+ extra=None, server=None):
+ super().__init__(extra)
+ self._set_extra(sock)
+ self._loop = loop
+ self._sock = sock
+ self._protocol = protocol
+ self._server = server
+ self._buffer = None # None or bytearray.
+ self._read_fut = None
+ self._write_fut = None
+ self._pending_write = 0
+ self._conn_lost = 0
+ self._closing = False # Set when close() called.
+ self._eof_written = False
+ if self._server is not None:
+ self._server.attach(self)
+ self._loop.call_soon(self._protocol.connection_made, self)
+ if waiter is not None:
+ self._loop.call_soon(waiter.set_result, None)
+
+ def _set_extra(self, sock):
+ self._extra['pipe'] = sock
+
+ def close(self):
+ if self._closing:
+ return
+ self._closing = True
+ self._conn_lost += 1
+ if not self._buffer and self._write_fut is None:
+ self._loop.call_soon(self._call_connection_lost, None)
+ if self._read_fut is not None:
+ self._read_fut.cancel()
+
+ def _fatal_error(self, exc, message='Fatal error on pipe transport'):
+ if not isinstance(exc, (BrokenPipeError, ConnectionResetError)):
+ self._loop.call_exception_handler({
+ 'message': message,
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+ self._force_close(exc)
+
+ def _force_close(self, exc):
+ if self._closing:
+ return
+ self._closing = True
+ self._conn_lost += 1
+ if self._write_fut:
+ self._write_fut.cancel()
+ if self._read_fut:
+ self._read_fut.cancel()
+ self._write_fut = self._read_fut = None
+ self._pending_write = 0
+ self._buffer = None
+ self._loop.call_soon(self._call_connection_lost, exc)
+
+ def _call_connection_lost(self, exc):
+ try:
+ self._protocol.connection_lost(exc)
+ finally:
+ # XXX If there is a pending overlapped read on the other
+ # end then it may fail with ERROR_NETNAME_DELETED if we
+ # just close our end. First calling shutdown() seems to
+ # cure it, but maybe using DisconnectEx() would be better.
+ if hasattr(self._sock, 'shutdown'):
+ self._sock.shutdown(socket.SHUT_RDWR)
+ self._sock.close()
+ server = self._server
+ if server is not None:
+ server.detach(self)
+ self._server = None
+
+ def get_write_buffer_size(self):
+ size = self._pending_write
+ if self._buffer is not None:
+ size += len(self._buffer)
+ return size
+
+
+class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
+ transports.ReadTransport):
+ """Transport for read pipes."""
+
+ def __init__(self, loop, sock, protocol, waiter=None,
+ extra=None, server=None):
+ super().__init__(loop, sock, protocol, waiter, extra, server)
+ self._read_fut = None
+ self._paused = False
+ self._loop.call_soon(self._loop_reading)
+
+ def pause_reading(self):
+ if self._closing:
+ raise RuntimeError('Cannot pause_reading() when closing')
+ if self._paused:
+ raise RuntimeError('Already paused')
+ self._paused = True
+
+ def resume_reading(self):
+ if not self._paused:
+ raise RuntimeError('Not paused')
+ self._paused = False
+ if self._closing:
+ return
+ self._loop.call_soon(self._loop_reading, self._read_fut)
+
+ def _loop_reading(self, fut=None):
+ if self._paused:
+ return
+ data = None
+
+ try:
+ if fut is not None:
+ assert self._read_fut is fut or (self._read_fut is None and
+ self._closing)
+ self._read_fut = None
+ data = fut.result() # deliver data later in "finally" clause
+
+ if self._closing:
+ # since close() has been called we ignore any read data
+ data = None
+ return
+
+ if data == b'':
+ # we got end-of-file so no need to reschedule a new read
+ return
+
+ # reschedule a new read
+ self._read_fut = self._loop._proactor.recv(self._sock, 4096)
+ except ConnectionAbortedError as exc:
+ if not self._closing:
+ self._fatal_error(exc, 'Fatal read error on pipe transport')
+ except ConnectionResetError as exc:
+ self._force_close(exc)
+ except OSError as exc:
+ self._fatal_error(exc, 'Fatal read error on pipe transport')
+ except futures.CancelledError:
+ if not self._closing:
+ raise
+ else:
+ self._read_fut.add_done_callback(self._loop_reading)
+ finally:
+ if data:
+ self._protocol.data_received(data)
+ elif data is not None:
+ keep_open = self._protocol.eof_received()
+ if not keep_open:
+ self.close()
+
+
+class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
+ transports.WriteTransport):
+ """Transport for write pipes."""
+
+ def write(self, data):
+ if not isinstance(data, (bytes, bytearray, memoryview)):
+ raise TypeError('data argument must be byte-ish (%r)',
+ type(data))
+ if self._eof_written:
+ raise RuntimeError('write_eof() already called')
+
+ if not data:
+ return
+
+ if self._conn_lost:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('socket.send() raised exception.')
+ self._conn_lost += 1
+ return
+
+ # Observable states:
+ # 1. IDLE: _write_fut and _buffer both None
+ # 2. WRITING: _write_fut set; _buffer None
+ # 3. BACKED UP: _write_fut set; _buffer a bytearray
+ # We always copy the data, so the caller can't modify it
+ # while we're still waiting for the I/O to happen.
+ if self._write_fut is None: # IDLE -> WRITING
+ assert self._buffer is None
+ # Pass a copy, except if it's already immutable.
+ self._loop_writing(data=bytes(data))
+ # XXX Should we pause the protocol at this point
+ # if len(data) > self._high_water? (That would
+ # require keeping track of the number of bytes passed
+ # to a send() that hasn't finished yet.)
+ elif not self._buffer: # WRITING -> BACKED UP
+ # Make a mutable copy which we can extend.
+ self._buffer = bytearray(data)
+ self._maybe_pause_protocol()
+ else: # BACKED UP
+ # Append to buffer (also copies).
+ self._buffer.extend(data)
+ self._maybe_pause_protocol()
+
+ def _loop_writing(self, f=None, data=None):
+ try:
+ assert f is self._write_fut
+ self._write_fut = None
+ self._pending_write = 0
+ if f:
+ f.result()
+ if data is None:
+ data = self._buffer
+ self._buffer = None
+ if not data:
+ if self._closing:
+ self._loop.call_soon(self._call_connection_lost, None)
+ if self._eof_written:
+ self._sock.shutdown(socket.SHUT_WR)
+ # Now that we've reduced the buffer size, tell the
+ # protocol to resume writing if it was paused. Note that
+ # we do this last since the callback is called immediately
+ # and it may add more data to the buffer (even causing the
+ # protocol to be paused again).
+ self._maybe_resume_protocol()
+ else:
+ self._write_fut = self._loop._proactor.send(self._sock, data)
+ if not self._write_fut.done():
+ assert self._pending_write == 0
+ self._pending_write = len(data)
+ self._write_fut.add_done_callback(self._loop_writing)
+ self._maybe_pause_protocol()
+ else:
+ self._write_fut.add_done_callback(self._loop_writing)
+ except ConnectionResetError as exc:
+ self._force_close(exc)
+ except OSError as exc:
+ self._fatal_error(exc, 'Fatal write error on pipe transport')
+
+ def can_write_eof(self):
+ return True
+
+ def write_eof(self):
+ self.close()
+
+ def abort(self):
+ self._force_close(None)
+
+
+class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
+ def __init__(self, *args, **kw):
+ super().__init__(*args, **kw)
+ self._read_fut = self._loop._proactor.recv(self._sock, 16)
+ self._read_fut.add_done_callback(self._pipe_closed)
+
+ def _pipe_closed(self, fut):
+ if fut.cancelled():
+ # the transport has been closed
+ return
+ assert fut.result() == b''
+ if self._closing:
+ assert self._read_fut is None
+ return
+ assert fut is self._read_fut, (fut, self._read_fut)
+ self._read_fut = None
+ if self._write_fut is not None:
+ self._force_close(BrokenPipeError())
+ else:
+ self.close()
+
+
+class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
+ _ProactorBaseWritePipeTransport,
+ transports.Transport):
+ """Transport for duplex pipes."""
+
+ def can_write_eof(self):
+ return False
+
+ def write_eof(self):
+ raise NotImplementedError
+
+
+class _ProactorSocketTransport(_ProactorReadPipeTransport,
+ _ProactorBaseWritePipeTransport,
+ transports.Transport):
+ """Transport for connected sockets."""
+
+ def _set_extra(self, sock):
+ self._extra['socket'] = sock
+ try:
+ self._extra['sockname'] = sock.getsockname()
+ except (socket.error, AttributeError):
+ pass
+ if 'peername' not in self._extra:
+ try:
+ self._extra['peername'] = sock.getpeername()
+ except (socket.error, AttributeError):
+ pass
+
+ def can_write_eof(self):
+ return True
+
+ def write_eof(self):
+ if self._closing or self._eof_written:
+ return
+ self._eof_written = True
+ if self._write_fut is None:
+ self._sock.shutdown(socket.SHUT_WR)
+
+
+class BaseProactorEventLoop(base_events.BaseEventLoop):
+
+ def __init__(self, proactor):
+ super().__init__()
+ logger.debug('Using proactor: %s', proactor.__class__.__name__)
+ self._proactor = proactor
+ self._selector = proactor # convenient alias
+ self._self_reading_future = None
+ self._accept_futures = {} # socket file descriptor => Future
+ proactor.set_loop(self)
+ self._make_self_pipe()
+
+ def _make_socket_transport(self, sock, protocol, waiter=None,
+ extra=None, server=None):
+ return _ProactorSocketTransport(self, sock, protocol, waiter,
+ extra, server)
+
+ def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
+ extra=None):
+ return _ProactorDuplexPipeTransport(self,
+ sock, protocol, waiter, extra)
+
+ def _make_read_pipe_transport(self, sock, protocol, waiter=None,
+ extra=None):
+ return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra)
+
+ def _make_write_pipe_transport(self, sock, protocol, waiter=None,
+ extra=None):
+ # We want connection_lost() to be called when other end closes
+ return _ProactorWritePipeTransport(self,
+ sock, protocol, waiter, extra)
+
+ def close(self):
+ if self._proactor is not None:
+ self._close_self_pipe()
+ self._proactor.close()
+ self._proactor = None
+ self._selector = None
+ super().close()
+ self._accept_futures.clear()
+
+ def sock_recv(self, sock, n):
+ return self._proactor.recv(sock, n)
+
+ def sock_sendall(self, sock, data):
+ return self._proactor.send(sock, data)
+
+ def sock_connect(self, sock, address):
+ try:
+ base_events._check_resolved_address(sock, address)
+ except ValueError as err:
+ fut = futures.Future(loop=self)
+ fut.set_exception(err)
+ return fut
+ else:
+ return self._proactor.connect(sock, address)
+
+ def sock_accept(self, sock):
+ return self._proactor.accept(sock)
+
+ def _socketpair(self):
+ raise NotImplementedError
+
+ def _close_self_pipe(self):
+ if self._self_reading_future is not None:
+ self._self_reading_future.cancel()
+ self._self_reading_future = None
+ self._ssock.close()
+ self._ssock = None
+ self._csock.close()
+ self._csock = None
+ self._internal_fds -= 1
+
+ def _make_self_pipe(self):
+ # A self-socket, really. :-)
+ self._ssock, self._csock = self._socketpair()
+ self._ssock.setblocking(False)
+ self._csock.setblocking(False)
+ self._internal_fds += 1
+ self.call_soon(self._loop_self_reading)
+
+ def _loop_self_reading(self, f=None):
+ try:
+ if f is not None:
+ f.result() # may raise
+ f = self._proactor.recv(self._ssock, 4096)
+ except:
+ self.close()
+ raise
+ else:
+ self._self_reading_future = f
+ f.add_done_callback(self._loop_self_reading)
+
+ def _write_to_self(self):
+ self._csock.send(b'x')
+
+ def _start_serving(self, protocol_factory, sock, ssl=None, server=None):
+ if ssl:
+ raise ValueError('IocpEventLoop is incompatible with SSL.')
+
+ def loop(f=None):
+ try:
+ if f is not None:
+ conn, addr = f.result()
+ protocol = protocol_factory()
+ self._make_socket_transport(
+ conn, protocol,
+ extra={'peername': addr}, server=server)
+ f = self._proactor.accept(sock)
+ except OSError as exc:
+ if sock.fileno() != -1:
+ self.call_exception_handler({
+ 'message': 'Accept failed',
+ 'exception': exc,
+ 'socket': sock,
+ })
+ sock.close()
+ except futures.CancelledError:
+ sock.close()
+ else:
+ self._accept_futures[sock.fileno()] = f
+ f.add_done_callback(loop)
+
+ self.call_soon(loop)
+
+ def _process_events(self, event_list):
+ pass # XXX hard work currently done in poll
+
+ def _stop_serving(self, sock):
+ for future in self._accept_futures.values():
+ future.cancel()
+ self._proactor._stop_serving(sock)
+ sock.close()
diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py
new file mode 100644
index 0000000..52fc25c
--- /dev/null
+++ b/Lib/asyncio/protocols.py
@@ -0,0 +1,129 @@
+"""Abstract Protocol class."""
+
+__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol',
+ 'SubprocessProtocol']
+
+
+class BaseProtocol:
+ """Common base class for protocol interfaces.
+
+ Usually user implements protocols that derived from BaseProtocol
+ like Protocol or ProcessProtocol.
+
+ The only case when BaseProtocol should be implemented directly is
+ write-only transport like write pipe
+ """
+
+ def connection_made(self, transport):
+ """Called when a connection is made.
+
+ The argument is the transport representing the pipe connection.
+ To receive data, wait for data_received() calls.
+ When the connection is closed, connection_lost() is called.
+ """
+
+ def connection_lost(self, exc):
+ """Called when the connection is lost or closed.
+
+ The argument is an exception object or None (the latter
+ meaning a regular EOF is received or the connection was
+ aborted or closed).
+ """
+
+ def pause_writing(self):
+ """Called when the transport's buffer goes over the high-water mark.
+
+ Pause and resume calls are paired -- pause_writing() is called
+ once when the buffer goes strictly over the high-water mark
+ (even if subsequent writes increases the buffer size even
+ more), and eventually resume_writing() is called once when the
+ buffer size reaches the low-water mark.
+
+ Note that if the buffer size equals the high-water mark,
+ pause_writing() is not called -- it must go strictly over.
+ Conversely, resume_writing() is called when the buffer size is
+ equal or lower than the low-water mark. These end conditions
+ are important to ensure that things go as expected when either
+ mark is zero.
+
+ NOTE: This is the only Protocol callback that is not called
+ through EventLoop.call_soon() -- if it were, it would have no
+ effect when it's most needed (when the app keeps writing
+ without yielding until pause_writing() is called).
+ """
+
+ def resume_writing(self):
+ """Called when the transport's buffer drains below the low-water mark.
+
+ See pause_writing() for details.
+ """
+
+
+class Protocol(BaseProtocol):
+ """Interface for stream protocol.
+
+ The user should implement this interface. They can inherit from
+ this class but don't need to. The implementations here do
+ nothing (they don't raise exceptions).
+
+ When the user wants to requests a transport, they pass a protocol
+ factory to a utility function (e.g., EventLoop.create_connection()).
+
+ When the connection is made successfully, connection_made() is
+ called with a suitable transport object. Then data_received()
+ will be called 0 or more times with data (bytes) received from the
+ transport; finally, connection_lost() will be called exactly once
+ with either an exception object or None as an argument.
+
+ State machine of calls:
+
+ start -> CM [-> DR*] [-> ER?] -> CL -> end
+ """
+
+ def data_received(self, data):
+ """Called when some data is received.
+
+ The argument is a bytes object.
+ """
+
+ def eof_received(self):
+ """Called when the other end calls write_eof() or equivalent.
+
+ If this returns a false value (including None), the transport
+ will close itself. If it returns a true value, closing the
+ transport is up to the protocol.
+ """
+
+
+class DatagramProtocol(BaseProtocol):
+ """Interface for datagram protocol."""
+
+ def datagram_received(self, data, addr):
+ """Called when some datagram is received."""
+
+ def error_received(self, exc):
+ """Called when a send or receive operation raises an OSError.
+
+ (Other than BlockingIOError or InterruptedError.)
+ """
+
+
+class SubprocessProtocol(BaseProtocol):
+ """Interface for protocol for subprocess calls."""
+
+ def pipe_data_received(self, fd, data):
+ """Called when the subprocess writes data into stdout/stderr pipe.
+
+ fd is int file descriptor.
+ data is bytes object.
+ """
+
+ def pipe_connection_lost(self, fd, exc):
+ """Called when a file descriptor associated with the child process is
+ closed.
+
+ fd is the int file descriptor that was closed.
+ """
+
+ def process_exited(self):
+ """Called when subprocess has exited."""
diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
new file mode 100644
index 0000000..6283db3
--- /dev/null
+++ b/Lib/asyncio/queues.py
@@ -0,0 +1,288 @@
+"""Queues"""
+
+__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'JoinableQueue',
+ 'QueueFull', 'QueueEmpty']
+
+import collections
+import heapq
+
+from . import events
+from . import futures
+from . import locks
+from .tasks import coroutine
+
+
+class QueueEmpty(Exception):
+ 'Exception raised by Queue.get(block=0)/get_nowait().'
+ pass
+
+
+class QueueFull(Exception):
+ 'Exception raised by Queue.put(block=0)/put_nowait().'
+ pass
+
+
+class Queue:
+ """A queue, useful for coordinating producer and consumer coroutines.
+
+ If maxsize is less than or equal to zero, the queue size is infinite. If it
+ is an integer greater than 0, then "yield from put()" will block when the
+ queue reaches maxsize, until an item is removed by get().
+
+ Unlike the standard library Queue, you can reliably know this Queue's size
+ with qsize(), since your single-threaded asyncio application won't be
+ interrupted between calling qsize() and doing an operation on the Queue.
+ """
+
+ def __init__(self, maxsize=0, *, loop=None):
+ if loop is None:
+ self._loop = events.get_event_loop()
+ else:
+ self._loop = loop
+ self._maxsize = maxsize
+
+ # Futures.
+ self._getters = collections.deque()
+ # Pairs of (item, Future).
+ self._putters = collections.deque()
+ self._init(maxsize)
+
+ def _init(self, maxsize):
+ self._queue = collections.deque()
+
+ def _get(self):
+ return self._queue.popleft()
+
+ def _put(self, item):
+ self._queue.append(item)
+
+ def __repr__(self):
+ return '<{} at {:#x} {}>'.format(
+ type(self).__name__, id(self), self._format())
+
+ def __str__(self):
+ return '<{} {}>'.format(type(self).__name__, self._format())
+
+ def _format(self):
+ result = 'maxsize={!r}'.format(self._maxsize)
+ if getattr(self, '_queue', None):
+ result += ' _queue={!r}'.format(list(self._queue))
+ if self._getters:
+ result += ' _getters[{}]'.format(len(self._getters))
+ if self._putters:
+ result += ' _putters[{}]'.format(len(self._putters))
+ return result
+
+ def _consume_done_getters(self):
+ # Delete waiters at the head of the get() queue who've timed out.
+ while self._getters and self._getters[0].done():
+ self._getters.popleft()
+
+ def _consume_done_putters(self):
+ # Delete waiters at the head of the put() queue who've timed out.
+ while self._putters and self._putters[0][1].done():
+ self._putters.popleft()
+
+ def qsize(self):
+ """Number of items in the queue."""
+ return len(self._queue)
+
+ @property
+ def maxsize(self):
+ """Number of items allowed in the queue."""
+ return self._maxsize
+
+ def empty(self):
+ """Return True if the queue is empty, False otherwise."""
+ return not self._queue
+
+ def full(self):
+ """Return True if there are maxsize items in the queue.
+
+ Note: if the Queue was initialized with maxsize=0 (the default),
+ then full() is never True.
+ """
+ if self._maxsize <= 0:
+ return False
+ else:
+ return self.qsize() == self._maxsize
+
+ @coroutine
+ def put(self, item):
+ """Put an item into the queue.
+
+ If you yield from put(), wait until a free slot is available
+ before adding item.
+ """
+ self._consume_done_getters()
+ if self._getters:
+ assert not self._queue, (
+ 'queue non-empty, why are getters waiting?')
+
+ getter = self._getters.popleft()
+
+ # Use _put and _get instead of passing item straight to getter, in
+ # case a subclass has logic that must run (e.g. JoinableQueue).
+ self._put(item)
+ getter.set_result(self._get())
+
+ elif self._maxsize > 0 and self._maxsize == self.qsize():
+ waiter = futures.Future(loop=self._loop)
+
+ self._putters.append((item, waiter))
+ yield from waiter
+
+ else:
+ self._put(item)
+
+ def put_nowait(self, item):
+ """Put an item into the queue without blocking.
+
+ If no free slot is immediately available, raise QueueFull.
+ """
+ self._consume_done_getters()
+ if self._getters:
+ assert not self._queue, (
+ 'queue non-empty, why are getters waiting?')
+
+ getter = self._getters.popleft()
+
+ # Use _put and _get instead of passing item straight to getter, in
+ # case a subclass has logic that must run (e.g. JoinableQueue).
+ self._put(item)
+ getter.set_result(self._get())
+
+ elif self._maxsize > 0 and self._maxsize == self.qsize():
+ raise QueueFull
+ else:
+ self._put(item)
+
+ @coroutine
+ def get(self):
+ """Remove and return an item from the queue.
+
+ If you yield from get(), wait until a item is available.
+ """
+ self._consume_done_putters()
+ if self._putters:
+ assert self.full(), 'queue not full, why are putters waiting?'
+ item, putter = self._putters.popleft()
+ self._put(item)
+
+ # When a getter runs and frees up a slot so this putter can
+ # run, we need to defer the put for a tick to ensure that
+ # getters and putters alternate perfectly. See
+ # ChannelTest.test_wait.
+ self._loop.call_soon(putter.set_result, None)
+
+ return self._get()
+
+ elif self.qsize():
+ return self._get()
+ else:
+ waiter = futures.Future(loop=self._loop)
+
+ self._getters.append(waiter)
+ return (yield from waiter)
+
+ def get_nowait(self):
+ """Remove and return an item from the queue.
+
+ Return an item if one is immediately available, else raise QueueEmpty.
+ """
+ self._consume_done_putters()
+ if self._putters:
+ assert self.full(), 'queue not full, why are putters waiting?'
+ item, putter = self._putters.popleft()
+ self._put(item)
+ # Wake putter on next tick.
+ putter.set_result(None)
+
+ return self._get()
+
+ elif self.qsize():
+ return self._get()
+ else:
+ raise QueueEmpty
+
+
+class PriorityQueue(Queue):
+ """A subclass of Queue; retrieves entries in priority order (lowest first).
+
+ Entries are typically tuples of the form: (priority number, data).
+ """
+
+ def _init(self, maxsize):
+ self._queue = []
+
+ def _put(self, item, heappush=heapq.heappush):
+ heappush(self._queue, item)
+
+ def _get(self, heappop=heapq.heappop):
+ return heappop(self._queue)
+
+
+class LifoQueue(Queue):
+ """A subclass of Queue that retrieves most recently added entries first."""
+
+ def _init(self, maxsize):
+ self._queue = []
+
+ def _put(self, item):
+ self._queue.append(item)
+
+ def _get(self):
+ return self._queue.pop()
+
+
+class JoinableQueue(Queue):
+ """A subclass of Queue with task_done() and join() methods."""
+
+ def __init__(self, maxsize=0, *, loop=None):
+ super().__init__(maxsize=maxsize, loop=loop)
+ self._unfinished_tasks = 0
+ self._finished = locks.Event(loop=self._loop)
+ self._finished.set()
+
+ def _format(self):
+ result = Queue._format(self)
+ if self._unfinished_tasks:
+ result += ' tasks={}'.format(self._unfinished_tasks)
+ return result
+
+ def _put(self, item):
+ super()._put(item)
+ self._unfinished_tasks += 1
+ self._finished.clear()
+
+ def task_done(self):
+ """Indicate that a formerly enqueued task is complete.
+
+ Used by queue consumers. For each get() used to fetch a task,
+ a subsequent call to task_done() tells the queue that the processing
+ on the task is complete.
+
+ If a join() is currently blocking, it will resume when all items have
+ been processed (meaning that a task_done() call was received for every
+ item that had been put() into the queue).
+
+ Raises ValueError if called more times than there were items placed in
+ the queue.
+ """
+ if self._unfinished_tasks <= 0:
+ raise ValueError('task_done() called too many times')
+ self._unfinished_tasks -= 1
+ if self._unfinished_tasks == 0:
+ self._finished.set()
+
+ @coroutine
+ def join(self):
+ """Block until all items in the queue have been gotten and processed.
+
+ The count of unfinished tasks goes up whenever an item is added to the
+ queue. The count goes down whenever a consumer thread calls task_done()
+ to indicate that the item was retrieved and all work on it is complete.
+ When the count of unfinished tasks drops to zero, join() unblocks.
+ """
+ if self._unfinished_tasks > 0:
+ yield from self._finished.wait()
diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py
new file mode 100644
index 0000000..367c5fb
--- /dev/null
+++ b/Lib/asyncio/selector_events.py
@@ -0,0 +1,846 @@
+"""Event loop using a selector and related classes.
+
+A selector is a "notify-when-ready" multiplexer. For a subclass which
+also includes support for signal handling, see the unix_events sub-module.
+"""
+
+__all__ = ['BaseSelectorEventLoop']
+
+import collections
+import errno
+import socket
+try:
+ import ssl
+except ImportError: # pragma: no cover
+ ssl = None
+
+from . import base_events
+from . import constants
+from . import events
+from . import futures
+from . import selectors
+from . import transports
+from .log import logger
+
+
+class BaseSelectorEventLoop(base_events.BaseEventLoop):
+ """Selector event loop.
+
+ See events.EventLoop for API specification.
+ """
+
+ def __init__(self, selector=None):
+ super().__init__()
+
+ if selector is None:
+ selector = selectors.DefaultSelector()
+ logger.debug('Using selector: %s', selector.__class__.__name__)
+ self._selector = selector
+ self._make_self_pipe()
+
+ def _make_socket_transport(self, sock, protocol, waiter=None, *,
+ extra=None, server=None):
+ return _SelectorSocketTransport(self, sock, protocol, waiter,
+ extra, server)
+
+ def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter, *,
+ server_side=False, server_hostname=None,
+ extra=None, server=None):
+ return _SelectorSslTransport(
+ self, rawsock, protocol, sslcontext, waiter,
+ server_side, server_hostname, extra, server)
+
+ def _make_datagram_transport(self, sock, protocol,
+ address=None, extra=None):
+ return _SelectorDatagramTransport(self, sock, protocol, address, extra)
+
+ def close(self):
+ if self._selector is not None:
+ self._close_self_pipe()
+ self._selector.close()
+ self._selector = None
+ super().close()
+
+ def _socketpair(self):
+ raise NotImplementedError
+
+ def _close_self_pipe(self):
+ self.remove_reader(self._ssock.fileno())
+ self._ssock.close()
+ self._ssock = None
+ self._csock.close()
+ self._csock = None
+ self._internal_fds -= 1
+
+ def _make_self_pipe(self):
+ # A self-socket, really. :-)
+ self._ssock, self._csock = self._socketpair()
+ self._ssock.setblocking(False)
+ self._csock.setblocking(False)
+ self._internal_fds += 1
+ self.add_reader(self._ssock.fileno(), self._read_from_self)
+
+ def _read_from_self(self):
+ try:
+ self._ssock.recv(1)
+ except (BlockingIOError, InterruptedError):
+ pass
+
+ def _write_to_self(self):
+ try:
+ self._csock.send(b'x')
+ except (BlockingIOError, InterruptedError):
+ pass
+
+ def _start_serving(self, protocol_factory, sock,
+ sslcontext=None, server=None):
+ self.add_reader(sock.fileno(), self._accept_connection,
+ protocol_factory, sock, sslcontext, server)
+
+ def _accept_connection(self, protocol_factory, sock,
+ sslcontext=None, server=None):
+ try:
+ conn, addr = sock.accept()
+ conn.setblocking(False)
+ except (BlockingIOError, InterruptedError, ConnectionAbortedError):
+ pass # False alarm.
+ except OSError as exc:
+ # There's nowhere to send the error, so just log it.
+ # TODO: Someone will want an error handler for this.
+ if exc.errno in (errno.EMFILE, errno.ENFILE,
+ errno.ENOBUFS, errno.ENOMEM):
+ # Some platforms (e.g. Linux keep reporting the FD as
+ # ready, so we remove the read handler temporarily.
+ # We'll try again in a while.
+ self.call_exception_handler({
+ 'message': 'socket.accept() out of system resource',
+ 'exception': exc,
+ 'socket': sock,
+ })
+ self.remove_reader(sock.fileno())
+ self.call_later(constants.ACCEPT_RETRY_DELAY,
+ self._start_serving,
+ protocol_factory, sock, sslcontext, server)
+ else:
+ raise # The event loop will catch, log and ignore it.
+ else:
+ if sslcontext:
+ self._make_ssl_transport(
+ conn, protocol_factory(), sslcontext, None,
+ server_side=True, extra={'peername': addr}, server=server)
+ else:
+ self._make_socket_transport(
+ conn, protocol_factory(), extra={'peername': addr},
+ server=server)
+ # It's now up to the protocol to handle the connection.
+
+ def add_reader(self, fd, callback, *args):
+ """Add a reader callback."""
+ if self._selector is None:
+ raise RuntimeError('Event loop is closed')
+ handle = events.Handle(callback, args, self)
+ try:
+ key = self._selector.get_key(fd)
+ except KeyError:
+ self._selector.register(fd, selectors.EVENT_READ,
+ (handle, None))
+ else:
+ mask, (reader, writer) = key.events, key.data
+ self._selector.modify(fd, mask | selectors.EVENT_READ,
+ (handle, writer))
+ if reader is not None:
+ reader.cancel()
+
+ def remove_reader(self, fd):
+ """Remove a reader callback."""
+ if self._selector is None:
+ return False
+ try:
+ key = self._selector.get_key(fd)
+ except KeyError:
+ return False
+ else:
+ mask, (reader, writer) = key.events, key.data
+ mask &= ~selectors.EVENT_READ
+ if not mask:
+ self._selector.unregister(fd)
+ else:
+ self._selector.modify(fd, mask, (None, writer))
+
+ if reader is not None:
+ reader.cancel()
+ return True
+ else:
+ return False
+
+ def add_writer(self, fd, callback, *args):
+ """Add a writer callback.."""
+ if self._selector is None:
+ raise RuntimeError('Event loop is closed')
+ handle = events.Handle(callback, args, self)
+ try:
+ key = self._selector.get_key(fd)
+ except KeyError:
+ self._selector.register(fd, selectors.EVENT_WRITE,
+ (None, handle))
+ else:
+ mask, (reader, writer) = key.events, key.data
+ self._selector.modify(fd, mask | selectors.EVENT_WRITE,
+ (reader, handle))
+ if writer is not None:
+ writer.cancel()
+
+ def remove_writer(self, fd):
+ """Remove a writer callback."""
+ if self._selector is None:
+ return False
+ try:
+ key = self._selector.get_key(fd)
+ except KeyError:
+ return False
+ else:
+ mask, (reader, writer) = key.events, key.data
+ # Remove both writer and connector.
+ mask &= ~selectors.EVENT_WRITE
+ if not mask:
+ self._selector.unregister(fd)
+ else:
+ self._selector.modify(fd, mask, (reader, None))
+
+ if writer is not None:
+ writer.cancel()
+ return True
+ else:
+ return False
+
+ def sock_recv(self, sock, n):
+ """XXX"""
+ fut = futures.Future(loop=self)
+ self._sock_recv(fut, False, sock, n)
+ return fut
+
+ def _sock_recv(self, fut, registered, sock, n):
+ # _sock_recv() can add itself as an I/O callback if the operation can't
+ # be done immediately. Don't use it directly, call sock_recv().
+ fd = sock.fileno()
+ if registered:
+ # Remove the callback early. It should be rare that the
+ # selector says the fd is ready but the call still returns
+ # EAGAIN, and I am willing to take a hit in that case in
+ # order to simplify the common case.
+ self.remove_reader(fd)
+ if fut.cancelled():
+ return
+ try:
+ data = sock.recv(n)
+ except (BlockingIOError, InterruptedError):
+ self.add_reader(fd, self._sock_recv, fut, True, sock, n)
+ except Exception as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result(data)
+
+ def sock_sendall(self, sock, data):
+ """XXX"""
+ fut = futures.Future(loop=self)
+ if data:
+ self._sock_sendall(fut, False, sock, data)
+ else:
+ fut.set_result(None)
+ return fut
+
+ def _sock_sendall(self, fut, registered, sock, data):
+ fd = sock.fileno()
+
+ if registered:
+ self.remove_writer(fd)
+ if fut.cancelled():
+ return
+
+ try:
+ n = sock.send(data)
+ except (BlockingIOError, InterruptedError):
+ n = 0
+ except Exception as exc:
+ fut.set_exception(exc)
+ return
+
+ if n == len(data):
+ fut.set_result(None)
+ else:
+ if n:
+ data = data[n:]
+ self.add_writer(fd, self._sock_sendall, fut, True, sock, data)
+
+ def sock_connect(self, sock, address):
+ """XXX"""
+ fut = futures.Future(loop=self)
+ try:
+ base_events._check_resolved_address(sock, address)
+ except ValueError as err:
+ fut.set_exception(err)
+ else:
+ self._sock_connect(fut, False, sock, address)
+ return fut
+
+ def _sock_connect(self, fut, registered, sock, address):
+ fd = sock.fileno()
+ if registered:
+ self.remove_writer(fd)
+ if fut.cancelled():
+ return
+ try:
+ if not registered:
+ # First time around.
+ sock.connect(address)
+ else:
+ err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+ if err != 0:
+ # Jump to the except clause below.
+ raise OSError(err, 'Connect call failed %s' % (address,))
+ except (BlockingIOError, InterruptedError):
+ self.add_writer(fd, self._sock_connect, fut, True, sock, address)
+ except Exception as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result(None)
+
+ def sock_accept(self, sock):
+ """XXX"""
+ fut = futures.Future(loop=self)
+ self._sock_accept(fut, False, sock)
+ return fut
+
+ def _sock_accept(self, fut, registered, sock):
+ fd = sock.fileno()
+ if registered:
+ self.remove_reader(fd)
+ if fut.cancelled():
+ return
+ try:
+ conn, address = sock.accept()
+ conn.setblocking(False)
+ except (BlockingIOError, InterruptedError):
+ self.add_reader(fd, self._sock_accept, fut, True, sock)
+ except Exception as exc:
+ fut.set_exception(exc)
+ else:
+ fut.set_result((conn, address))
+
+ def _process_events(self, event_list):
+ for key, mask in event_list:
+ fileobj, (reader, writer) = key.fileobj, key.data
+ if mask & selectors.EVENT_READ and reader is not None:
+ if reader._cancelled:
+ self.remove_reader(fileobj)
+ else:
+ self._add_callback(reader)
+ if mask & selectors.EVENT_WRITE and writer is not None:
+ if writer._cancelled:
+ self.remove_writer(fileobj)
+ else:
+ self._add_callback(writer)
+
+ def _stop_serving(self, sock):
+ self.remove_reader(sock.fileno())
+ sock.close()
+
+
+class _SelectorTransport(transports._FlowControlMixin,
+ transports.Transport):
+
+ max_size = 256 * 1024 # Buffer size passed to recv().
+
+ _buffer_factory = bytearray # Constructs initial value for self._buffer.
+
+ def __init__(self, loop, sock, protocol, extra, server=None):
+ super().__init__(extra)
+ self._extra['socket'] = sock
+ self._extra['sockname'] = sock.getsockname()
+ if 'peername' not in self._extra:
+ try:
+ self._extra['peername'] = sock.getpeername()
+ except socket.error:
+ self._extra['peername'] = None
+ self._loop = loop
+ self._sock = sock
+ self._sock_fd = sock.fileno()
+ self._protocol = protocol
+ self._server = server
+ self._buffer = self._buffer_factory()
+ self._conn_lost = 0 # Set when call to connection_lost scheduled.
+ self._closing = False # Set when close() called.
+ if self._server is not None:
+ self._server.attach(self)
+
+ def abort(self):
+ self._force_close(None)
+
+ def close(self):
+ if self._closing:
+ return
+ self._closing = True
+ self._loop.remove_reader(self._sock_fd)
+ if not self._buffer:
+ self._conn_lost += 1
+ self._loop.call_soon(self._call_connection_lost, None)
+
+ def _fatal_error(self, exc, message='Fatal error on transport'):
+ # Should be called from exception handler only.
+ if not isinstance(exc, (BrokenPipeError, ConnectionResetError)):
+ self._loop.call_exception_handler({
+ 'message': message,
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+ self._force_close(exc)
+
+ def _force_close(self, exc):
+ if self._conn_lost:
+ return
+ if self._buffer:
+ self._buffer.clear()
+ self._loop.remove_writer(self._sock_fd)
+ if not self._closing:
+ self._closing = True
+ self._loop.remove_reader(self._sock_fd)
+ self._conn_lost += 1
+ self._loop.call_soon(self._call_connection_lost, exc)
+
+ def _call_connection_lost(self, exc):
+ try:
+ self._protocol.connection_lost(exc)
+ finally:
+ self._sock.close()
+ self._sock = None
+ self._protocol = None
+ self._loop = None
+ server = self._server
+ if server is not None:
+ server.detach(self)
+ self._server = None
+
+ def get_write_buffer_size(self):
+ return len(self._buffer)
+
+
+class _SelectorSocketTransport(_SelectorTransport):
+
+ def __init__(self, loop, sock, protocol, waiter=None,
+ extra=None, server=None):
+ super().__init__(loop, sock, protocol, extra, server)
+ self._eof = False
+ self._paused = False
+
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+ self._loop.call_soon(self._protocol.connection_made, self)
+ if waiter is not None:
+ self._loop.call_soon(waiter.set_result, None)
+
+ def pause_reading(self):
+ if self._closing:
+ raise RuntimeError('Cannot pause_reading() when closing')
+ if self._paused:
+ raise RuntimeError('Already paused')
+ self._paused = True
+ self._loop.remove_reader(self._sock_fd)
+
+ def resume_reading(self):
+ if not self._paused:
+ raise RuntimeError('Not paused')
+ self._paused = False
+ if self._closing:
+ return
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+
+ def _read_ready(self):
+ try:
+ data = self._sock.recv(self.max_size)
+ except (BlockingIOError, InterruptedError):
+ pass
+ except Exception as exc:
+ self._fatal_error(exc, 'Fatal read error on socket transport')
+ else:
+ if data:
+ self._protocol.data_received(data)
+ else:
+ keep_open = self._protocol.eof_received()
+ if keep_open:
+ # We're keeping the connection open so the
+ # protocol can write more, but we still can't
+ # receive more, so remove the reader callback.
+ self._loop.remove_reader(self._sock_fd)
+ else:
+ self.close()
+
+ def write(self, data):
+ if not isinstance(data, (bytes, bytearray, memoryview)):
+ raise TypeError('data argument must be byte-ish (%r)',
+ type(data))
+ if self._eof:
+ raise RuntimeError('Cannot call write() after write_eof()')
+ if not data:
+ return
+
+ if self._conn_lost:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('socket.send() raised exception.')
+ self._conn_lost += 1
+ return
+
+ if not self._buffer:
+ # Optimization: try to send now.
+ try:
+ n = self._sock.send(data)
+ except (BlockingIOError, InterruptedError):
+ pass
+ except Exception as exc:
+ self._fatal_error(exc, 'Fatal write error on socket transport')
+ return
+ else:
+ data = data[n:]
+ if not data:
+ return
+ # Not all was written; register write handler.
+ self._loop.add_writer(self._sock_fd, self._write_ready)
+
+ # Add it to the buffer.
+ self._buffer.extend(data)
+ self._maybe_pause_protocol()
+
+ def _write_ready(self):
+ assert self._buffer, 'Data should not be empty'
+
+ try:
+ n = self._sock.send(self._buffer)
+ except (BlockingIOError, InterruptedError):
+ pass
+ except Exception as exc:
+ self._loop.remove_writer(self._sock_fd)
+ self._buffer.clear()
+ self._fatal_error(exc, 'Fatal write error on socket transport')
+ else:
+ if n:
+ del self._buffer[:n]
+ self._maybe_resume_protocol() # May append to buffer.
+ if not self._buffer:
+ self._loop.remove_writer(self._sock_fd)
+ if self._closing:
+ self._call_connection_lost(None)
+ elif self._eof:
+ self._sock.shutdown(socket.SHUT_WR)
+
+ def write_eof(self):
+ if self._eof:
+ return
+ self._eof = True
+ if not self._buffer:
+ self._sock.shutdown(socket.SHUT_WR)
+
+ def can_write_eof(self):
+ return True
+
+
+class _SelectorSslTransport(_SelectorTransport):
+
+ _buffer_factory = bytearray
+
+ def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None,
+ server_side=False, server_hostname=None,
+ extra=None, server=None):
+ if ssl is None:
+ raise RuntimeError('stdlib ssl module not available')
+
+ if server_side:
+ if not sslcontext:
+ raise ValueError('Server side ssl needs a valid SSLContext')
+ else:
+ if not sslcontext:
+ # Client side may pass ssl=True to use a default
+ # context; in that case the sslcontext passed is None.
+ # The default is the same as used by urllib with
+ # cadefault=True.
+ if hasattr(ssl, '_create_stdlib_context'):
+ sslcontext = ssl._create_stdlib_context(
+ cert_reqs=ssl.CERT_REQUIRED,
+ check_hostname=bool(server_hostname))
+ else:
+ # Fallback for Python 3.3.
+ sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext.options |= ssl.OP_NO_SSLv2
+ sslcontext.set_default_verify_paths()
+ sslcontext.verify_mode = ssl.CERT_REQUIRED
+
+ wrap_kwargs = {
+ 'server_side': server_side,
+ 'do_handshake_on_connect': False,
+ }
+ if server_hostname and not server_side and ssl.HAS_SNI:
+ wrap_kwargs['server_hostname'] = server_hostname
+ sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs)
+
+ super().__init__(loop, sslsock, protocol, extra, server)
+
+ self._server_hostname = server_hostname
+ self._waiter = waiter
+ self._rawsock = rawsock
+ self._sslcontext = sslcontext
+ self._paused = False
+
+ # SSL-specific extra info. (peercert is set later)
+ self._extra.update(sslcontext=sslcontext)
+
+ self._on_handshake()
+
+ def _on_handshake(self):
+ try:
+ self._sock.do_handshake()
+ except ssl.SSLWantReadError:
+ self._loop.add_reader(self._sock_fd, self._on_handshake)
+ return
+ except ssl.SSLWantWriteError:
+ self._loop.add_writer(self._sock_fd, self._on_handshake)
+ return
+ except Exception as exc:
+ self._loop.remove_reader(self._sock_fd)
+ self._loop.remove_writer(self._sock_fd)
+ self._sock.close()
+ if self._waiter is not None:
+ self._waiter.set_exception(exc)
+ return
+ except BaseException as exc:
+ self._loop.remove_reader(self._sock_fd)
+ self._loop.remove_writer(self._sock_fd)
+ self._sock.close()
+ if self._waiter is not None:
+ self._waiter.set_exception(exc)
+ raise
+
+ self._loop.remove_reader(self._sock_fd)
+ self._loop.remove_writer(self._sock_fd)
+
+ peercert = self._sock.getpeercert()
+ if not hasattr(self._sslcontext, 'check_hostname'):
+ # Verify hostname if requested, Python 3.4+ uses check_hostname
+ # and checks the hostname in do_handshake()
+ if (self._server_hostname and
+ self._sslcontext.verify_mode != ssl.CERT_NONE):
+ try:
+ ssl.match_hostname(peercert, self._server_hostname)
+ except Exception as exc:
+ self._sock.close()
+ if self._waiter is not None:
+ self._waiter.set_exception(exc)
+ return
+
+ # Add extra info that becomes available after handshake.
+ self._extra.update(peercert=peercert,
+ cipher=self._sock.cipher(),
+ compression=self._sock.compression(),
+ )
+
+ self._read_wants_write = False
+ self._write_wants_read = False
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+ self._loop.call_soon(self._protocol.connection_made, self)
+ if self._waiter is not None:
+ self._loop.call_soon(self._waiter.set_result, None)
+
+ def pause_reading(self):
+ # XXX This is a bit icky, given the comment at the top of
+ # _read_ready(). Is it possible to evoke a deadlock? I don't
+ # know, although it doesn't look like it; write() will still
+ # accept more data for the buffer and eventually the app will
+ # call resume_reading() again, and things will flow again.
+
+ if self._closing:
+ raise RuntimeError('Cannot pause_reading() when closing')
+ if self._paused:
+ raise RuntimeError('Already paused')
+ self._paused = True
+ self._loop.remove_reader(self._sock_fd)
+
+ def resume_reading(self):
+ if not self._paused:
+ raise ('Not paused')
+ self._paused = False
+ if self._closing:
+ return
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+
+ def _read_ready(self):
+ if self._write_wants_read:
+ self._write_wants_read = False
+ self._write_ready()
+
+ if self._buffer:
+ self._loop.add_writer(self._sock_fd, self._write_ready)
+
+ try:
+ data = self._sock.recv(self.max_size)
+ except (BlockingIOError, InterruptedError, ssl.SSLWantReadError):
+ pass
+ except ssl.SSLWantWriteError:
+ self._read_wants_write = True
+ self._loop.remove_reader(self._sock_fd)
+ self._loop.add_writer(self._sock_fd, self._write_ready)
+ except Exception as exc:
+ self._fatal_error(exc, 'Fatal read error on SSL transport')
+ else:
+ if data:
+ self._protocol.data_received(data)
+ else:
+ try:
+ keep_open = self._protocol.eof_received()
+ if keep_open:
+ logger.warning('returning true from eof_received() '
+ 'has no effect when using ssl')
+ finally:
+ self.close()
+
+ def _write_ready(self):
+ if self._read_wants_write:
+ self._read_wants_write = False
+ self._read_ready()
+
+ if not (self._paused or self._closing):
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+
+ if self._buffer:
+ try:
+ n = self._sock.send(self._buffer)
+ except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError):
+ n = 0
+ except ssl.SSLWantReadError:
+ n = 0
+ self._loop.remove_writer(self._sock_fd)
+ self._write_wants_read = True
+ except Exception as exc:
+ self._loop.remove_writer(self._sock_fd)
+ self._buffer.clear()
+ self._fatal_error(exc, 'Fatal write error on SSL transport')
+ return
+
+ if n:
+ del self._buffer[:n]
+
+ self._maybe_resume_protocol() # May append to buffer.
+
+ if not self._buffer:
+ self._loop.remove_writer(self._sock_fd)
+ if self._closing:
+ self._call_connection_lost(None)
+
+ def write(self, data):
+ if not isinstance(data, (bytes, bytearray, memoryview)):
+ raise TypeError('data argument must be byte-ish (%r)',
+ type(data))
+ if not data:
+ return
+
+ if self._conn_lost:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('socket.send() raised exception.')
+ self._conn_lost += 1
+ return
+
+ if not self._buffer:
+ self._loop.add_writer(self._sock_fd, self._write_ready)
+
+ # Add it to the buffer.
+ self._buffer.extend(data)
+ self._maybe_pause_protocol()
+
+ def can_write_eof(self):
+ return False
+
+
+class _SelectorDatagramTransport(_SelectorTransport):
+
+ _buffer_factory = collections.deque
+
+ def __init__(self, loop, sock, protocol, address=None, extra=None):
+ super().__init__(loop, sock, protocol, extra)
+ self._address = address
+ self._loop.add_reader(self._sock_fd, self._read_ready)
+ self._loop.call_soon(self._protocol.connection_made, self)
+
+ def get_write_buffer_size(self):
+ return sum(len(data) for data, _ in self._buffer)
+
+ def _read_ready(self):
+ try:
+ data, addr = self._sock.recvfrom(self.max_size)
+ except (BlockingIOError, InterruptedError):
+ pass
+ except OSError as exc:
+ self._protocol.error_received(exc)
+ except Exception as exc:
+ self._fatal_error(exc, 'Fatal read error on datagram transport')
+ else:
+ self._protocol.datagram_received(data, addr)
+
+ def sendto(self, data, addr=None):
+ if not isinstance(data, (bytes, bytearray, memoryview)):
+ raise TypeError('data argument must be byte-ish (%r)',
+ type(data))
+ if not data:
+ return
+
+ if self._address and addr not in (None, self._address):
+ raise ValueError('Invalid address: must be None or %s' %
+ (self._address,))
+
+ if self._conn_lost and self._address:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('socket.send() raised exception.')
+ self._conn_lost += 1
+ return
+
+ if not self._buffer:
+ # Attempt to send it right away first.
+ try:
+ if self._address:
+ self._sock.send(data)
+ else:
+ self._sock.sendto(data, addr)
+ return
+ except (BlockingIOError, InterruptedError):
+ self._loop.add_writer(self._sock_fd, self._sendto_ready)
+ except OSError as exc:
+ self._protocol.error_received(exc)
+ return
+ except Exception as exc:
+ self._fatal_error(exc,
+ 'Fatal write error on datagram transport')
+ return
+
+ # Ensure that what we buffer is immutable.
+ self._buffer.append((bytes(data), addr))
+ self._maybe_pause_protocol()
+
+ def _sendto_ready(self):
+ while self._buffer:
+ data, addr = self._buffer.popleft()
+ try:
+ if self._address:
+ self._sock.send(data)
+ else:
+ self._sock.sendto(data, addr)
+ except (BlockingIOError, InterruptedError):
+ self._buffer.appendleft((data, addr)) # Try again later.
+ break
+ except OSError as exc:
+ self._protocol.error_received(exc)
+ return
+ except Exception as exc:
+ self._fatal_error(exc,
+ 'Fatal write error on datagram transport')
+ return
+
+ self._maybe_resume_protocol() # May append to buffer.
+ if not self._buffer:
+ self._loop.remove_writer(self._sock_fd)
+ if self._closing:
+ self._call_connection_lost(None)
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
new file mode 100644
index 0000000..27d595f
--- /dev/null
+++ b/Lib/asyncio/streams.py
@@ -0,0 +1,468 @@
+"""Stream-related things."""
+
+__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol',
+ 'open_connection', 'start_server',
+ 'IncompleteReadError',
+ ]
+
+import socket
+
+if hasattr(socket, 'AF_UNIX'):
+ __all__.extend(['open_unix_connection', 'start_unix_server'])
+
+from . import events
+from . import futures
+from . import protocols
+from . import tasks
+
+
+_DEFAULT_LIMIT = 2**16
+
+
+class IncompleteReadError(EOFError):
+ """
+ Incomplete read error. Attributes:
+
+ - partial: read bytes string before the end of stream was reached
+ - expected: total number of expected bytes
+ """
+ def __init__(self, partial, expected):
+ EOFError.__init__(self, "%s bytes read on a total of %s expected bytes"
+ % (len(partial), expected))
+ self.partial = partial
+ self.expected = expected
+
+
+@tasks.coroutine
+def open_connection(host=None, port=None, *,
+ loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ """A wrapper for create_connection() returning a (reader, writer) pair.
+
+ The reader returned is a StreamReader instance; the writer is a
+ StreamWriter instance.
+
+ The arguments are all the usual arguments to create_connection()
+ except protocol_factory; most common are positional host and port,
+ with various optional keyword arguments following.
+
+ Additional optional keyword arguments are loop (to set the event loop
+ instance to use) and limit (to set the buffer limit passed to the
+ StreamReader).
+
+ (If you want to customize the StreamReader and/or
+ StreamReaderProtocol classes, just copy the code -- there's
+ really nothing special here except some convenience.)
+ """
+ if loop is None:
+ loop = events.get_event_loop()
+ reader = StreamReader(limit=limit, loop=loop)
+ protocol = StreamReaderProtocol(reader, loop=loop)
+ transport, _ = yield from loop.create_connection(
+ lambda: protocol, host, port, **kwds)
+ writer = StreamWriter(transport, protocol, reader, loop)
+ return reader, writer
+
+
+@tasks.coroutine
+def start_server(client_connected_cb, host=None, port=None, *,
+ loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ """Start a socket server, call back for each client connected.
+
+ The first parameter, `client_connected_cb`, takes two parameters:
+ client_reader, client_writer. client_reader is a StreamReader
+ object, while client_writer is a StreamWriter object. This
+ parameter can either be a plain callback function or a coroutine;
+ if it is a coroutine, it will be automatically converted into a
+ Task.
+
+ The rest of the arguments are all the usual arguments to
+ loop.create_server() except protocol_factory; most common are
+ positional host and port, with various optional keyword arguments
+ following. The return value is the same as loop.create_server().
+
+ Additional optional keyword arguments are loop (to set the event loop
+ instance to use) and limit (to set the buffer limit passed to the
+ StreamReader).
+
+ The return value is the same as loop.create_server(), i.e. a
+ Server object which can be used to stop the service.
+ """
+ if loop is None:
+ loop = events.get_event_loop()
+
+ def factory():
+ reader = StreamReader(limit=limit, loop=loop)
+ protocol = StreamReaderProtocol(reader, client_connected_cb,
+ loop=loop)
+ return protocol
+
+ return (yield from loop.create_server(factory, host, port, **kwds))
+
+
+if hasattr(socket, 'AF_UNIX'):
+ # UNIX Domain Sockets are supported on this platform
+
+ @tasks.coroutine
+ def open_unix_connection(path=None, *,
+ loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ """Similar to `open_connection` but works with UNIX Domain Sockets."""
+ if loop is None:
+ loop = events.get_event_loop()
+ reader = StreamReader(limit=limit, loop=loop)
+ protocol = StreamReaderProtocol(reader, loop=loop)
+ transport, _ = yield from loop.create_unix_connection(
+ lambda: protocol, path, **kwds)
+ writer = StreamWriter(transport, protocol, reader, loop)
+ return reader, writer
+
+
+ @tasks.coroutine
+ def start_unix_server(client_connected_cb, path=None, *,
+ loop=None, limit=_DEFAULT_LIMIT, **kwds):
+ """Similar to `start_server` but works with UNIX Domain Sockets."""
+ if loop is None:
+ loop = events.get_event_loop()
+
+ def factory():
+ reader = StreamReader(limit=limit, loop=loop)
+ protocol = StreamReaderProtocol(reader, client_connected_cb,
+ loop=loop)
+ return protocol
+
+ return (yield from loop.create_unix_server(factory, path, **kwds))
+
+
+class FlowControlMixin(protocols.Protocol):
+ """Reusable flow control logic for StreamWriter.drain().
+
+ This implements the protocol methods pause_writing(),
+ resume_reading() and connection_lost(). If the subclass overrides
+ these it must call the super methods.
+
+ StreamWriter.drain() must check for error conditions and then call
+ _make_drain_waiter(), which will return either () or a Future
+ depending on the paused state.
+ """
+
+ def __init__(self, loop=None):
+ self._loop = loop # May be None; we may never need it.
+ self._paused = False
+ self._drain_waiter = None
+
+ def pause_writing(self):
+ assert not self._paused
+ self._paused = True
+
+ def resume_writing(self):
+ assert self._paused
+ self._paused = False
+ waiter = self._drain_waiter
+ if waiter is not None:
+ self._drain_waiter = None
+ if not waiter.done():
+ waiter.set_result(None)
+
+ def connection_lost(self, exc):
+ # Wake up the writer if currently paused.
+ if not self._paused:
+ return
+ waiter = self._drain_waiter
+ if waiter is None:
+ return
+ self._drain_waiter = None
+ if waiter.done():
+ return
+ if exc is None:
+ waiter.set_result(None)
+ else:
+ waiter.set_exception(exc)
+
+ def _make_drain_waiter(self):
+ if not self._paused:
+ return ()
+ waiter = self._drain_waiter
+ assert waiter is None or waiter.cancelled()
+ waiter = futures.Future(loop=self._loop)
+ self._drain_waiter = waiter
+ return waiter
+
+
+class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
+ """Helper class to adapt between Protocol and StreamReader.
+
+ (This is a helper class instead of making StreamReader itself a
+ Protocol subclass, because the StreamReader has other potential
+ uses, and to prevent the user of the StreamReader to accidentally
+ call inappropriate methods of the protocol.)
+ """
+
+ def __init__(self, stream_reader, client_connected_cb=None, loop=None):
+ super().__init__(loop=loop)
+ self._stream_reader = stream_reader
+ self._stream_writer = None
+ self._client_connected_cb = client_connected_cb
+
+ def connection_made(self, transport):
+ self._stream_reader.set_transport(transport)
+ if self._client_connected_cb is not None:
+ self._stream_writer = StreamWriter(transport, self,
+ self._stream_reader,
+ self._loop)
+ res = self._client_connected_cb(self._stream_reader,
+ self._stream_writer)
+ if tasks.iscoroutine(res):
+ tasks.Task(res, loop=self._loop)
+
+ def connection_lost(self, exc):
+ if exc is None:
+ self._stream_reader.feed_eof()
+ else:
+ self._stream_reader.set_exception(exc)
+ super().connection_lost(exc)
+
+ def data_received(self, data):
+ self._stream_reader.feed_data(data)
+
+ def eof_received(self):
+ self._stream_reader.feed_eof()
+
+
+class StreamWriter:
+ """Wraps a Transport.
+
+ This exposes write(), writelines(), [can_]write_eof(),
+ get_extra_info() and close(). It adds drain() which returns an
+ optional Future on which you can wait for flow control. It also
+ adds a transport property which references the Transport
+ directly.
+ """
+
+ def __init__(self, transport, protocol, reader, loop):
+ self._transport = transport
+ self._protocol = protocol
+ self._reader = reader
+ self._loop = loop
+
+ @property
+ def transport(self):
+ return self._transport
+
+ def write(self, data):
+ self._transport.write(data)
+
+ def writelines(self, data):
+ self._transport.writelines(data)
+
+ def write_eof(self):
+ return self._transport.write_eof()
+
+ def can_write_eof(self):
+ return self._transport.can_write_eof()
+
+ def close(self):
+ return self._transport.close()
+
+ def get_extra_info(self, name, default=None):
+ return self._transport.get_extra_info(name, default)
+
+ def drain(self):
+ """This method has an unusual return value.
+
+ The intended use is to write
+
+ w.write(data)
+ yield from w.drain()
+
+ When there's nothing to wait for, drain() returns (), and the
+ yield-from continues immediately. When the transport buffer
+ is full (the protocol is paused), drain() creates and returns
+ a Future and the yield-from will block until that Future is
+ completed, which will happen when the buffer is (partially)
+ drained and the protocol is resumed.
+ """
+ if self._reader is not None and self._reader._exception is not None:
+ raise self._reader._exception
+ if self._transport._conn_lost: # Uses private variable.
+ raise ConnectionResetError('Connection lost')
+ return self._protocol._make_drain_waiter()
+
+
+class StreamReader:
+
+ def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
+ # The line length limit is a security feature;
+ # it also doubles as half the buffer limit.
+ self._limit = limit
+ if loop is None:
+ loop = events.get_event_loop()
+ self._loop = loop
+ self._buffer = bytearray()
+ self._eof = False # Whether we're done.
+ self._waiter = None # A future.
+ self._exception = None
+ self._transport = None
+ self._paused = False
+
+ def exception(self):
+ return self._exception
+
+ def set_exception(self, exc):
+ self._exception = exc
+
+ waiter = self._waiter
+ if waiter is not None:
+ self._waiter = None
+ if not waiter.cancelled():
+ waiter.set_exception(exc)
+
+ def set_transport(self, transport):
+ assert self._transport is None, 'Transport already set'
+ self._transport = transport
+
+ def _maybe_resume_transport(self):
+ if self._paused and len(self._buffer) <= self._limit:
+ self._paused = False
+ self._transport.resume_reading()
+
+ def feed_eof(self):
+ self._eof = True
+ waiter = self._waiter
+ if waiter is not None:
+ self._waiter = None
+ if not waiter.cancelled():
+ waiter.set_result(True)
+
+ def at_eof(self):
+ """Return True if the buffer is empty and 'feed_eof' was called."""
+ return self._eof and not self._buffer
+
+ def feed_data(self, data):
+ assert not self._eof, 'feed_data after feed_eof'
+
+ if not data:
+ return
+
+ self._buffer.extend(data)
+
+ waiter = self._waiter
+ if waiter is not None:
+ self._waiter = None
+ if not waiter.cancelled():
+ waiter.set_result(False)
+
+ if (self._transport is not None and
+ not self._paused and
+ len(self._buffer) > 2*self._limit):
+ try:
+ self._transport.pause_reading()
+ except NotImplementedError:
+ # The transport can't be paused.
+ # We'll just have to buffer all data.
+ # Forget the transport so we don't keep trying.
+ self._transport = None
+ else:
+ self._paused = True
+
+ def _create_waiter(self, func_name):
+ # StreamReader uses a future to link the protocol feed_data() method
+ # to a read coroutine. Running two read coroutines at the same time
+ # would have an unexpected behaviour. It would not possible to know
+ # which coroutine would get the next data.
+ if self._waiter is not None:
+ raise RuntimeError('%s() called while another coroutine is '
+ 'already waiting for incoming data' % func_name)
+ return futures.Future(loop=self._loop)
+
+ @tasks.coroutine
+ def readline(self):
+ if self._exception is not None:
+ raise self._exception
+
+ line = bytearray()
+ not_enough = True
+
+ while not_enough:
+ while self._buffer and not_enough:
+ ichar = self._buffer.find(b'\n')
+ if ichar < 0:
+ line.extend(self._buffer)
+ self._buffer.clear()
+ else:
+ ichar += 1
+ line.extend(self._buffer[:ichar])
+ del self._buffer[:ichar]
+ not_enough = False
+
+ if len(line) > self._limit:
+ self._maybe_resume_transport()
+ raise ValueError('Line is too long')
+
+ if self._eof:
+ break
+
+ if not_enough:
+ self._waiter = self._create_waiter('readline')
+ try:
+ yield from self._waiter
+ finally:
+ self._waiter = None
+
+ self._maybe_resume_transport()
+ return bytes(line)
+
+ @tasks.coroutine
+ def read(self, n=-1):
+ if self._exception is not None:
+ raise self._exception
+
+ if not n:
+ return b''
+
+ if n < 0:
+ while not self._eof:
+ self._waiter = self._create_waiter('read')
+ try:
+ yield from self._waiter
+ finally:
+ self._waiter = None
+ else:
+ if not self._buffer and not self._eof:
+ self._waiter = self._create_waiter('read')
+ try:
+ yield from self._waiter
+ finally:
+ self._waiter = None
+
+ if n < 0 or len(self._buffer) <= n:
+ data = bytes(self._buffer)
+ self._buffer.clear()
+ else:
+ # n > 0 and len(self._buffer) > n
+ data = bytes(self._buffer[:n])
+ del self._buffer[:n]
+
+ self._maybe_resume_transport()
+ return data
+
+ @tasks.coroutine
+ def readexactly(self, n):
+ if self._exception is not None:
+ raise self._exception
+
+ # There used to be "optimized" code here. It created its own
+ # Future and waited until self._buffer had at least the n
+ # bytes, then called read(n). Unfortunately, this could pause
+ # the transport if the argument was larger than the pause
+ # limit (which is twice self._limit). So now we just read()
+ # into a local buffer.
+
+ blocks = []
+ while n > 0:
+ block = yield from self.read(n)
+ if not block:
+ partial = b''.join(blocks)
+ raise IncompleteReadError(partial, len(partial) + n)
+ blocks.append(block)
+ n -= len(block)
+
+ return b''.join(blocks)
diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py
new file mode 100644
index 0000000..414e023
--- /dev/null
+++ b/Lib/asyncio/subprocess.py
@@ -0,0 +1,195 @@
+__all__ = ['create_subprocess_exec', 'create_subprocess_shell']
+
+import collections
+import subprocess
+
+from . import events
+from . import futures
+from . import protocols
+from . import streams
+from . import tasks
+
+
+PIPE = subprocess.PIPE
+STDOUT = subprocess.STDOUT
+DEVNULL = subprocess.DEVNULL
+
+
+class SubprocessStreamProtocol(streams.FlowControlMixin,
+ protocols.SubprocessProtocol):
+ """Like StreamReaderProtocol, but for a subprocess."""
+
+ def __init__(self, limit, loop):
+ super().__init__(loop=loop)
+ self._limit = limit
+ self.stdin = self.stdout = self.stderr = None
+ self.waiter = futures.Future(loop=loop)
+ self._waiters = collections.deque()
+ self._transport = None
+
+ def connection_made(self, transport):
+ self._transport = transport
+ if transport.get_pipe_transport(1):
+ self.stdout = streams.StreamReader(limit=self._limit,
+ loop=self._loop)
+ if transport.get_pipe_transport(2):
+ self.stderr = streams.StreamReader(limit=self._limit,
+ loop=self._loop)
+ stdin = transport.get_pipe_transport(0)
+ if stdin is not None:
+ self.stdin = streams.StreamWriter(stdin,
+ protocol=self,
+ reader=None,
+ loop=self._loop)
+ self.waiter.set_result(None)
+
+ def pipe_data_received(self, fd, data):
+ if fd == 1:
+ reader = self.stdout
+ elif fd == 2:
+ reader = self.stderr
+ else:
+ reader = None
+ if reader is not None:
+ reader.feed_data(data)
+
+ def pipe_connection_lost(self, fd, exc):
+ if fd == 0:
+ pipe = self.stdin
+ if pipe is not None:
+ pipe.close()
+ self.connection_lost(exc)
+ return
+ if fd == 1:
+ reader = self.stdout
+ elif fd == 2:
+ reader = self.stderr
+ else:
+ reader = None
+ if reader != None:
+ if exc is None:
+ reader.feed_eof()
+ else:
+ reader.set_exception(exc)
+
+ def process_exited(self):
+ # wake up futures waiting for wait()
+ returncode = self._transport.get_returncode()
+ while self._waiters:
+ waiter = self._waiters.popleft()
+ waiter.set_result(returncode)
+
+
+class Process:
+ def __init__(self, transport, protocol, loop):
+ self._transport = transport
+ self._protocol = protocol
+ self._loop = loop
+ self.stdin = protocol.stdin
+ self.stdout = protocol.stdout
+ self.stderr = protocol.stderr
+ self.pid = transport.get_pid()
+
+ @property
+ def returncode(self):
+ return self._transport.get_returncode()
+
+ @tasks.coroutine
+ def wait(self):
+ """Wait until the process exit and return the process return code."""
+ returncode = self._transport.get_returncode()
+ if returncode is not None:
+ return returncode
+
+ waiter = futures.Future(loop=self._loop)
+ self._protocol._waiters.append(waiter)
+ yield from waiter
+ return waiter.result()
+
+ def _check_alive(self):
+ if self._transport.get_returncode() is not None:
+ raise ProcessLookupError()
+
+ def send_signal(self, signal):
+ self._check_alive()
+ self._transport.send_signal(signal)
+
+ def terminate(self):
+ self._check_alive()
+ self._transport.terminate()
+
+ def kill(self):
+ self._check_alive()
+ self._transport.kill()
+
+ @tasks.coroutine
+ def _feed_stdin(self, input):
+ self.stdin.write(input)
+ yield from self.stdin.drain()
+ self.stdin.close()
+
+ @tasks.coroutine
+ def _noop(self):
+ return None
+
+ @tasks.coroutine
+ def _read_stream(self, fd):
+ transport = self._transport.get_pipe_transport(fd)
+ if fd == 2:
+ stream = self.stderr
+ else:
+ assert fd == 1
+ stream = self.stdout
+ output = yield from stream.read()
+ transport.close()
+ return output
+
+ @tasks.coroutine
+ def communicate(self, input=None):
+ if input:
+ stdin = self._feed_stdin(input)
+ else:
+ stdin = self._noop()
+ if self.stdout is not None:
+ stdout = self._read_stream(1)
+ else:
+ stdout = self._noop()
+ if self.stderr is not None:
+ stderr = self._read_stream(2)
+ else:
+ stderr = self._noop()
+ stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr,
+ loop=self._loop)
+ yield from self.wait()
+ return (stdout, stderr)
+
+
+@tasks.coroutine
+def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None,
+ loop=None, limit=streams._DEFAULT_LIMIT, **kwds):
+ if loop is None:
+ loop = events.get_event_loop()
+ protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
+ loop=loop)
+ transport, protocol = yield from loop.subprocess_shell(
+ protocol_factory,
+ cmd, stdin=stdin, stdout=stdout,
+ stderr=stderr, **kwds)
+ yield from protocol.waiter
+ return Process(transport, protocol, loop)
+
+@tasks.coroutine
+def create_subprocess_exec(program, *args, stdin=None, stdout=None,
+ stderr=None, loop=None,
+ limit=streams._DEFAULT_LIMIT, **kwds):
+ if loop is None:
+ loop = events.get_event_loop()
+ protocol_factory = lambda: SubprocessStreamProtocol(limit=limit,
+ loop=loop)
+ transport, protocol = yield from loop.subprocess_exec(
+ protocol_factory,
+ program, *args,
+ stdin=stdin, stdout=stdout,
+ stderr=stderr, **kwds)
+ yield from protocol.waiter
+ return Process(transport, protocol, loop)
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
new file mode 100644
index 0000000..0967e7e
--- /dev/null
+++ b/Lib/asyncio/tasks.py
@@ -0,0 +1,682 @@
+"""Support for tasks, coroutines and the scheduler."""
+
+__all__ = ['coroutine', 'Task',
+ 'iscoroutinefunction', 'iscoroutine',
+ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED',
+ 'wait', 'wait_for', 'as_completed', 'sleep', 'async',
+ 'gather', 'shield',
+ ]
+
+import concurrent.futures
+import functools
+import inspect
+import linecache
+import os
+import sys
+import traceback
+import weakref
+
+from . import events
+from . import futures
+from .log import logger
+
+# If you set _DEBUG to true, @coroutine will wrap the resulting
+# generator objects in a CoroWrapper instance (defined below). That
+# instance will log a message when the generator is never iterated
+# over, which may happen when you forget to use "yield from" with a
+# coroutine call. Note that the value of the _DEBUG flag is taken
+# when the decorator is used, so to be of any use it must be set
+# before you define your coroutines. A downside of using this feature
+# is that tracebacks show entries for the CoroWrapper.__next__ method
+# when _DEBUG is true.
+_DEBUG = (not sys.flags.ignore_environment
+ and bool(os.environ.get('PYTHONASYNCIODEBUG')))
+
+
+class CoroWrapper:
+ # Wrapper for coroutine in _DEBUG mode.
+
+ __slots__ = ['gen', 'func', '__name__', '__doc__']
+
+ def __init__(self, gen, func):
+ assert inspect.isgenerator(gen), gen
+ self.gen = gen
+ self.func = func
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.gen)
+
+ def send(self, value):
+ return self.gen.send(value)
+
+ def throw(self, exc):
+ return self.gen.throw(exc)
+
+ def close(self):
+ return self.gen.close()
+
+ def __del__(self):
+ frame = self.gen.gi_frame
+ if frame is not None and frame.f_lasti == -1:
+ func = self.func
+ code = func.__code__
+ filename = code.co_filename
+ lineno = code.co_firstlineno
+ logger.error(
+ 'Coroutine %r defined at %s:%s was never yielded from',
+ func.__name__, filename, lineno)
+
+
+def coroutine(func):
+ """Decorator to mark coroutines.
+
+ If the coroutine is not yielded from before it is destroyed,
+ an error message is logged.
+ """
+ if inspect.isgeneratorfunction(func):
+ coro = func
+ else:
+ @functools.wraps(func)
+ def coro(*args, **kw):
+ res = func(*args, **kw)
+ if isinstance(res, futures.Future) or inspect.isgenerator(res):
+ res = yield from res
+ return res
+
+ if not _DEBUG:
+ wrapper = coro
+ else:
+ @functools.wraps(func)
+ def wrapper(*args, **kwds):
+ w = CoroWrapper(coro(*args, **kwds), func)
+ w.__name__ = coro.__name__
+ w.__doc__ = coro.__doc__
+ return w
+
+ wrapper._is_coroutine = True # For iscoroutinefunction().
+ return wrapper
+
+
+def iscoroutinefunction(func):
+ """Return True if func is a decorated coroutine function."""
+ return getattr(func, '_is_coroutine', False)
+
+
+def iscoroutine(obj):
+ """Return True if obj is a coroutine object."""
+ return isinstance(obj, CoroWrapper) or inspect.isgenerator(obj)
+
+
+class Task(futures.Future):
+ """A coroutine wrapped in a Future."""
+
+ # An important invariant maintained while a Task not done:
+ #
+ # - Either _fut_waiter is None, and _step() is scheduled;
+ # - or _fut_waiter is some Future, and _step() is *not* scheduled.
+ #
+ # The only transition from the latter to the former is through
+ # _wakeup(). When _fut_waiter is not None, one of its callbacks
+ # must be _wakeup().
+
+ # Weak set containing all tasks alive.
+ _all_tasks = weakref.WeakSet()
+
+ # Dictionary containing tasks that are currently active in
+ # all running event loops. {EventLoop: Task}
+ _current_tasks = {}
+
+ @classmethod
+ def current_task(cls, loop=None):
+ """Return the currently running task in an event loop or None.
+
+ By default the current task for the current event loop is returned.
+
+ None is returned when called not in the context of a Task.
+ """
+ if loop is None:
+ loop = events.get_event_loop()
+ return cls._current_tasks.get(loop)
+
+ @classmethod
+ def all_tasks(cls, loop=None):
+ """Return a set of all tasks for an event loop.
+
+ By default all tasks for the current event loop are returned.
+ """
+ if loop is None:
+ loop = events.get_event_loop()
+ return {t for t in cls._all_tasks if t._loop is loop}
+
+ def __init__(self, coro, *, loop=None):
+ assert iscoroutine(coro), repr(coro) # Not a coroutine function!
+ super().__init__(loop=loop)
+ self._coro = iter(coro) # Use the iterator just in case.
+ self._fut_waiter = None
+ self._must_cancel = False
+ self._loop.call_soon(self._step)
+ self.__class__._all_tasks.add(self)
+
+ def __repr__(self):
+ res = super().__repr__()
+ if (self._must_cancel and
+ self._state == futures._PENDING and
+ '<PENDING' in res):
+ res = res.replace('<PENDING', '<CANCELLING', 1)
+ i = res.find('<')
+ if i < 0:
+ i = len(res)
+ res = res[:i] + '(<{}>)'.format(self._coro.__name__) + res[i:]
+ return res
+
+ def get_stack(self, *, limit=None):
+ """Return the list of stack frames for this task's coroutine.
+
+ If the coroutine is active, this returns the stack where it is
+ suspended. If the coroutine has completed successfully or was
+ cancelled, this returns an empty list. If the coroutine was
+ terminated by an exception, this returns the list of traceback
+ frames.
+
+ The frames are always ordered from oldest to newest.
+
+ The optional limit gives the maximum number of frames to
+ return; by default all available frames are returned. Its
+ meaning differs depending on whether a stack or a traceback is
+ returned: the newest frames of a stack are returned, but the
+ oldest frames of a traceback are returned. (This matches the
+ behavior of the traceback module.)
+
+ For reasons beyond our control, only one stack frame is
+ returned for a suspended coroutine.
+ """
+ frames = []
+ f = self._coro.gi_frame
+ if f is not None:
+ while f is not None:
+ if limit is not None:
+ if limit <= 0:
+ break
+ limit -= 1
+ frames.append(f)
+ f = f.f_back
+ frames.reverse()
+ elif self._exception is not None:
+ tb = self._exception.__traceback__
+ while tb is not None:
+ if limit is not None:
+ if limit <= 0:
+ break
+ limit -= 1
+ frames.append(tb.tb_frame)
+ tb = tb.tb_next
+ return frames
+
+ def print_stack(self, *, limit=None, file=None):
+ """Print the stack or traceback for this task's coroutine.
+
+ This produces output similar to that of the traceback module,
+ for the frames retrieved by get_stack(). The limit argument
+ is passed to get_stack(). The file argument is an I/O stream
+ to which the output goes; by default it goes to sys.stderr.
+ """
+ extracted_list = []
+ checked = set()
+ for f in self.get_stack(limit=limit):
+ lineno = f.f_lineno
+ co = f.f_code
+ filename = co.co_filename
+ name = co.co_name
+ if filename not in checked:
+ checked.add(filename)
+ linecache.checkcache(filename)
+ line = linecache.getline(filename, lineno, f.f_globals)
+ extracted_list.append((filename, lineno, name, line))
+ exc = self._exception
+ if not extracted_list:
+ print('No stack for %r' % self, file=file)
+ elif exc is not None:
+ print('Traceback for %r (most recent call last):' % self,
+ file=file)
+ else:
+ print('Stack for %r (most recent call last):' % self,
+ file=file)
+ traceback.print_list(extracted_list, file=file)
+ if exc is not None:
+ for line in traceback.format_exception_only(exc.__class__, exc):
+ print(line, file=file, end='')
+
+ def cancel(self):
+ if self.done():
+ return False
+ if self._fut_waiter is not None:
+ if self._fut_waiter.cancel():
+ # Leave self._fut_waiter; it may be a Task that
+ # catches and ignores the cancellation so we may have
+ # to cancel it again later.
+ return True
+ # It must be the case that self._step is already scheduled.
+ self._must_cancel = True
+ return True
+
+ def _step(self, value=None, exc=None):
+ assert not self.done(), \
+ '_step(): already done: {!r}, {!r}, {!r}'.format(self, value, exc)
+ if self._must_cancel:
+ if not isinstance(exc, futures.CancelledError):
+ exc = futures.CancelledError()
+ self._must_cancel = False
+ coro = self._coro
+ self._fut_waiter = None
+
+ self.__class__._current_tasks[self._loop] = self
+ # Call either coro.throw(exc) or coro.send(value).
+ try:
+ if exc is not None:
+ result = coro.throw(exc)
+ elif value is not None:
+ result = coro.send(value)
+ else:
+ result = next(coro)
+ except StopIteration as exc:
+ self.set_result(exc.value)
+ except futures.CancelledError as exc:
+ super().cancel() # I.e., Future.cancel(self).
+ except Exception as exc:
+ self.set_exception(exc)
+ except BaseException as exc:
+ self.set_exception(exc)
+ raise
+ else:
+ if isinstance(result, futures.Future):
+ # Yielded Future must come from Future.__iter__().
+ if result._blocking:
+ result._blocking = False
+ result.add_done_callback(self._wakeup)
+ self._fut_waiter = result
+ if self._must_cancel:
+ if self._fut_waiter.cancel():
+ self._must_cancel = False
+ else:
+ self._loop.call_soon(
+ self._step, None,
+ RuntimeError(
+ 'yield was used instead of yield from '
+ 'in task {!r} with {!r}'.format(self, result)))
+ elif result is None:
+ # Bare yield relinquishes control for one event loop iteration.
+ self._loop.call_soon(self._step)
+ elif inspect.isgenerator(result):
+ # Yielding a generator is just wrong.
+ self._loop.call_soon(
+ self._step, None,
+ RuntimeError(
+ 'yield was used instead of yield from for '
+ 'generator in task {!r} with {}'.format(
+ self, result)))
+ else:
+ # Yielding something else is an error.
+ self._loop.call_soon(
+ self._step, None,
+ RuntimeError(
+ 'Task got bad yield: {!r}'.format(result)))
+ finally:
+ self.__class__._current_tasks.pop(self._loop)
+ self = None # Needed to break cycles when an exception occurs.
+
+ def _wakeup(self, future):
+ try:
+ value = future.result()
+ except Exception as exc:
+ # This may also be a cancellation.
+ self._step(None, exc)
+ else:
+ self._step(value, None)
+ self = None # Needed to break cycles when an exception occurs.
+
+
+# wait() and as_completed() similar to those in PEP 3148.
+
+FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED
+FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION
+ALL_COMPLETED = concurrent.futures.ALL_COMPLETED
+
+
+@coroutine
+def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED):
+ """Wait for the Futures and coroutines given by fs to complete.
+
+ Coroutines will be wrapped in Tasks.
+
+ Returns two sets of Future: (done, pending).
+
+ Usage:
+
+ done, pending = yield from asyncio.wait(fs)
+
+ Note: This does not raise TimeoutError! Futures that aren't done
+ when the timeout occurs are returned in the second set.
+ """
+ if isinstance(fs, futures.Future) or iscoroutine(fs):
+ raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
+ if not fs:
+ raise ValueError('Set of coroutines/Futures is empty.')
+
+ if loop is None:
+ loop = events.get_event_loop()
+
+ fs = {async(f, loop=loop) for f in set(fs)}
+
+ if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED):
+ raise ValueError('Invalid return_when value: {}'.format(return_when))
+ return (yield from _wait(fs, timeout, return_when, loop))
+
+
+def _release_waiter(waiter, value=True, *args):
+ if not waiter.done():
+ waiter.set_result(value)
+
+
+@coroutine
+def wait_for(fut, timeout, *, loop=None):
+ """Wait for the single Future or coroutine to complete, with timeout.
+
+ Coroutine will be wrapped in Task.
+
+ Returns result of the Future or coroutine. When a timeout occurs,
+ it cancels the task and raises TimeoutError. To avoid the task
+ cancellation, wrap it in shield().
+
+ Usage:
+
+ result = yield from asyncio.wait_for(fut, 10.0)
+
+ """
+ if loop is None:
+ loop = events.get_event_loop()
+
+ if timeout is None:
+ return (yield from fut)
+
+ waiter = futures.Future(loop=loop)
+ timeout_handle = loop.call_later(timeout, _release_waiter, waiter, False)
+ cb = functools.partial(_release_waiter, waiter, True)
+
+ fut = async(fut, loop=loop)
+ fut.add_done_callback(cb)
+
+ try:
+ if (yield from waiter):
+ return fut.result()
+ else:
+ fut.remove_done_callback(cb)
+ fut.cancel()
+ raise futures.TimeoutError()
+ finally:
+ timeout_handle.cancel()
+
+
+@coroutine
+def _wait(fs, timeout, return_when, loop):
+ """Internal helper for wait() and _wait_for().
+
+ The fs argument must be a collection of Futures.
+ """
+ assert fs, 'Set of Futures is empty.'
+ waiter = futures.Future(loop=loop)
+ timeout_handle = None
+ if timeout is not None:
+ timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
+ counter = len(fs)
+
+ def _on_completion(f):
+ nonlocal counter
+ counter -= 1
+ if (counter <= 0 or
+ return_when == FIRST_COMPLETED or
+ return_when == FIRST_EXCEPTION and (not f.cancelled() and
+ f.exception() is not None)):
+ if timeout_handle is not None:
+ timeout_handle.cancel()
+ if not waiter.done():
+ waiter.set_result(False)
+
+ for f in fs:
+ f.add_done_callback(_on_completion)
+
+ try:
+ yield from waiter
+ finally:
+ if timeout_handle is not None:
+ timeout_handle.cancel()
+
+ done, pending = set(), set()
+ for f in fs:
+ f.remove_done_callback(_on_completion)
+ if f.done():
+ done.add(f)
+ else:
+ pending.add(f)
+ return done, pending
+
+
+# This is *not* a @coroutine! It is just an iterator (yielding Futures).
+def as_completed(fs, *, loop=None, timeout=None):
+ """Return an iterator whose values are coroutines.
+
+ When waiting for the yielded coroutines you'll get the results (or
+ exceptions!) of the original Futures (or coroutines), in the order
+ in which and as soon as they complete.
+
+ This differs from PEP 3148; the proper way to use this is:
+
+ for f in as_completed(fs):
+ result = yield from f # The 'yield from' may raise.
+ # Use result.
+
+ If a timeout is specified, the 'yield from' will raise
+ TimeoutError when the timeout occurs before all Futures are done.
+
+ Note: The futures 'f' are not necessarily members of fs.
+ """
+ if isinstance(fs, futures.Future) or iscoroutine(fs):
+ raise TypeError("expect a list of futures, not %s" % type(fs).__name__)
+ loop = loop if loop is not None else events.get_event_loop()
+ todo = {async(f, loop=loop) for f in set(fs)}
+ from .queues import Queue # Import here to avoid circular import problem.
+ done = Queue(loop=loop)
+ timeout_handle = None
+
+ def _on_timeout():
+ for f in todo:
+ f.remove_done_callback(_on_completion)
+ done.put_nowait(None) # Queue a dummy value for _wait_for_one().
+ todo.clear() # Can't do todo.remove(f) in the loop.
+
+ def _on_completion(f):
+ if not todo:
+ return # _on_timeout() was here first.
+ todo.remove(f)
+ done.put_nowait(f)
+ if not todo and timeout_handle is not None:
+ timeout_handle.cancel()
+
+ @coroutine
+ def _wait_for_one():
+ f = yield from done.get()
+ if f is None:
+ # Dummy value from _on_timeout().
+ raise futures.TimeoutError
+ return f.result() # May raise f.exception().
+
+ for f in todo:
+ f.add_done_callback(_on_completion)
+ if todo and timeout is not None:
+ timeout_handle = loop.call_later(timeout, _on_timeout)
+ for _ in range(len(todo)):
+ yield _wait_for_one()
+
+
+@coroutine
+def sleep(delay, result=None, *, loop=None):
+ """Coroutine that completes after a given time (in seconds)."""
+ future = futures.Future(loop=loop)
+ h = future._loop.call_later(delay, future.set_result, result)
+ try:
+ return (yield from future)
+ finally:
+ h.cancel()
+
+
+def async(coro_or_future, *, loop=None):
+ """Wrap a coroutine in a future.
+
+ If the argument is a Future, it is returned directly.
+ """
+ if isinstance(coro_or_future, futures.Future):
+ if loop is not None and loop is not coro_or_future._loop:
+ raise ValueError('loop argument must agree with Future')
+ return coro_or_future
+ elif iscoroutine(coro_or_future):
+ return Task(coro_or_future, loop=loop)
+ else:
+ raise TypeError('A Future or coroutine is required')
+
+
+class _GatheringFuture(futures.Future):
+ """Helper for gather().
+
+ This overrides cancel() to cancel all the children and act more
+ like Task.cancel(), which doesn't immediately mark itself as
+ cancelled.
+ """
+
+ def __init__(self, children, *, loop=None):
+ super().__init__(loop=loop)
+ self._children = children
+
+ def cancel(self):
+ if self.done():
+ return False
+ for child in self._children:
+ child.cancel()
+ return True
+
+
+def gather(*coros_or_futures, loop=None, return_exceptions=False):
+ """Return a future aggregating results from the given coroutines
+ or futures.
+
+ All futures must share the same event loop. If all the tasks are
+ done successfully, the returned future's result is the list of
+ results (in the order of the original sequence, not necessarily
+ the order of results arrival). If *return_exceptions* is True,
+ exceptions in the tasks are treated the same as successful
+ results, and gathered in the result list; otherwise, the first
+ raised exception will be immediately propagated to the returned
+ future.
+
+ Cancellation: if the outer Future is cancelled, all children (that
+ have not completed yet) are also cancelled. If any child is
+ cancelled, this is treated as if it raised CancelledError --
+ the outer Future is *not* cancelled in this case. (This is to
+ prevent the cancellation of one child to cause other children to
+ be cancelled.)
+ """
+ arg_to_fut = {arg: async(arg, loop=loop) for arg in set(coros_or_futures)}
+ children = [arg_to_fut[arg] for arg in coros_or_futures]
+ n = len(children)
+ if n == 0:
+ outer = futures.Future(loop=loop)
+ outer.set_result([])
+ return outer
+ if loop is None:
+ loop = children[0]._loop
+ for fut in children:
+ if fut._loop is not loop:
+ raise ValueError("futures are tied to different event loops")
+ outer = _GatheringFuture(children, loop=loop)
+ nfinished = 0
+ results = [None] * n
+
+ def _done_callback(i, fut):
+ nonlocal nfinished
+ if outer._state != futures._PENDING:
+ if fut._exception is not None:
+ # Mark exception retrieved.
+ fut.exception()
+ return
+ if fut._state == futures._CANCELLED:
+ res = futures.CancelledError()
+ if not return_exceptions:
+ outer.set_exception(res)
+ return
+ elif fut._exception is not None:
+ res = fut.exception() # Mark exception retrieved.
+ if not return_exceptions:
+ outer.set_exception(res)
+ return
+ else:
+ res = fut._result
+ results[i] = res
+ nfinished += 1
+ if nfinished == n:
+ outer.set_result(results)
+
+ for i, fut in enumerate(children):
+ fut.add_done_callback(functools.partial(_done_callback, i))
+ return outer
+
+
+def shield(arg, *, loop=None):
+ """Wait for a future, shielding it from cancellation.
+
+ The statement
+
+ res = yield from shield(something())
+
+ is exactly equivalent to the statement
+
+ res = yield from something()
+
+ *except* that if the coroutine containing it is cancelled, the
+ task running in something() is not cancelled. From the POV of
+ something(), the cancellation did not happen. But its caller is
+ still cancelled, so the yield-from expression still raises
+ CancelledError. Note: If something() is cancelled by other means
+ this will still cancel shield().
+
+ If you want to completely ignore cancellation (not recommended)
+ you can combine shield() with a try/except clause, as follows:
+
+ try:
+ res = yield from shield(something())
+ except CancelledError:
+ res = None
+ """
+ inner = async(arg, loop=loop)
+ if inner.done():
+ # Shortcut.
+ return inner
+ loop = inner._loop
+ outer = futures.Future(loop=loop)
+
+ def _done_callback(inner):
+ if outer.cancelled():
+ # Mark inner's result as retrieved.
+ inner.cancelled() or inner.exception()
+ return
+ if inner.cancelled():
+ outer.cancel()
+ else:
+ exc = inner.exception()
+ if exc is not None:
+ outer.set_exception(exc)
+ else:
+ outer.set_result(inner.result())
+
+ inner.add_done_callback(_done_callback)
+ return outer
diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py
new file mode 100644
index 0000000..9c3656a
--- /dev/null
+++ b/Lib/asyncio/test_utils.py
@@ -0,0 +1,374 @@
+"""Utilities shared by tests."""
+
+import collections
+import contextlib
+import io
+import os
+import re
+import socket
+import socketserver
+import sys
+import tempfile
+import threading
+import time
+from unittest import mock
+
+from http.server import HTTPServer
+from wsgiref.simple_server import WSGIRequestHandler, WSGIServer
+
+try:
+ import ssl
+except ImportError: # pragma: no cover
+ ssl = None
+
+from . import base_events
+from . import events
+from . import futures
+from . import selectors
+from . import tasks
+
+
+if sys.platform == 'win32': # pragma: no cover
+ from .windows_utils import socketpair
+else:
+ from socket import socketpair # pragma: no cover
+
+
+def dummy_ssl_context():
+ if ssl is None:
+ return None
+ else:
+ return ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+
+
+def run_briefly(loop):
+ @tasks.coroutine
+ def once():
+ pass
+ gen = once()
+ t = tasks.Task(gen, loop=loop)
+ try:
+ loop.run_until_complete(t)
+ finally:
+ gen.close()
+
+
+def run_until(loop, pred, timeout=30):
+ deadline = time.time() + timeout
+ while not pred():
+ if timeout is not None:
+ timeout = deadline - time.time()
+ if timeout <= 0:
+ raise futures.TimeoutError()
+ loop.run_until_complete(tasks.sleep(0.001, loop=loop))
+
+
+def run_once(loop):
+ """loop.stop() schedules _raise_stop_error()
+ and run_forever() runs until _raise_stop_error() callback.
+ this wont work if test waits for some IO events, because
+ _raise_stop_error() runs before any of io events callbacks.
+ """
+ loop.stop()
+ loop.run_forever()
+
+
+class SilentWSGIRequestHandler(WSGIRequestHandler):
+
+ def get_stderr(self):
+ return io.StringIO()
+
+ def log_message(self, format, *args):
+ pass
+
+
+class SilentWSGIServer(WSGIServer):
+
+ def handle_error(self, request, client_address):
+ pass
+
+
+class SSLWSGIServerMixin:
+
+ def finish_request(self, request, client_address):
+ # The relative location of our test directory (which
+ # contains the ssl key and certificate files) differs
+ # between the stdlib and stand-alone asyncio.
+ # Prefer our own if we can find it.
+ here = os.path.join(os.path.dirname(__file__), '..', 'tests')
+ if not os.path.isdir(here):
+ here = os.path.join(os.path.dirname(os.__file__),
+ 'test', 'test_asyncio')
+ keyfile = os.path.join(here, 'ssl_key.pem')
+ certfile = os.path.join(here, 'ssl_cert.pem')
+ ssock = ssl.wrap_socket(request,
+ keyfile=keyfile,
+ certfile=certfile,
+ server_side=True)
+ try:
+ self.RequestHandlerClass(ssock, client_address, self)
+ ssock.close()
+ except OSError:
+ # maybe socket has been closed by peer
+ pass
+
+
+class SSLWSGIServer(SSLWSGIServerMixin, SilentWSGIServer):
+ pass
+
+
+def _run_test_server(*, address, use_ssl=False, server_cls, server_ssl_cls):
+
+ def app(environ, start_response):
+ status = '200 OK'
+ headers = [('Content-type', 'text/plain')]
+ start_response(status, headers)
+ return [b'Test message']
+
+ # Run the test WSGI server in a separate thread in order not to
+ # interfere with event handling in the main thread
+ server_class = server_ssl_cls if use_ssl else server_cls
+ httpd = server_class(address, SilentWSGIRequestHandler)
+ httpd.set_app(app)
+ httpd.address = httpd.server_address
+ server_thread = threading.Thread(target=httpd.serve_forever)
+ server_thread.start()
+ try:
+ yield httpd
+ finally:
+ httpd.shutdown()
+ httpd.server_close()
+ server_thread.join()
+
+
+if hasattr(socket, 'AF_UNIX'):
+
+ class UnixHTTPServer(socketserver.UnixStreamServer, HTTPServer):
+
+ def server_bind(self):
+ socketserver.UnixStreamServer.server_bind(self)
+ self.server_name = '127.0.0.1'
+ self.server_port = 80
+
+
+ class UnixWSGIServer(UnixHTTPServer, WSGIServer):
+
+ def server_bind(self):
+ UnixHTTPServer.server_bind(self)
+ self.setup_environ()
+
+ def get_request(self):
+ request, client_addr = super().get_request()
+ # Code in the stdlib expects that get_request
+ # will return a socket and a tuple (host, port).
+ # However, this isn't true for UNIX sockets,
+ # as the second return value will be a path;
+ # hence we return some fake data sufficient
+ # to get the tests going
+ return request, ('127.0.0.1', '')
+
+
+ class SilentUnixWSGIServer(UnixWSGIServer):
+
+ def handle_error(self, request, client_address):
+ pass
+
+
+ class UnixSSLWSGIServer(SSLWSGIServerMixin, SilentUnixWSGIServer):
+ pass
+
+
+ def gen_unix_socket_path():
+ with tempfile.NamedTemporaryFile() as file:
+ return file.name
+
+
+ @contextlib.contextmanager
+ def unix_socket_path():
+ path = gen_unix_socket_path()
+ try:
+ yield path
+ finally:
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+ @contextlib.contextmanager
+ def run_test_unix_server(*, use_ssl=False):
+ with unix_socket_path() as path:
+ yield from _run_test_server(address=path, use_ssl=use_ssl,
+ server_cls=SilentUnixWSGIServer,
+ server_ssl_cls=UnixSSLWSGIServer)
+
+
+@contextlib.contextmanager
+def run_test_server(*, host='127.0.0.1', port=0, use_ssl=False):
+ yield from _run_test_server(address=(host, port), use_ssl=use_ssl,
+ server_cls=SilentWSGIServer,
+ server_ssl_cls=SSLWSGIServer)
+
+
+def make_test_protocol(base):
+ dct = {}
+ for name in dir(base):
+ if name.startswith('__') and name.endswith('__'):
+ # skip magic names
+ continue
+ dct[name] = MockCallback(return_value=None)
+ return type('TestProtocol', (base,) + base.__bases__, dct)()
+
+
+class TestSelector(selectors.BaseSelector):
+
+ def __init__(self):
+ self.keys = {}
+
+ def register(self, fileobj, events, data=None):
+ key = selectors.SelectorKey(fileobj, 0, events, data)
+ self.keys[fileobj] = key
+ return key
+
+ def unregister(self, fileobj):
+ return self.keys.pop(fileobj)
+
+ def select(self, timeout):
+ return []
+
+ def get_map(self):
+ return self.keys
+
+
+class TestLoop(base_events.BaseEventLoop):
+ """Loop for unittests.
+
+ It manages self time directly.
+ If something scheduled to be executed later then
+ on next loop iteration after all ready handlers done
+ generator passed to __init__ is calling.
+
+ Generator should be like this:
+
+ def gen():
+ ...
+ when = yield ...
+ ... = yield time_advance
+
+ Value returned by yield is absolute time of next scheduled handler.
+ Value passed to yield is time advance to move loop's time forward.
+ """
+
+ def __init__(self, gen=None):
+ super().__init__()
+
+ if gen is None:
+ def gen():
+ yield
+ self._check_on_close = False
+ else:
+ self._check_on_close = True
+
+ self._gen = gen()
+ next(self._gen)
+ self._time = 0
+ self._clock_resolution = 1e-9
+ self._timers = []
+ self._selector = TestSelector()
+
+ self.readers = {}
+ self.writers = {}
+ self.reset_counters()
+
+ def time(self):
+ return self._time
+
+ def advance_time(self, advance):
+ """Move test time forward."""
+ if advance:
+ self._time += advance
+
+ def close(self):
+ if self._check_on_close:
+ try:
+ self._gen.send(0)
+ except StopIteration:
+ pass
+ else: # pragma: no cover
+ raise AssertionError("Time generator is not finished")
+
+ def add_reader(self, fd, callback, *args):
+ self.readers[fd] = events.Handle(callback, args, self)
+
+ def remove_reader(self, fd):
+ self.remove_reader_count[fd] += 1
+ if fd in self.readers:
+ del self.readers[fd]
+ return True
+ else:
+ return False
+
+ def assert_reader(self, fd, callback, *args):
+ assert fd in self.readers, 'fd {} is not registered'.format(fd)
+ handle = self.readers[fd]
+ assert handle._callback == callback, '{!r} != {!r}'.format(
+ handle._callback, callback)
+ assert handle._args == args, '{!r} != {!r}'.format(
+ handle._args, args)
+
+ def add_writer(self, fd, callback, *args):
+ self.writers[fd] = events.Handle(callback, args, self)
+
+ def remove_writer(self, fd):
+ self.remove_writer_count[fd] += 1
+ if fd in self.writers:
+ del self.writers[fd]
+ return True
+ else:
+ return False
+
+ def assert_writer(self, fd, callback, *args):
+ assert fd in self.writers, 'fd {} is not registered'.format(fd)
+ handle = self.writers[fd]
+ assert handle._callback == callback, '{!r} != {!r}'.format(
+ handle._callback, callback)
+ assert handle._args == args, '{!r} != {!r}'.format(
+ handle._args, args)
+
+ def reset_counters(self):
+ self.remove_reader_count = collections.defaultdict(int)
+ self.remove_writer_count = collections.defaultdict(int)
+
+ def _run_once(self):
+ super()._run_once()
+ for when in self._timers:
+ advance = self._gen.send(when)
+ self.advance_time(advance)
+ self._timers = []
+
+ def call_at(self, when, callback, *args):
+ self._timers.append(when)
+ return super().call_at(when, callback, *args)
+
+ def _process_events(self, event_list):
+ return
+
+ def _write_to_self(self):
+ pass
+
+
+def MockCallback(**kwargs):
+ return mock.Mock(spec=['__call__'], **kwargs)
+
+
+class MockPattern(str):
+ """A regex based str with a fuzzy __eq__.
+
+ Use this helper with 'mock.assert_called_with', or anywhere
+ where a regex comparison between strings is needed.
+
+ For instance:
+ mock_call.assert_called_with(MockPattern('spam.*ham'))
+ """
+ def __eq__(self, other):
+ return bool(re.search(str(self), other, re.S))
diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py
new file mode 100644
index 0000000..5f674f9
--- /dev/null
+++ b/Lib/asyncio/transports.py
@@ -0,0 +1,295 @@
+"""Abstract Transport class."""
+
+import sys
+
+_PY34 = sys.version_info >= (3, 4)
+
+__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport',
+ 'Transport', 'DatagramTransport', 'SubprocessTransport',
+ ]
+
+
+class BaseTransport:
+ """Base class for transports."""
+
+ def __init__(self, extra=None):
+ if extra is None:
+ extra = {}
+ self._extra = extra
+
+ def get_extra_info(self, name, default=None):
+ """Get optional transport information."""
+ return self._extra.get(name, default)
+
+ def close(self):
+ """Close the transport.
+
+ Buffered data will be flushed asynchronously. No more data
+ will be received. After all buffered data is flushed, the
+ protocol's connection_lost() method will (eventually) called
+ with None as its argument.
+ """
+ raise NotImplementedError
+
+
+class ReadTransport(BaseTransport):
+ """Interface for read-only transports."""
+
+ def pause_reading(self):
+ """Pause the receiving end.
+
+ No data will be passed to the protocol's data_received()
+ method until resume_reading() is called.
+ """
+ raise NotImplementedError
+
+ def resume_reading(self):
+ """Resume the receiving end.
+
+ Data received will once again be passed to the protocol's
+ data_received() method.
+ """
+ raise NotImplementedError
+
+
+class WriteTransport(BaseTransport):
+ """Interface for write-only transports."""
+
+ def set_write_buffer_limits(self, high=None, low=None):
+ """Set the high- and low-water limits for write flow control.
+
+ These two values control when to call the protocol's
+ pause_writing() and resume_writing() methods. If specified,
+ the low-water limit must be less than or equal to the
+ high-water limit. Neither value can be negative.
+
+ The defaults are implementation-specific. If only the
+ high-water limit is given, the low-water limit defaults to a
+ implementation-specific value less than or equal to the
+ high-water limit. Setting high to zero forces low to zero as
+ well, and causes pause_writing() to be called whenever the
+ buffer becomes non-empty. Setting low to zero causes
+ resume_writing() to be called only once the buffer is empty.
+ Use of zero for either limit is generally sub-optimal as it
+ reduces opportunities for doing I/O and computation
+ concurrently.
+ """
+ raise NotImplementedError
+
+ def get_write_buffer_size(self):
+ """Return the current size of the write buffer."""
+ raise NotImplementedError
+
+ def write(self, data):
+ """Write some data bytes to the transport.
+
+ This does not block; it buffers the data and arranges for it
+ to be sent out asynchronously.
+ """
+ raise NotImplementedError
+
+ def writelines(self, list_of_data):
+ """Write a list (or any iterable) of data bytes to the transport.
+
+ The default implementation concatenates the arguments and
+ calls write() on the result.
+ """
+ if not _PY34:
+ # In Python 3.3, bytes.join() doesn't handle memoryview.
+ list_of_data = (
+ bytes(data) if isinstance(data, memoryview) else data
+ for data in list_of_data)
+ self.write(b''.join(list_of_data))
+
+ def write_eof(self):
+ """Close the write end after flushing buffered data.
+
+ (This is like typing ^D into a UNIX program reading from stdin.)
+
+ Data may still be received.
+ """
+ raise NotImplementedError
+
+ def can_write_eof(self):
+ """Return True if this transport supports write_eof(), False if not."""
+ raise NotImplementedError
+
+ def abort(self):
+ """Close the transport immediately.
+
+ Buffered data will be lost. No more data will be received.
+ The protocol's connection_lost() method will (eventually) be
+ called with None as its argument.
+ """
+ raise NotImplementedError
+
+
+class Transport(ReadTransport, WriteTransport):
+ """Interface representing a bidirectional transport.
+
+ There may be several implementations, but typically, the user does
+ not implement new transports; rather, the platform provides some
+ useful transports that are implemented using the platform's best
+ practices.
+
+ The user never instantiates a transport directly; they call a
+ utility function, passing it a protocol factory and other
+ information necessary to create the transport and protocol. (E.g.
+ EventLoop.create_connection() or EventLoop.create_server().)
+
+ The utility function will asynchronously create a transport and a
+ protocol and hook them up by calling the protocol's
+ connection_made() method, passing it the transport.
+
+ The implementation here raises NotImplemented for every method
+ except writelines(), which calls write() in a loop.
+ """
+
+
+class DatagramTransport(BaseTransport):
+ """Interface for datagram (UDP) transports."""
+
+ def sendto(self, data, addr=None):
+ """Send data to the transport.
+
+ This does not block; it buffers the data and arranges for it
+ to be sent out asynchronously.
+ addr is target socket address.
+ If addr is None use target address pointed on transport creation.
+ """
+ raise NotImplementedError
+
+ def abort(self):
+ """Close the transport immediately.
+
+ Buffered data will be lost. No more data will be received.
+ The protocol's connection_lost() method will (eventually) be
+ called with None as its argument.
+ """
+ raise NotImplementedError
+
+
+class SubprocessTransport(BaseTransport):
+
+ def get_pid(self):
+ """Get subprocess id."""
+ raise NotImplementedError
+
+ def get_returncode(self):
+ """Get subprocess returncode.
+
+ See also
+ http://docs.python.org/3/library/subprocess#subprocess.Popen.returncode
+ """
+ raise NotImplementedError
+
+ def get_pipe_transport(self, fd):
+ """Get transport for pipe with number fd."""
+ raise NotImplementedError
+
+ def send_signal(self, signal):
+ """Send signal to subprocess.
+
+ See also:
+ docs.python.org/3/library/subprocess#subprocess.Popen.send_signal
+ """
+ raise NotImplementedError
+
+ def terminate(self):
+ """Stop the subprocess.
+
+ Alias for close() method.
+
+ On Posix OSs the method sends SIGTERM to the subprocess.
+ On Windows the Win32 API function TerminateProcess()
+ is called to stop the subprocess.
+
+ See also:
+ http://docs.python.org/3/library/subprocess#subprocess.Popen.terminate
+ """
+ raise NotImplementedError
+
+ def kill(self):
+ """Kill the subprocess.
+
+ On Posix OSs the function sends SIGKILL to the subprocess.
+ On Windows kill() is an alias for terminate().
+
+ See also:
+ http://docs.python.org/3/library/subprocess#subprocess.Popen.kill
+ """
+ raise NotImplementedError
+
+
+class _FlowControlMixin(Transport):
+ """All the logic for (write) flow control in a mix-in base class.
+
+ The subclass must implement get_write_buffer_size(). It must call
+ _maybe_pause_protocol() whenever the write buffer size increases,
+ and _maybe_resume_protocol() whenever it decreases. It may also
+ override set_write_buffer_limits() (e.g. to specify different
+ defaults).
+
+ The subclass constructor must call super().__init__(extra). This
+ will call set_write_buffer_limits().
+
+ The user may call set_write_buffer_limits() and
+ get_write_buffer_size(), and their protocol's pause_writing() and
+ resume_writing() may be called.
+ """
+
+ def __init__(self, extra=None):
+ super().__init__(extra)
+ self._protocol_paused = False
+ self._set_write_buffer_limits()
+
+ def _maybe_pause_protocol(self):
+ size = self.get_write_buffer_size()
+ if size <= self._high_water:
+ return
+ if not self._protocol_paused:
+ self._protocol_paused = True
+ try:
+ self._protocol.pause_writing()
+ except Exception as exc:
+ self._loop.call_exception_handler({
+ 'message': 'protocol.pause_writing() failed',
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+
+ def _maybe_resume_protocol(self):
+ if (self._protocol_paused and
+ self.get_write_buffer_size() <= self._low_water):
+ self._protocol_paused = False
+ try:
+ self._protocol.resume_writing()
+ except Exception as exc:
+ self._loop.call_exception_handler({
+ 'message': 'protocol.resume_writing() failed',
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+
+ def _set_write_buffer_limits(self, high=None, low=None):
+ if high is None:
+ if low is None:
+ high = 64*1024
+ else:
+ high = 4*low
+ if low is None:
+ low = high // 4
+ if not high >= low >= 0:
+ raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
+ (high, low))
+ self._high_water = high
+ self._low_water = low
+
+ def set_write_buffer_limits(self, high=None, low=None):
+ self._set_write_buffer_limits(high=high, low=low)
+ self._maybe_pause_protocol()
+
+ def get_write_buffer_size(self):
+ raise NotImplementedError
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
new file mode 100644
index 0000000..2125548
--- /dev/null
+++ b/Lib/asyncio/unix_events.py
@@ -0,0 +1,838 @@
+"""Selector event loop for Unix with signal handling."""
+
+import errno
+import fcntl
+import os
+import signal
+import socket
+import stat
+import subprocess
+import sys
+import threading
+
+
+from . import base_events
+from . import base_subprocess
+from . import constants
+from . import events
+from . import selector_events
+from . import tasks
+from . import transports
+from .log import logger
+
+
+__all__ = ['SelectorEventLoop',
+ 'AbstractChildWatcher', 'SafeChildWatcher',
+ 'FastChildWatcher', 'DefaultEventLoopPolicy',
+ ]
+
+if sys.platform == 'win32': # pragma: no cover
+ raise ImportError('Signals are not really supported on Windows')
+
+
+class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
+ """Unix event loop.
+
+ Adds signal handling and UNIX Domain Socket support to SelectorEventLoop.
+ """
+
+ def __init__(self, selector=None):
+ super().__init__(selector)
+ self._signal_handlers = {}
+
+ def _socketpair(self):
+ return socket.socketpair()
+
+ def close(self):
+ for sig in list(self._signal_handlers):
+ self.remove_signal_handler(sig)
+ super().close()
+
+ def add_signal_handler(self, sig, callback, *args):
+ """Add a handler for a signal. UNIX only.
+
+ Raise ValueError if the signal number is invalid or uncatchable.
+ Raise RuntimeError if there is a problem setting up the handler.
+ """
+ self._check_signal(sig)
+ try:
+ # set_wakeup_fd() raises ValueError if this is not the
+ # main thread. By calling it early we ensure that an
+ # event loop running in another thread cannot add a signal
+ # handler.
+ signal.set_wakeup_fd(self._csock.fileno())
+ except ValueError as exc:
+ raise RuntimeError(str(exc))
+
+ handle = events.Handle(callback, args, self)
+ self._signal_handlers[sig] = handle
+
+ try:
+ signal.signal(sig, self._handle_signal)
+ # Set SA_RESTART to limit EINTR occurrences.
+ signal.siginterrupt(sig, False)
+ except OSError as exc:
+ del self._signal_handlers[sig]
+ if not self._signal_handlers:
+ try:
+ signal.set_wakeup_fd(-1)
+ except ValueError as nexc:
+ logger.info('set_wakeup_fd(-1) failed: %s', nexc)
+
+ if exc.errno == errno.EINVAL:
+ raise RuntimeError('sig {} cannot be caught'.format(sig))
+ else:
+ raise
+
+ def _handle_signal(self, sig, arg):
+ """Internal helper that is the actual signal handler."""
+ handle = self._signal_handlers.get(sig)
+ if handle is None:
+ return # Assume it's some race condition.
+ if handle._cancelled:
+ self.remove_signal_handler(sig) # Remove it properly.
+ else:
+ self._add_callback_signalsafe(handle)
+
+ def remove_signal_handler(self, sig):
+ """Remove a handler for a signal. UNIX only.
+
+ Return True if a signal handler was removed, False if not.
+ """
+ self._check_signal(sig)
+ try:
+ del self._signal_handlers[sig]
+ except KeyError:
+ return False
+
+ if sig == signal.SIGINT:
+ handler = signal.default_int_handler
+ else:
+ handler = signal.SIG_DFL
+
+ try:
+ signal.signal(sig, handler)
+ except OSError as exc:
+ if exc.errno == errno.EINVAL:
+ raise RuntimeError('sig {} cannot be caught'.format(sig))
+ else:
+ raise
+
+ if not self._signal_handlers:
+ try:
+ signal.set_wakeup_fd(-1)
+ except ValueError as exc:
+ logger.info('set_wakeup_fd(-1) failed: %s', exc)
+
+ return True
+
+ def _check_signal(self, sig):
+ """Internal helper to validate a signal.
+
+ Raise ValueError if the signal number is invalid or uncatchable.
+ Raise RuntimeError if there is a problem setting up the handler.
+ """
+ if not isinstance(sig, int):
+ raise TypeError('sig must be an int, not {!r}'.format(sig))
+
+ if not (1 <= sig < signal.NSIG):
+ raise ValueError(
+ 'sig {} out of range(1, {})'.format(sig, signal.NSIG))
+
+ def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
+ extra=None):
+ return _UnixReadPipeTransport(self, pipe, protocol, waiter, extra)
+
+ def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
+ extra=None):
+ return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra)
+
+ @tasks.coroutine
+ def _make_subprocess_transport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=None, **kwargs):
+ with events.get_child_watcher() as watcher:
+ transp = _UnixSubprocessTransport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=extra, **kwargs)
+ yield from transp._post_init()
+ watcher.add_child_handler(transp.get_pid(),
+ self._child_watcher_callback, transp)
+
+ return transp
+
+ def _child_watcher_callback(self, pid, returncode, transp):
+ self.call_soon_threadsafe(transp._process_exited, returncode)
+
+ @tasks.coroutine
+ def create_unix_connection(self, protocol_factory, path, *,
+ ssl=None, sock=None,
+ server_hostname=None):
+ assert server_hostname is None or isinstance(server_hostname, str)
+ if ssl:
+ if server_hostname is None:
+ raise ValueError(
+ 'you have to pass server_hostname when using ssl')
+ else:
+ if server_hostname is not None:
+ raise ValueError('server_hostname is only meaningful with ssl')
+
+ if path is not None:
+ if sock is not None:
+ raise ValueError(
+ 'path and sock can not be specified at the same time')
+
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+ try:
+ sock.setblocking(False)
+ yield from self.sock_connect(sock, path)
+ except:
+ sock.close()
+ raise
+
+ else:
+ if sock is None:
+ raise ValueError('no path and sock were specified')
+ sock.setblocking(False)
+
+ transport, protocol = yield from self._create_connection_transport(
+ sock, protocol_factory, ssl, server_hostname)
+ return transport, protocol
+
+ @tasks.coroutine
+ def create_unix_server(self, protocol_factory, path=None, *,
+ sock=None, backlog=100, ssl=None):
+ if isinstance(ssl, bool):
+ raise TypeError('ssl argument must be an SSLContext or None')
+
+ if path is not None:
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+ try:
+ sock.bind(path)
+ except OSError as exc:
+ sock.close()
+ if exc.errno == errno.EADDRINUSE:
+ # Let's improve the error message by adding
+ # with what exact address it occurs.
+ msg = 'Address {!r} is already in use'.format(path)
+ raise OSError(errno.EADDRINUSE, msg) from None
+ else:
+ raise
+ else:
+ if sock is None:
+ raise ValueError(
+ 'path was not specified, and no sock specified')
+
+ if sock.family != socket.AF_UNIX:
+ raise ValueError(
+ 'A UNIX Domain Socket was expected, got {!r}'.format(sock))
+
+ server = base_events.Server(self, [sock])
+ sock.listen(backlog)
+ sock.setblocking(False)
+ self._start_serving(protocol_factory, sock, ssl, server)
+ return server
+
+
+def _set_nonblocking(fd):
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+ flags = flags | os.O_NONBLOCK
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+
+class _UnixReadPipeTransport(transports.ReadTransport):
+
+ max_size = 256 * 1024 # max bytes we read in one event loop iteration
+
+ def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
+ super().__init__(extra)
+ self._extra['pipe'] = pipe
+ self._loop = loop
+ self._pipe = pipe
+ self._fileno = pipe.fileno()
+ mode = os.fstat(self._fileno).st_mode
+ if not (stat.S_ISFIFO(mode) or
+ stat.S_ISSOCK(mode) or
+ stat.S_ISCHR(mode)):
+ raise ValueError("Pipe transport is for pipes/sockets only.")
+ _set_nonblocking(self._fileno)
+ self._protocol = protocol
+ self._closing = False
+ self._loop.add_reader(self._fileno, self._read_ready)
+ self._loop.call_soon(self._protocol.connection_made, self)
+ if waiter is not None:
+ self._loop.call_soon(waiter.set_result, None)
+
+ def _read_ready(self):
+ try:
+ data = os.read(self._fileno, self.max_size)
+ except (BlockingIOError, InterruptedError):
+ pass
+ except OSError as exc:
+ self._fatal_error(exc, 'Fatal read error on pipe transport')
+ else:
+ if data:
+ self._protocol.data_received(data)
+ else:
+ self._closing = True
+ self._loop.remove_reader(self._fileno)
+ self._loop.call_soon(self._protocol.eof_received)
+ self._loop.call_soon(self._call_connection_lost, None)
+
+ def pause_reading(self):
+ self._loop.remove_reader(self._fileno)
+
+ def resume_reading(self):
+ self._loop.add_reader(self._fileno, self._read_ready)
+
+ def close(self):
+ if not self._closing:
+ self._close(None)
+
+ def _fatal_error(self, exc, message='Fatal error on pipe transport'):
+ # should be called by exception handler only
+ if not (isinstance(exc, OSError) and exc.errno == errno.EIO):
+ self._loop.call_exception_handler({
+ 'message': message,
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+ self._close(exc)
+
+ def _close(self, exc):
+ self._closing = True
+ self._loop.remove_reader(self._fileno)
+ self._loop.call_soon(self._call_connection_lost, exc)
+
+ def _call_connection_lost(self, exc):
+ try:
+ self._protocol.connection_lost(exc)
+ finally:
+ self._pipe.close()
+ self._pipe = None
+ self._protocol = None
+ self._loop = None
+
+
+class _UnixWritePipeTransport(transports._FlowControlMixin,
+ transports.WriteTransport):
+
+ def __init__(self, loop, pipe, protocol, waiter=None, extra=None):
+ super().__init__(extra)
+ self._extra['pipe'] = pipe
+ self._loop = loop
+ self._pipe = pipe
+ self._fileno = pipe.fileno()
+ mode = os.fstat(self._fileno).st_mode
+ is_socket = stat.S_ISSOCK(mode)
+ if not (is_socket or
+ stat.S_ISFIFO(mode) or
+ stat.S_ISCHR(mode)):
+ raise ValueError("Pipe transport is only for "
+ "pipes, sockets and character devices")
+ _set_nonblocking(self._fileno)
+ self._protocol = protocol
+ self._buffer = []
+ self._conn_lost = 0
+ self._closing = False # Set when close() or write_eof() called.
+
+ # On AIX, the reader trick only works for sockets.
+ # On other platforms it works for pipes and sockets.
+ # (Exception: OS X 10.4? Issue #19294.)
+ if is_socket or not sys.platform.startswith("aix"):
+ self._loop.add_reader(self._fileno, self._read_ready)
+
+ self._loop.call_soon(self._protocol.connection_made, self)
+ if waiter is not None:
+ self._loop.call_soon(waiter.set_result, None)
+
+ def get_write_buffer_size(self):
+ return sum(len(data) for data in self._buffer)
+
+ def _read_ready(self):
+ # Pipe was closed by peer.
+ if self._buffer:
+ self._close(BrokenPipeError())
+ else:
+ self._close()
+
+ def write(self, data):
+ assert isinstance(data, (bytes, bytearray, memoryview)), repr(data)
+ if isinstance(data, bytearray):
+ data = memoryview(data)
+ if not data:
+ return
+
+ if self._conn_lost or self._closing:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('pipe closed by peer or '
+ 'os.write(pipe, data) raised exception.')
+ self._conn_lost += 1
+ return
+
+ if not self._buffer:
+ # Attempt to send it right away first.
+ try:
+ n = os.write(self._fileno, data)
+ except (BlockingIOError, InterruptedError):
+ n = 0
+ except Exception as exc:
+ self._conn_lost += 1
+ self._fatal_error(exc, 'Fatal write error on pipe transport')
+ return
+ if n == len(data):
+ return
+ elif n > 0:
+ data = data[n:]
+ self._loop.add_writer(self._fileno, self._write_ready)
+
+ self._buffer.append(data)
+ self._maybe_pause_protocol()
+
+ def _write_ready(self):
+ data = b''.join(self._buffer)
+ assert data, 'Data should not be empty'
+
+ self._buffer.clear()
+ try:
+ n = os.write(self._fileno, data)
+ except (BlockingIOError, InterruptedError):
+ self._buffer.append(data)
+ except Exception as exc:
+ self._conn_lost += 1
+ # Remove writer here, _fatal_error() doesn't it
+ # because _buffer is empty.
+ self._loop.remove_writer(self._fileno)
+ self._fatal_error(exc, 'Fatal write error on pipe transport')
+ else:
+ if n == len(data):
+ self._loop.remove_writer(self._fileno)
+ self._maybe_resume_protocol() # May append to buffer.
+ if not self._buffer and self._closing:
+ self._loop.remove_reader(self._fileno)
+ self._call_connection_lost(None)
+ return
+ elif n > 0:
+ data = data[n:]
+
+ self._buffer.append(data) # Try again later.
+
+ def can_write_eof(self):
+ return True
+
+ # TODO: Make the relationships between write_eof(), close(),
+ # abort(), _fatal_error() and _close() more straightforward.
+
+ def write_eof(self):
+ if self._closing:
+ return
+ assert self._pipe
+ self._closing = True
+ if not self._buffer:
+ self._loop.remove_reader(self._fileno)
+ self._loop.call_soon(self._call_connection_lost, None)
+
+ def close(self):
+ if not self._closing:
+ # write_eof is all what we needed to close the write pipe
+ self.write_eof()
+
+ def abort(self):
+ self._close(None)
+
+ def _fatal_error(self, exc, message='Fatal error on pipe transport'):
+ # should be called by exception handler only
+ if not isinstance(exc, (BrokenPipeError, ConnectionResetError)):
+ self._loop.call_exception_handler({
+ 'message': message,
+ 'exception': exc,
+ 'transport': self,
+ 'protocol': self._protocol,
+ })
+ self._close(exc)
+
+ def _close(self, exc=None):
+ self._closing = True
+ if self._buffer:
+ self._loop.remove_writer(self._fileno)
+ self._buffer.clear()
+ self._loop.remove_reader(self._fileno)
+ self._loop.call_soon(self._call_connection_lost, exc)
+
+ def _call_connection_lost(self, exc):
+ try:
+ self._protocol.connection_lost(exc)
+ finally:
+ self._pipe.close()
+ self._pipe = None
+ self._protocol = None
+ self._loop = None
+
+
+class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport):
+
+ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
+ stdin_w = None
+ if stdin == subprocess.PIPE:
+ # Use a socket pair for stdin, since not all platforms
+ # support selecting read events on the write end of a
+ # socket (which we use in order to detect closing of the
+ # other end). Notably this is needed on AIX, and works
+ # just fine on other platforms.
+ stdin, stdin_w = self._loop._socketpair()
+ self._proc = subprocess.Popen(
+ args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr,
+ universal_newlines=False, bufsize=bufsize, **kwargs)
+ if stdin_w is not None:
+ stdin.close()
+ self._proc.stdin = open(stdin_w.detach(), 'rb', buffering=bufsize)
+
+
+class AbstractChildWatcher:
+ """Abstract base class for monitoring child processes.
+
+ Objects derived from this class monitor a collection of subprocesses and
+ report their termination or interruption by a signal.
+
+ New callbacks are registered with .add_child_handler(). Starting a new
+ process must be done within a 'with' block to allow the watcher to suspend
+ its activity until the new process if fully registered (this is needed to
+ prevent a race condition in some implementations).
+
+ Example:
+ with watcher:
+ proc = subprocess.Popen("sleep 1")
+ watcher.add_child_handler(proc.pid, callback)
+
+ Notes:
+ Implementations of this class must be thread-safe.
+
+ Since child watcher objects may catch the SIGCHLD signal and call
+ waitpid(-1), there should be only one active object per process.
+ """
+
+ def add_child_handler(self, pid, callback, *args):
+ """Register a new child handler.
+
+ Arrange for callback(pid, returncode, *args) to be called when
+ process 'pid' terminates. Specifying another callback for the same
+ process replaces the previous handler.
+
+ Note: callback() must be thread-safe
+ """
+ raise NotImplementedError()
+
+ def remove_child_handler(self, pid):
+ """Removes the handler for process 'pid'.
+
+ The function returns True if the handler was successfully removed,
+ False if there was nothing to remove."""
+
+ raise NotImplementedError()
+
+ def attach_loop(self, loop):
+ """Attach the watcher to an event loop.
+
+ If the watcher was previously attached to an event loop, then it is
+ first detached before attaching to the new loop.
+
+ Note: loop may be None.
+ """
+ raise NotImplementedError()
+
+ def close(self):
+ """Close the watcher.
+
+ This must be called to make sure that any underlying resource is freed.
+ """
+ raise NotImplementedError()
+
+ def __enter__(self):
+ """Enter the watcher's context and allow starting new processes
+
+ This function must return self"""
+ raise NotImplementedError()
+
+ def __exit__(self, a, b, c):
+ """Exit the watcher's context"""
+ raise NotImplementedError()
+
+
+class BaseChildWatcher(AbstractChildWatcher):
+
+ def __init__(self):
+ self._loop = None
+
+ def close(self):
+ self.attach_loop(None)
+
+ def _do_waitpid(self, expected_pid):
+ raise NotImplementedError()
+
+ def _do_waitpid_all(self):
+ raise NotImplementedError()
+
+ def attach_loop(self, loop):
+ assert loop is None or isinstance(loop, events.AbstractEventLoop)
+
+ if self._loop is not None:
+ self._loop.remove_signal_handler(signal.SIGCHLD)
+
+ self._loop = loop
+ if loop is not None:
+ loop.add_signal_handler(signal.SIGCHLD, self._sig_chld)
+
+ # Prevent a race condition in case a child terminated
+ # during the switch.
+ self._do_waitpid_all()
+
+ def _sig_chld(self):
+ try:
+ self._do_waitpid_all()
+ except Exception as exc:
+ # self._loop should always be available here
+ # as '_sig_chld' is added as a signal handler
+ # in 'attach_loop'
+ self._loop.call_exception_handler({
+ 'message': 'Unknown exception in SIGCHLD handler',
+ 'exception': exc,
+ })
+
+ def _compute_returncode(self, status):
+ if os.WIFSIGNALED(status):
+ # The child process died because of a signal.
+ return -os.WTERMSIG(status)
+ elif os.WIFEXITED(status):
+ # The child process exited (e.g sys.exit()).
+ return os.WEXITSTATUS(status)
+ else:
+ # The child exited, but we don't understand its status.
+ # This shouldn't happen, but if it does, let's just
+ # return that status; perhaps that helps debug it.
+ return status
+
+
+class SafeChildWatcher(BaseChildWatcher):
+ """'Safe' child watcher implementation.
+
+ This implementation avoids disrupting other code spawning processes by
+ polling explicitly each process in the SIGCHLD handler instead of calling
+ os.waitpid(-1).
+
+ This is a safe solution but it has a significant overhead when handling a
+ big number of children (O(n) each time SIGCHLD is raised)
+ """
+
+ def __init__(self):
+ super().__init__()
+ self._callbacks = {}
+
+ def close(self):
+ self._callbacks.clear()
+ super().close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, a, b, c):
+ pass
+
+ def add_child_handler(self, pid, callback, *args):
+ self._callbacks[pid] = callback, args
+
+ # Prevent a race condition in case the child is already terminated.
+ self._do_waitpid(pid)
+
+ def remove_child_handler(self, pid):
+ try:
+ del self._callbacks[pid]
+ return True
+ except KeyError:
+ return False
+
+ def _do_waitpid_all(self):
+
+ for pid in list(self._callbacks):
+ self._do_waitpid(pid)
+
+ def _do_waitpid(self, expected_pid):
+ assert expected_pid > 0
+
+ try:
+ pid, status = os.waitpid(expected_pid, os.WNOHANG)
+ except ChildProcessError:
+ # The child process is already reaped
+ # (may happen if waitpid() is called elsewhere).
+ pid = expected_pid
+ returncode = 255
+ logger.warning(
+ "Unknown child process pid %d, will report returncode 255",
+ pid)
+ else:
+ if pid == 0:
+ # The child process is still alive.
+ return
+
+ returncode = self._compute_returncode(status)
+
+ try:
+ callback, args = self._callbacks.pop(pid)
+ except KeyError: # pragma: no cover
+ # May happen if .remove_child_handler() is called
+ # after os.waitpid() returns.
+ pass
+ else:
+ callback(pid, returncode, *args)
+
+
+class FastChildWatcher(BaseChildWatcher):
+ """'Fast' child watcher implementation.
+
+ This implementation reaps every terminated processes by calling
+ os.waitpid(-1) directly, possibly breaking other code spawning processes
+ and waiting for their termination.
+
+ There is no noticeable overhead when handling a big number of children
+ (O(1) each time a child terminates).
+ """
+ def __init__(self):
+ super().__init__()
+ self._callbacks = {}
+ self._lock = threading.Lock()
+ self._zombies = {}
+ self._forks = 0
+
+ def close(self):
+ self._callbacks.clear()
+ self._zombies.clear()
+ super().close()
+
+ def __enter__(self):
+ with self._lock:
+ self._forks += 1
+
+ return self
+
+ def __exit__(self, a, b, c):
+ with self._lock:
+ self._forks -= 1
+
+ if self._forks or not self._zombies:
+ return
+
+ collateral_victims = str(self._zombies)
+ self._zombies.clear()
+
+ logger.warning(
+ "Caught subprocesses termination from unknown pids: %s",
+ collateral_victims)
+
+ def add_child_handler(self, pid, callback, *args):
+ assert self._forks, "Must use the context manager"
+ with self._lock:
+ try:
+ returncode = self._zombies.pop(pid)
+ except KeyError:
+ # The child is running.
+ self._callbacks[pid] = callback, args
+ return
+
+ # The child is dead already. We can fire the callback.
+ callback(pid, returncode, *args)
+
+ def remove_child_handler(self, pid):
+ try:
+ del self._callbacks[pid]
+ return True
+ except KeyError:
+ return False
+
+ def _do_waitpid_all(self):
+ # Because of signal coalescing, we must keep calling waitpid() as
+ # long as we're able to reap a child.
+ while True:
+ try:
+ pid, status = os.waitpid(-1, os.WNOHANG)
+ except ChildProcessError:
+ # No more child processes exist.
+ return
+ else:
+ if pid == 0:
+ # A child process is still alive.
+ return
+
+ returncode = self._compute_returncode(status)
+
+ with self._lock:
+ try:
+ callback, args = self._callbacks.pop(pid)
+ except KeyError:
+ # unknown child
+ if self._forks:
+ # It may not be registered yet.
+ self._zombies[pid] = returncode
+ continue
+ callback = None
+
+ if callback is None:
+ logger.warning(
+ "Caught subprocess termination from unknown pid: "
+ "%d -> %d", pid, returncode)
+ else:
+ callback(pid, returncode, *args)
+
+
+class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+ """XXX"""
+ _loop_factory = _UnixSelectorEventLoop
+
+ def __init__(self):
+ super().__init__()
+ self._watcher = None
+
+ def _init_watcher(self):
+ with events._lock:
+ if self._watcher is None: # pragma: no branch
+ self._watcher = SafeChildWatcher()
+ if isinstance(threading.current_thread(),
+ threading._MainThread):
+ self._watcher.attach_loop(self._local._loop)
+
+ def set_event_loop(self, loop):
+ """Set the event loop.
+
+ As a side effect, if a child watcher was set before, then calling
+ .set_event_loop() from the main thread will call .attach_loop(loop) on
+ the child watcher.
+ """
+
+ super().set_event_loop(loop)
+
+ if self._watcher is not None and \
+ isinstance(threading.current_thread(), threading._MainThread):
+ self._watcher.attach_loop(loop)
+
+ def get_child_watcher(self):
+ """Get the child watcher
+
+ If not yet set, a SafeChildWatcher object is automatically created.
+ """
+ if self._watcher is None:
+ self._init_watcher()
+
+ return self._watcher
+
+ def set_child_watcher(self, watcher):
+ """Set the child watcher"""
+
+ assert watcher is None or isinstance(watcher, AbstractChildWatcher)
+
+ if self._watcher is not None:
+ self._watcher.close()
+
+ self._watcher = watcher
+
+SelectorEventLoop = _UnixSelectorEventLoop
+DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy
diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py
new file mode 100644
index 0000000..19f2588
--- /dev/null
+++ b/Lib/asyncio/windows_events.py
@@ -0,0 +1,489 @@
+"""Selector and proactor event loops for Windows."""
+
+import _winapi
+import errno
+import math
+import socket
+import struct
+import weakref
+
+from . import events
+from . import base_subprocess
+from . import futures
+from . import proactor_events
+from . import selector_events
+from . import tasks
+from . import windows_utils
+from .log import logger
+from . import _overlapped
+
+
+__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
+ 'DefaultEventLoopPolicy',
+ ]
+
+
+NULL = 0
+INFINITE = 0xffffffff
+ERROR_CONNECTION_REFUSED = 1225
+ERROR_CONNECTION_ABORTED = 1236
+
+
+class _OverlappedFuture(futures.Future):
+ """Subclass of Future which represents an overlapped operation.
+
+ Cancelling it will immediately cancel the overlapped operation.
+ """
+
+ def __init__(self, ov, *, loop=None):
+ super().__init__(loop=loop)
+ self.ov = ov
+
+ def cancel(self):
+ try:
+ self.ov.cancel()
+ except OSError:
+ pass
+ return super().cancel()
+
+
+class _WaitHandleFuture(futures.Future):
+ """Subclass of Future which represents a wait handle."""
+
+ def __init__(self, wait_handle, *, loop=None):
+ super().__init__(loop=loop)
+ self._wait_handle = wait_handle
+
+ def cancel(self):
+ super().cancel()
+ try:
+ _overlapped.UnregisterWait(self._wait_handle)
+ except OSError as e:
+ if e.winerror != _overlapped.ERROR_IO_PENDING:
+ raise
+
+
+class PipeServer(object):
+ """Class representing a pipe server.
+
+ This is much like a bound, listening socket.
+ """
+ def __init__(self, address):
+ self._address = address
+ self._free_instances = weakref.WeakSet()
+ self._pipe = self._server_pipe_handle(True)
+
+ def _get_unconnected_pipe(self):
+ # Create new instance and return previous one. This ensures
+ # that (until the server is closed) there is always at least
+ # one pipe handle for address. Therefore if a client attempt
+ # to connect it will not fail with FileNotFoundError.
+ tmp, self._pipe = self._pipe, self._server_pipe_handle(False)
+ return tmp
+
+ def _server_pipe_handle(self, first):
+ # Return a wrapper for a new pipe handle.
+ if self._address is None:
+ return None
+ flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
+ if first:
+ flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
+ h = _winapi.CreateNamedPipe(
+ self._address, flags,
+ _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
+ _winapi.PIPE_WAIT,
+ _winapi.PIPE_UNLIMITED_INSTANCES,
+ windows_utils.BUFSIZE, windows_utils.BUFSIZE,
+ _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
+ pipe = windows_utils.PipeHandle(h)
+ self._free_instances.add(pipe)
+ return pipe
+
+ def close(self):
+ # Close all instances which have not been connected to by a client.
+ if self._address is not None:
+ for pipe in self._free_instances:
+ pipe.close()
+ self._pipe = None
+ self._address = None
+ self._free_instances.clear()
+
+ __del__ = close
+
+
+class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop):
+ """Windows version of selector event loop."""
+
+ def _socketpair(self):
+ return windows_utils.socketpair()
+
+
+class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
+ """Windows version of proactor event loop using IOCP."""
+
+ def __init__(self, proactor=None):
+ if proactor is None:
+ proactor = IocpProactor()
+ super().__init__(proactor)
+
+ def _socketpair(self):
+ return windows_utils.socketpair()
+
+ @tasks.coroutine
+ def create_pipe_connection(self, protocol_factory, address):
+ f = self._proactor.connect_pipe(address)
+ pipe = yield from f
+ protocol = protocol_factory()
+ trans = self._make_duplex_pipe_transport(pipe, protocol,
+ extra={'addr': address})
+ return trans, protocol
+
+ @tasks.coroutine
+ def start_serving_pipe(self, protocol_factory, address):
+ server = PipeServer(address)
+
+ def loop(f=None):
+ pipe = None
+ try:
+ if f:
+ pipe = f.result()
+ server._free_instances.discard(pipe)
+ protocol = protocol_factory()
+ self._make_duplex_pipe_transport(
+ pipe, protocol, extra={'addr': address})
+ pipe = server._get_unconnected_pipe()
+ if pipe is None:
+ return
+ f = self._proactor.accept_pipe(pipe)
+ except OSError as exc:
+ if pipe and pipe.fileno() != -1:
+ self.call_exception_handler({
+ 'message': 'Pipe accept failed',
+ 'exception': exc,
+ 'pipe': pipe,
+ })
+ pipe.close()
+ except futures.CancelledError:
+ if pipe:
+ pipe.close()
+ else:
+ f.add_done_callback(loop)
+
+ self.call_soon(loop)
+ return [server]
+
+ @tasks.coroutine
+ def _make_subprocess_transport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=None, **kwargs):
+ transp = _WindowsSubprocessTransport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ extra=extra, **kwargs)
+ yield from transp._post_init()
+ return transp
+
+
+class IocpProactor:
+ """Proactor implementation using IOCP."""
+
+ def __init__(self, concurrency=0xffffffff):
+ self._loop = None
+ self._results = []
+ self._iocp = _overlapped.CreateIoCompletionPort(
+ _overlapped.INVALID_HANDLE_VALUE, NULL, 0, concurrency)
+ self._cache = {}
+ self._registered = weakref.WeakSet()
+ self._stopped_serving = weakref.WeakSet()
+
+ def set_loop(self, loop):
+ self._loop = loop
+
+ def select(self, timeout=None):
+ if not self._results:
+ self._poll(timeout)
+ tmp = self._results
+ self._results = []
+ return tmp
+
+ def recv(self, conn, nbytes, flags=0):
+ self._register_with_iocp(conn)
+ ov = _overlapped.Overlapped(NULL)
+ if isinstance(conn, socket.socket):
+ ov.WSARecv(conn.fileno(), nbytes, flags)
+ else:
+ ov.ReadFile(conn.fileno(), nbytes)
+
+ def finish_recv(trans, key, ov):
+ try:
+ return ov.getresult()
+ except OSError as exc:
+ if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
+ raise ConnectionResetError(*exc.args)
+ else:
+ raise
+
+ return self._register(ov, conn, finish_recv)
+
+ def send(self, conn, buf, flags=0):
+ self._register_with_iocp(conn)
+ ov = _overlapped.Overlapped(NULL)
+ if isinstance(conn, socket.socket):
+ ov.WSASend(conn.fileno(), buf, flags)
+ else:
+ ov.WriteFile(conn.fileno(), buf)
+
+ def finish_send(trans, key, ov):
+ try:
+ return ov.getresult()
+ except OSError as exc:
+ if exc.winerror == _overlapped.ERROR_NETNAME_DELETED:
+ raise ConnectionResetError(*exc.args)
+ else:
+ raise
+
+ return self._register(ov, conn, finish_send)
+
+ def accept(self, listener):
+ self._register_with_iocp(listener)
+ conn = self._get_accept_socket(listener.family)
+ ov = _overlapped.Overlapped(NULL)
+ ov.AcceptEx(listener.fileno(), conn.fileno())
+
+ def finish_accept(trans, key, ov):
+ ov.getresult()
+ # Use SO_UPDATE_ACCEPT_CONTEXT so getsockname() etc work.
+ buf = struct.pack('@P', listener.fileno())
+ conn.setsockopt(socket.SOL_SOCKET,
+ _overlapped.SO_UPDATE_ACCEPT_CONTEXT, buf)
+ conn.settimeout(listener.gettimeout())
+ return conn, conn.getpeername()
+
+ @tasks.coroutine
+ def accept_coro(future, conn):
+ # Coroutine closing the accept socket if the future is cancelled
+ try:
+ yield from future
+ except futures.CancelledError:
+ conn.close()
+ raise
+
+ future = self._register(ov, listener, finish_accept)
+ coro = accept_coro(future, conn)
+ tasks.async(coro, loop=self._loop)
+ return future
+
+ def connect(self, conn, address):
+ self._register_with_iocp(conn)
+ # The socket needs to be locally bound before we call ConnectEx().
+ try:
+ _overlapped.BindLocal(conn.fileno(), conn.family)
+ except OSError as e:
+ if e.winerror != errno.WSAEINVAL:
+ raise
+ # Probably already locally bound; check using getsockname().
+ if conn.getsockname()[1] == 0:
+ raise
+ ov = _overlapped.Overlapped(NULL)
+ ov.ConnectEx(conn.fileno(), address)
+
+ def finish_connect(trans, key, ov):
+ ov.getresult()
+ # Use SO_UPDATE_CONNECT_CONTEXT so getsockname() etc work.
+ conn.setsockopt(socket.SOL_SOCKET,
+ _overlapped.SO_UPDATE_CONNECT_CONTEXT, 0)
+ return conn
+
+ return self._register(ov, conn, finish_connect)
+
+ def accept_pipe(self, pipe):
+ self._register_with_iocp(pipe)
+ ov = _overlapped.Overlapped(NULL)
+ ov.ConnectNamedPipe(pipe.fileno())
+
+ def finish_accept_pipe(trans, key, ov):
+ ov.getresult()
+ return pipe
+
+ return self._register(ov, pipe, finish_accept_pipe)
+
+ def connect_pipe(self, address):
+ ov = _overlapped.Overlapped(NULL)
+ ov.WaitNamedPipeAndConnect(address, self._iocp, ov.address)
+
+ def finish_connect_pipe(err, handle, ov):
+ # err, handle were arguments passed to PostQueuedCompletionStatus()
+ # in a function run in a thread pool.
+ if err == _overlapped.ERROR_SEM_TIMEOUT:
+ # Connection did not succeed within time limit.
+ msg = _overlapped.FormatMessage(err)
+ raise ConnectionRefusedError(0, msg, None, err)
+ elif err != 0:
+ msg = _overlapped.FormatMessage(err)
+ raise OSError(0, msg, None, err)
+ else:
+ return windows_utils.PipeHandle(handle)
+
+ return self._register(ov, None, finish_connect_pipe, wait_for_post=True)
+
+ def wait_for_handle(self, handle, timeout=None):
+ if timeout is None:
+ ms = _winapi.INFINITE
+ else:
+ # RegisterWaitForSingleObject() has a resolution of 1 millisecond,
+ # round away from zero to wait *at least* timeout seconds.
+ ms = math.ceil(timeout * 1e3)
+
+ # We only create ov so we can use ov.address as a key for the cache.
+ ov = _overlapped.Overlapped(NULL)
+ wh = _overlapped.RegisterWaitWithQueue(
+ handle, self._iocp, ov.address, ms)
+ f = _WaitHandleFuture(wh, loop=self._loop)
+
+ def finish_wait_for_handle(trans, key, ov):
+ if not f.cancelled():
+ try:
+ _overlapped.UnregisterWait(wh)
+ except OSError as e:
+ if e.winerror != _overlapped.ERROR_IO_PENDING:
+ raise
+ # Note that this second wait means that we should only use
+ # this with handles types where a successful wait has no
+ # effect. So events or processes are all right, but locks
+ # or semaphores are not. Also note if the handle is
+ # signalled and then quickly reset, then we may return
+ # False even though we have not timed out.
+ return (_winapi.WaitForSingleObject(handle, 0) ==
+ _winapi.WAIT_OBJECT_0)
+
+ self._cache[ov.address] = (f, ov, None, finish_wait_for_handle)
+ return f
+
+ def _register_with_iocp(self, obj):
+ # To get notifications of finished ops on this objects sent to the
+ # completion port, were must register the handle.
+ if obj not in self._registered:
+ self._registered.add(obj)
+ _overlapped.CreateIoCompletionPort(obj.fileno(), self._iocp, 0, 0)
+ # XXX We could also use SetFileCompletionNotificationModes()
+ # to avoid sending notifications to completion port of ops
+ # that succeed immediately.
+
+ def _register(self, ov, obj, callback, wait_for_post=False):
+ # Return a future which will be set with the result of the
+ # operation when it completes. The future's value is actually
+ # the value returned by callback().
+ f = _OverlappedFuture(ov, loop=self._loop)
+ if ov.pending or wait_for_post:
+ # Register the overlapped operation for later. Note that
+ # we only store obj to prevent it from being garbage
+ # collected too early.
+ self._cache[ov.address] = (f, ov, obj, callback)
+ else:
+ # The operation has completed, so no need to postpone the
+ # work. We cannot take this short cut if we need the
+ # NumberOfBytes, CompletionKey values returned by
+ # PostQueuedCompletionStatus().
+ try:
+ value = callback(None, None, ov)
+ except OSError as e:
+ f.set_exception(e)
+ else:
+ f.set_result(value)
+ return f
+
+ def _get_accept_socket(self, family):
+ s = socket.socket(family)
+ s.settimeout(0)
+ return s
+
+ def _poll(self, timeout=None):
+ if timeout is None:
+ ms = INFINITE
+ elif timeout < 0:
+ raise ValueError("negative timeout")
+ else:
+ # GetQueuedCompletionStatus() has a resolution of 1 millisecond,
+ # round away from zero to wait *at least* timeout seconds.
+ ms = math.ceil(timeout * 1e3)
+ if ms >= INFINITE:
+ raise ValueError("timeout too big")
+ while True:
+ status = _overlapped.GetQueuedCompletionStatus(self._iocp, ms)
+ if status is None:
+ return
+ err, transferred, key, address = status
+ try:
+ f, ov, obj, callback = self._cache.pop(address)
+ except KeyError:
+ # key is either zero, or it is used to return a pipe
+ # handle which should be closed to avoid a leak.
+ if key not in (0, _overlapped.INVALID_HANDLE_VALUE):
+ _winapi.CloseHandle(key)
+ ms = 0
+ continue
+ if obj in self._stopped_serving:
+ f.cancel()
+ elif not f.cancelled():
+ try:
+ value = callback(transferred, key, ov)
+ except OSError as e:
+ f.set_exception(e)
+ self._results.append(f)
+ else:
+ f.set_result(value)
+ self._results.append(f)
+ ms = 0
+
+ def _stop_serving(self, obj):
+ # obj is a socket or pipe handle. It will be closed in
+ # BaseProactorEventLoop._stop_serving() which will make any
+ # pending operations fail quickly.
+ self._stopped_serving.add(obj)
+
+ def close(self):
+ # Cancel remaining registered operations.
+ for address, (f, ov, obj, callback) in list(self._cache.items()):
+ if obj is None:
+ # The operation was started with connect_pipe() which
+ # queues a task to Windows' thread pool. This cannot
+ # be cancelled, so just forget it.
+ del self._cache[address]
+ else:
+ try:
+ ov.cancel()
+ except OSError:
+ pass
+
+ while self._cache:
+ if not self._poll(1):
+ logger.debug('taking long time to close proactor')
+
+ self._results = []
+ if self._iocp is not None:
+ _winapi.CloseHandle(self._iocp)
+ self._iocp = None
+
+
+class _WindowsSubprocessTransport(base_subprocess.BaseSubprocessTransport):
+
+ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
+ self._proc = windows_utils.Popen(
+ args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr,
+ bufsize=bufsize, **kwargs)
+
+ def callback(f):
+ returncode = self._proc.poll()
+ self._process_exited(returncode)
+
+ f = self._loop._proactor.wait_for_handle(int(self._proc._handle))
+ f.add_done_callback(callback)
+
+
+SelectorEventLoop = _WindowsSelectorEventLoop
+
+
+class _WindowsDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+ _loop_factory = SelectorEventLoop
+
+
+DefaultEventLoopPolicy = _WindowsDefaultEventLoopPolicy
diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py
new file mode 100644
index 0000000..2a196cc
--- /dev/null
+++ b/Lib/asyncio/windows_utils.py
@@ -0,0 +1,205 @@
+"""
+Various Windows specific bits and pieces
+"""
+
+import sys
+
+if sys.platform != 'win32': # pragma: no cover
+ raise ImportError('win32 only')
+
+import socket
+import itertools
+import msvcrt
+import os
+import subprocess
+import tempfile
+import _winapi
+
+
+__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle']
+
+
+# Constants/globals
+
+
+BUFSIZE = 8192
+PIPE = subprocess.PIPE
+STDOUT = subprocess.STDOUT
+_mmap_counter = itertools.count()
+
+
+# Replacement for socket.socketpair()
+
+
+def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
+ """A socket pair usable as a self-pipe, for Windows.
+
+ Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain.
+ """
+ if family == socket.AF_INET:
+ host = '127.0.0.1'
+ elif family == socket.AF_INET6:
+ host = '::1'
+ else:
+ raise ValueError("Ony AF_INET and AF_INET6 socket address families "
+ "are supported")
+ if type != socket.SOCK_STREAM:
+ raise ValueError("Only SOCK_STREAM socket type is supported")
+ if proto != 0:
+ raise ValueError("Only protocol zero is supported")
+
+ # We create a connected TCP socket. Note the trick with setblocking(0)
+ # that prevents us from having to create a thread.
+ lsock = socket.socket(family, type, proto)
+ lsock.bind((host, 0))
+ lsock.listen(1)
+ # On IPv6, ignore flow_info and scope_id
+ addr, port = lsock.getsockname()[:2]
+ csock = socket.socket(family, type, proto)
+ csock.setblocking(False)
+ try:
+ csock.connect((addr, port))
+ except (BlockingIOError, InterruptedError):
+ pass
+ except Exception:
+ lsock.close()
+ csock.close()
+ raise
+ ssock, _ = lsock.accept()
+ csock.setblocking(True)
+ lsock.close()
+ return (ssock, csock)
+
+
+# Replacement for os.pipe() using handles instead of fds
+
+
+def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
+ """Like os.pipe() but with overlapped support and using handles not fds."""
+ address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' %
+ (os.getpid(), next(_mmap_counter)))
+
+ if duplex:
+ openmode = _winapi.PIPE_ACCESS_DUPLEX
+ access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
+ obsize, ibsize = bufsize, bufsize
+ else:
+ openmode = _winapi.PIPE_ACCESS_INBOUND
+ access = _winapi.GENERIC_WRITE
+ obsize, ibsize = 0, bufsize
+
+ openmode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
+
+ if overlapped[0]:
+ openmode |= _winapi.FILE_FLAG_OVERLAPPED
+
+ if overlapped[1]:
+ flags_and_attribs = _winapi.FILE_FLAG_OVERLAPPED
+ else:
+ flags_and_attribs = 0
+
+ h1 = h2 = None
+ try:
+ h1 = _winapi.CreateNamedPipe(
+ address, openmode, _winapi.PIPE_WAIT,
+ 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
+
+ h2 = _winapi.CreateFile(
+ address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
+ flags_and_attribs, _winapi.NULL)
+
+ ov = _winapi.ConnectNamedPipe(h1, overlapped=True)
+ ov.GetOverlappedResult(True)
+ return h1, h2
+ except:
+ if h1 is not None:
+ _winapi.CloseHandle(h1)
+ if h2 is not None:
+ _winapi.CloseHandle(h2)
+ raise
+
+
+# Wrapper for a pipe handle
+
+
+class PipeHandle:
+ """Wrapper for an overlapped pipe handle which is vaguely file-object like.
+
+ The IOCP event loop can use these instead of socket objects.
+ """
+ def __init__(self, handle):
+ self._handle = handle
+
+ @property
+ def handle(self):
+ return self._handle
+
+ def fileno(self):
+ return self._handle
+
+ def close(self, *, CloseHandle=_winapi.CloseHandle):
+ if self._handle != -1:
+ CloseHandle(self._handle)
+ self._handle = -1
+
+ __del__ = close
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, t, v, tb):
+ self.close()
+
+
+# Replacement for subprocess.Popen using overlapped pipe handles
+
+
+class Popen(subprocess.Popen):
+ """Replacement for subprocess.Popen using overlapped pipe handles.
+
+ The stdin, stdout, stderr are None or instances of PipeHandle.
+ """
+ def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwds):
+ assert not kwds.get('universal_newlines')
+ assert kwds.get('bufsize', 0) == 0
+ stdin_rfd = stdout_wfd = stderr_wfd = None
+ stdin_wh = stdout_rh = stderr_rh = None
+ if stdin == PIPE:
+ stdin_rh, stdin_wh = pipe(overlapped=(False, True), duplex=True)
+ stdin_rfd = msvcrt.open_osfhandle(stdin_rh, os.O_RDONLY)
+ else:
+ stdin_rfd = stdin
+ if stdout == PIPE:
+ stdout_rh, stdout_wh = pipe(overlapped=(True, False))
+ stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0)
+ else:
+ stdout_wfd = stdout
+ if stderr == PIPE:
+ stderr_rh, stderr_wh = pipe(overlapped=(True, False))
+ stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0)
+ elif stderr == STDOUT:
+ stderr_wfd = stdout_wfd
+ else:
+ stderr_wfd = stderr
+ try:
+ super().__init__(args, stdin=stdin_rfd, stdout=stdout_wfd,
+ stderr=stderr_wfd, **kwds)
+ except:
+ for h in (stdin_wh, stdout_rh, stderr_rh):
+ if h is not None:
+ _winapi.CloseHandle(h)
+ raise
+ else:
+ if stdin_wh is not None:
+ self.stdin = PipeHandle(stdin_wh)
+ if stdout_rh is not None:
+ self.stdout = PipeHandle(stdout_rh)
+ if stderr_rh is not None:
+ self.stderr = PipeHandle(stderr_rh)
+ finally:
+ if stdin == PIPE:
+ os.close(stdin_rfd)
+ if stdout == PIPE:
+ os.close(stdout_wfd)
+ if stderr == PIPE:
+ os.close(stderr_wfd)
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 909d9f6..75481dd 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -112,7 +112,7 @@ def readwrite(obj, flags):
obj.handle_expt_event()
if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
obj.handle_close()
- except socket.error as e:
+ except OSError as e:
if e.args[0] not in _DISCONNECTED:
obj.handle_error()
else:
@@ -240,7 +240,7 @@ class dispatcher:
# passed be connected.
try:
self.addr = sock.getpeername()
- except socket.error as err:
+ except OSError as err:
if err.args[0] in (ENOTCONN, EINVAL):
# To handle the case where we got an unconnected
# socket.
@@ -304,7 +304,7 @@ class dispatcher:
self.socket.getsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR) | 1
)
- except socket.error:
+ except OSError:
pass
# ==================================================
@@ -345,7 +345,7 @@ class dispatcher:
self.addr = address
self.handle_connect_event()
else:
- raise socket.error(err, errorcode[err])
+ raise OSError(err, errorcode[err])
def accept(self):
# XXX can return either an address pair or None
@@ -353,7 +353,7 @@ class dispatcher:
conn, addr = self.socket.accept()
except TypeError:
return None
- except socket.error as why:
+ except OSError as why:
if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
return None
else:
@@ -365,7 +365,7 @@ class dispatcher:
try:
result = self.socket.send(data)
return result
- except socket.error as why:
+ except OSError as why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
@@ -384,7 +384,7 @@ class dispatcher:
return b''
else:
return data
- except socket.error as why:
+ except OSError as why:
# winsock sometimes raises ENOTCONN
if why.args[0] in _DISCONNECTED:
self.handle_close()
@@ -397,11 +397,12 @@ class dispatcher:
self.accepting = False
self.connecting = False
self.del_channel()
- try:
- self.socket.close()
- except socket.error as why:
- if why.args[0] not in (ENOTCONN, EBADF):
- raise
+ if self.socket is not None:
+ try:
+ self.socket.close()
+ except OSError as why:
+ if why.args[0] not in (ENOTCONN, EBADF):
+ raise
# cheap inheritance, used to pass all other attribute
# references to the underlying socket object.
@@ -443,7 +444,7 @@ class dispatcher:
def handle_connect_event(self):
err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
- raise socket.error(err, _strerror(err))
+ raise OSError(err, _strerror(err))
self.handle_connect()
self.connected = True
self.connecting = False
@@ -532,7 +533,7 @@ class dispatcher_with_send(dispatcher):
def initiate_send(self):
num_sent = 0
- num_sent = dispatcher.send(self, self.out_buffer[:512])
+ num_sent = dispatcher.send(self, self.out_buffer[:65536])
self.out_buffer = self.out_buffer[num_sent:]
def handle_write(self):
diff --git a/Lib/base64.py b/Lib/base64.py
index b6e82b6..36c68a6 100755
--- a/Lib/base64.py
+++ b/Lib/base64.py
@@ -1,6 +1,6 @@
#! /usr/bin/env python3
-"""RFC 3548: Base16, Base32, Base64 Data Encodings"""
+"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
# Modified 04-Oct-1995 by Jack Jansen to use binascii module
# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
@@ -17,6 +17,8 @@ __all__ = [
# Generalized interface for other encodings
'b64encode', 'b64decode', 'b32encode', 'b32decode',
'b16encode', 'b16decode',
+ # Base85 and Ascii85 encodings
+ 'b85encode', 'b85decode', 'a85encode', 'a85decode',
# Standard Base64 encoding
'standard_b64encode', 'standard_b64decode',
# Some common Base64 alternatives. As referenced by RFC 3458, see thread
@@ -35,11 +37,13 @@ def _bytes_from_decode_data(s):
return s.encode('ascii')
except UnicodeEncodeError:
raise ValueError('string argument should contain only ASCII characters')
- elif isinstance(s, bytes_types):
+ if isinstance(s, bytes_types):
return s
- else:
- raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__)
-
+ try:
+ return memoryview(s).tobytes()
+ except TypeError:
+ raise TypeError("argument should be a bytes-like object or ASCII "
+ "string, not %r" % s.__class__.__name__) from None
# Base64 encoding/decoding uses binascii
@@ -54,14 +58,9 @@ def b64encode(s, altchars=None):
The encoded byte string is returned.
"""
- if not isinstance(s, bytes_types):
- raise TypeError("expected bytes, not %s" % s.__class__.__name__)
# Strip off the trailing newline
encoded = binascii.b2a_base64(s)[:-1]
if altchars is not None:
- if not isinstance(altchars, bytes_types):
- raise TypeError("expected bytes, not %s"
- % altchars.__class__.__name__)
assert len(altchars) == 2, repr(altchars)
return encoded.translate(bytes.maketrans(b'+/', altchars))
return encoded
@@ -138,53 +137,39 @@ def urlsafe_b64decode(s):
# Base32 encoding/decoding must be done in Python
-_b32alphabet = {
- 0: b'A', 9: b'J', 18: b'S', 27: b'3',
- 1: b'B', 10: b'K', 19: b'T', 28: b'4',
- 2: b'C', 11: b'L', 20: b'U', 29: b'5',
- 3: b'D', 12: b'M', 21: b'V', 30: b'6',
- 4: b'E', 13: b'N', 22: b'W', 31: b'7',
- 5: b'F', 14: b'O', 23: b'X',
- 6: b'G', 15: b'P', 24: b'Y',
- 7: b'H', 16: b'Q', 25: b'Z',
- 8: b'I', 17: b'R', 26: b'2',
- }
-
-_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())]
-_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()])
-
+_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+_b32tab2 = None
+_b32rev = None
def b32encode(s):
"""Encode a byte string using Base32.
s is the byte string to encode. The encoded byte string is returned.
"""
+ global _b32tab2
+ # Delay the initialization of the table to not waste memory
+ # if the function is never called
+ if _b32tab2 is None:
+ b32tab = [bytes((i,)) for i in _b32alphabet]
+ _b32tab2 = [a + b for a in b32tab for b in b32tab]
+ b32tab = None
+
if not isinstance(s, bytes_types):
- raise TypeError("expected bytes, not %s" % s.__class__.__name__)
- quanta, leftover = divmod(len(s), 5)
+ s = memoryview(s).tobytes()
+ leftover = len(s) % 5
# Pad the last quantum with zero bits if necessary
if leftover:
s = s + bytes(5 - leftover) # Don't use += !
- quanta += 1
encoded = bytearray()
- for i in range(quanta):
- # c1 and c2 are 16 bits wide, c3 is 8 bits wide. The intent of this
- # code is to process the 40 bits in units of 5 bits. So we take the 1
- # leftover bit of c1 and tack it onto c2. Then we take the 2 leftover
- # bits of c2 and tack them onto c3. The shifts and masks are intended
- # to give us values of exactly 5 bits in width.
- c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])
- c2 += (c1 & 1) << 16 # 17 bits wide
- c3 += (c2 & 3) << 8 # 10 bits wide
- encoded += bytes([_b32tab[c1 >> 11], # bits 1 - 5
- _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10
- _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15
- _b32tab[c2 >> 12], # bits 16 - 20 (1 - 5)
- _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)
- _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)
- _b32tab[c3 >> 5], # bits 31 - 35 (1 - 5)
- _b32tab[c3 & 0x1f], # bits 36 - 40 (1 - 5)
- ])
+ from_bytes = int.from_bytes
+ b32tab2 = _b32tab2
+ for i in range(0, len(s), 5):
+ c = from_bytes(s[i: i + 5], 'big')
+ encoded += (b32tab2[c >> 30] + # bits 1 - 10
+ b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
+ b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
+ b32tab2[c & 0x3ff] # bits 31 - 40
+ )
# Adjust for any leftover partial quanta
if leftover == 1:
encoded[-6:] = b'======'
@@ -196,7 +181,6 @@ def b32encode(s):
encoded[-1:] = b'='
return bytes(encoded)
-
def b32decode(s, casefold=False, map01=None):
"""Decode a Base32 encoded byte string.
@@ -216,9 +200,13 @@ def b32decode(s, casefold=False, map01=None):
the input is incorrectly padded or if there are non-alphabet
characters present in the input.
"""
+ global _b32rev
+ # Delay the initialization of the table to not waste memory
+ # if the function is never called
+ if _b32rev is None:
+ _b32rev = {v: k for k, v in enumerate(_b32alphabet)}
s = _bytes_from_decode_data(s)
- quanta, leftover = divmod(len(s), 8)
- if leftover:
+ if len(s) % 8:
raise binascii.Error('Incorrect padding')
# Handle section 2.4 zero and one mapping. The flag map01 will be either
# False, or the character to map the digit 1 (one) to. It should be
@@ -232,42 +220,36 @@ def b32decode(s, casefold=False, map01=None):
# Strip off pad characters from the right. We need to count the pad
# characters because this will tell us how many null bytes to remove from
# the end of the decoded string.
- padchars = 0
- mo = re.search(b'(?P<pad>[=]*)$', s)
- if mo:
- padchars = len(mo.group('pad'))
- if padchars > 0:
- s = s[:-padchars]
+ l = len(s)
+ s = s.rstrip(b'=')
+ padchars = l - len(s)
# Now decode the full quanta
- parts = []
- acc = 0
- shift = 35
- for c in s:
- val = _b32rev.get(c)
- if val is None:
- raise binascii.Error('Non-base32 digit found')
- acc += _b32rev[c] << shift
- shift -= 5
- if shift < 0:
- parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii")))
- acc = 0
- shift = 35
+ decoded = bytearray()
+ b32rev = _b32rev
+ for i in range(0, len(s), 8):
+ quanta = s[i: i + 8]
+ acc = 0
+ try:
+ for c in quanta:
+ acc = (acc << 5) + b32rev[c]
+ except KeyError:
+ raise binascii.Error('Non-base32 digit found') from None
+ decoded += acc.to_bytes(5, 'big')
# Process the last, partial quanta
- last = binascii.unhexlify(bytes('%010x' % acc, "ascii"))
- if padchars == 0:
- last = b'' # No characters
- elif padchars == 1:
- last = last[:-1]
- elif padchars == 3:
- last = last[:-2]
- elif padchars == 4:
- last = last[:-3]
- elif padchars == 6:
- last = last[:-4]
- else:
- raise binascii.Error('Incorrect padding')
- parts.append(last)
- return b''.join(parts)
+ if padchars:
+ acc <<= 5 * padchars
+ last = acc.to_bytes(5, 'big')
+ if padchars == 1:
+ decoded[-5:] = last[:-1]
+ elif padchars == 3:
+ decoded[-5:] = last[:-2]
+ elif padchars == 4:
+ decoded[-5:] = last[:-3]
+ elif padchars == 6:
+ decoded[-5:] = last[:-4]
+ else:
+ raise binascii.Error('Incorrect padding')
+ return bytes(decoded)
@@ -279,8 +261,6 @@ def b16encode(s):
s is the byte string to encode. The encoded byte string is returned.
"""
- if not isinstance(s, bytes_types):
- raise TypeError("expected bytes, not %s" % s.__class__.__name__)
return binascii.hexlify(s).upper()
@@ -302,7 +282,206 @@ def b16decode(s, casefold=False):
raise binascii.Error('Non-base16 digit found')
return binascii.unhexlify(s)
+#
+# Ascii85 encoding/decoding
+#
+
+_a85chars = None
+_a85chars2 = None
+_A85START = b"<~"
+_A85END = b"~>"
+
+def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False):
+ # Helper function for a85encode and b85encode
+ if not isinstance(b, bytes_types):
+ b = memoryview(b).tobytes()
+
+ padding = (-len(b)) % 4
+ if padding:
+ b = b + b'\0' * padding
+ words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b)
+
+ chunks = [b'z' if foldnuls and not word else
+ b'y' if foldspaces and word == 0x20202020 else
+ (chars2[word // 614125] +
+ chars2[word // 85 % 7225] +
+ chars[word % 85])
+ for word in words]
+
+ if padding and not pad:
+ if chunks[-1] == b'z':
+ chunks[-1] = chars[0] * 5
+ chunks[-1] = chunks[-1][:-padding]
+
+ return b''.join(chunks)
+def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
+ """Encode a byte string using Ascii85.
+
+ b is the byte string to encode. The encoded byte string is returned.
+
+ foldspaces is an optional flag that uses the special short sequence 'y'
+ instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This
+ feature is not supported by the "standard" Adobe encoding.
+
+ wrapcol controls whether the output should have newline ('\n') characters
+ added to it. If this is non-zero, each output line will be at most this
+ many characters long.
+
+ pad controls whether the input string is padded to a multiple of 4 before
+ encoding. Note that the btoa implementation always pads.
+
+ adobe controls whether the encoded byte sequence is framed with <~ and ~>,
+ which is used by the Adobe implementation.
+ """
+ global _a85chars, _a85chars2
+ # Delay the initialization of tables to not waste memory
+ # if the function is never called
+ if _a85chars is None:
+ _a85chars = [bytes((i,)) for i in range(33, 118)]
+ _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
+
+ result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces)
+
+ if adobe:
+ result = _A85START + result
+ if wrapcol:
+ wrapcol = max(2 if adobe else 1, wrapcol)
+ chunks = [result[i: i + wrapcol]
+ for i in range(0, len(result), wrapcol)]
+ if adobe:
+ if len(chunks[-1]) + 2 > wrapcol:
+ chunks.append(b'')
+ result = b'\n'.join(chunks)
+ if adobe:
+ result += _A85END
+
+ return result
+
+def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
+ """Decode an Ascii85 encoded byte string.
+
+ s is the byte string to decode.
+
+ foldspaces is a flag that specifies whether the 'y' short sequence should be
+ accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is
+ not supported by the "standard" Adobe encoding.
+
+ adobe controls whether the input sequence is in Adobe Ascii85 format (i.e.
+ is framed with <~ and ~>).
+
+ ignorechars should be a byte string containing characters to ignore from the
+ input. This should only contain whitespace characters, and by default
+ contains all whitespace characters in ASCII.
+ """
+ b = _bytes_from_decode_data(b)
+ if adobe:
+ if not (b.startswith(_A85START) and b.endswith(_A85END)):
+ raise ValueError("Ascii85 encoded byte sequences must be bracketed "
+ "by {!r} and {!r}".format(_A85START, _A85END))
+ b = b[2:-2] # Strip off start/end markers
+ #
+ # We have to go through this stepwise, so as to ignore spaces and handle
+ # special short sequences
+ #
+ packI = struct.Struct('!I').pack
+ decoded = []
+ decoded_append = decoded.append
+ curr = []
+ curr_append = curr.append
+ curr_clear = curr.clear
+ for x in b + b'u' * 4:
+ if b'!'[0] <= x <= b'u'[0]:
+ curr_append(x)
+ if len(curr) == 5:
+ acc = 0
+ for x in curr:
+ acc = 85 * acc + (x - 33)
+ try:
+ decoded_append(packI(acc))
+ except struct.error:
+ raise ValueError('Ascii85 overflow') from None
+ curr_clear()
+ elif x == b'z'[0]:
+ if curr:
+ raise ValueError('z inside Ascii85 5-tuple')
+ decoded_append(b'\0\0\0\0')
+ elif foldspaces and x == b'y'[0]:
+ if curr:
+ raise ValueError('y inside Ascii85 5-tuple')
+ decoded_append(b'\x20\x20\x20\x20')
+ elif x in ignorechars:
+ # Skip whitespace
+ continue
+ else:
+ raise ValueError('Non-Ascii85 digit found: %c' % x)
+
+ result = b''.join(decoded)
+ padding = 4 - len(curr)
+ if padding:
+ # Throw away the extra padding
+ result = result[:-padding]
+ return result
+
+# The following code is originally taken (with permission) from Mercurial
+
+_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")
+_b85chars = None
+_b85chars2 = None
+_b85dec = None
+
+def b85encode(b, pad=False):
+ """Encode an ASCII-encoded byte array in base85 format.
+
+ If pad is true, the input is padded with "\0" so its length is a multiple of
+ 4 characters before encoding.
+ """
+ global _b85chars, _b85chars2
+ # Delay the initialization of tables to not waste memory
+ # if the function is never called
+ if _b85chars is None:
+ _b85chars = [bytes((i,)) for i in _b85alphabet]
+ _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
+ return _85encode(b, _b85chars, _b85chars2, pad)
+
+def b85decode(b):
+ """Decode base85-encoded byte array"""
+ global _b85dec
+ # Delay the initialization of tables to not waste memory
+ # if the function is never called
+ if _b85dec is None:
+ _b85dec = [None] * 256
+ for i, c in enumerate(_b85alphabet):
+ _b85dec[c] = i
+
+ b = _bytes_from_decode_data(b)
+ padding = (-len(b)) % 5
+ b = b + b'~' * padding
+ out = []
+ packI = struct.Struct('!I').pack
+ for i in range(0, len(b), 5):
+ chunk = b[i:i + 5]
+ acc = 0
+ try:
+ for c in chunk:
+ acc = acc * 85 + _b85dec[c]
+ except TypeError:
+ for j, c in enumerate(chunk):
+ if _b85dec[c] is None:
+ raise ValueError('bad base85 character at position %d'
+ % (i + j)) from None
+ raise
+ try:
+ out.append(packI(acc))
+ except struct.error:
+ raise ValueError('base85 overflow in hunk starting at byte %d'
+ % i) from None
+
+ result = b''.join(out)
+ if padding:
+ result = result[:-padding]
+ return result
# Legacy interface. This code could be cleaned up since I don't believe
# binascii has any line length limitations. It just doesn't seem worth it
@@ -335,12 +514,26 @@ def decode(input, output):
s = binascii.a2b_base64(line)
output.write(s)
+def _input_type_check(s):
+ try:
+ m = memoryview(s)
+ except TypeError as err:
+ msg = "expected bytes-like object, not %s" % s.__class__.__name__
+ raise TypeError(msg) from err
+ if m.format not in ('c', 'b', 'B'):
+ msg = ("expected single byte elements, not %r from %s" %
+ (m.format, s.__class__.__name__))
+ raise TypeError(msg)
+ if m.ndim != 1:
+ msg = ("expected 1-D data, not %d-D data from %s" %
+ (m.ndim, s.__class__.__name__))
+ raise TypeError(msg)
+
def encodebytes(s):
"""Encode a bytestring into a bytestring containing multiple lines
of base-64 data."""
- if not isinstance(s, bytes_types):
- raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+ _input_type_check(s)
pieces = []
for i in range(0, len(s), MAXBINSIZE):
chunk = s[i : i + MAXBINSIZE]
@@ -357,8 +550,7 @@ def encodestring(s):
def decodebytes(s):
"""Decode a bytestring of base-64 data into a bytestring."""
- if not isinstance(s, bytes_types):
- raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+ _input_type_check(s)
return binascii.a2b_base64(s)
def decodestring(s):
diff --git a/Lib/bdb.py b/Lib/bdb.py
index dd1f4287..67a0846 100644
--- a/Lib/bdb.py
+++ b/Lib/bdb.py
@@ -3,6 +3,7 @@
import fnmatch
import sys
import os
+from inspect import CO_GENERATOR
__all__ = ["BdbQuit", "Bdb", "Breakpoint"]
@@ -75,24 +76,48 @@ class Bdb:
if not (self.stop_here(frame) or self.break_anywhere(frame)):
# No need to trace this function
return # None
+ # Ignore call events in generator except when stepping.
+ if self.stopframe and frame.f_code.co_flags & CO_GENERATOR:
+ return self.trace_dispatch
self.user_call(frame, arg)
if self.quitting: raise BdbQuit
return self.trace_dispatch
def dispatch_return(self, frame, arg):
if self.stop_here(frame) or frame == self.returnframe:
+ # Ignore return events in generator except when stepping.
+ if self.stopframe and frame.f_code.co_flags & CO_GENERATOR:
+ return self.trace_dispatch
try:
self.frame_returning = frame
self.user_return(frame, arg)
finally:
self.frame_returning = None
if self.quitting: raise BdbQuit
+ # The user issued a 'next' or 'until' command.
+ if self.stopframe is frame and self.stoplineno != -1:
+ self._set_stopinfo(None, None)
return self.trace_dispatch
def dispatch_exception(self, frame, arg):
if self.stop_here(frame):
+ # When stepping with next/until/return in a generator frame, skip
+ # the internal StopIteration exception (with no traceback)
+ # triggered by a subiterator run with the 'yield from' statement.
+ if not (frame.f_code.co_flags & CO_GENERATOR
+ and arg[0] is StopIteration and arg[2] is None):
+ self.user_exception(frame, arg)
+ if self.quitting: raise BdbQuit
+ # Stop at the StopIteration or GeneratorExit exception when the user
+ # has set stopframe in a generator by issuing a return command, or a
+ # next/until command at the last statement in the generator before the
+ # exception.
+ elif (self.stopframe and frame is not self.stopframe
+ and self.stopframe.f_code.co_flags & CO_GENERATOR
+ and arg[0] in (StopIteration, GeneratorExit)):
self.user_exception(frame, arg)
if self.quitting: raise BdbQuit
+
return self.trace_dispatch
# Normally derived classes don't override the following
@@ -115,10 +140,8 @@ class Bdb:
if self.stoplineno == -1:
return False
return frame.f_lineno >= self.stoplineno
- while frame is not None and frame is not self.stopframe:
- if frame is self.botframe:
- return True
- frame = frame.f_back
+ if not self.stopframe:
+ return True
return False
def break_here(self, frame):
@@ -207,7 +230,10 @@ class Bdb:
def set_return(self, frame):
"""Stop when returning from the given frame."""
- self._set_stopinfo(frame.f_back, frame)
+ if frame.f_code.co_flags & CO_GENERATOR:
+ self._set_stopinfo(frame, None, -1)
+ else:
+ self._set_stopinfo(frame.f_back, frame)
def set_trace(self, frame=None):
"""Start debugging from `frame`.
diff --git a/Lib/bz2.py b/Lib/bz2.py
index 1de8f3c..6f47bfa 100644
--- a/Lib/bz2.py
+++ b/Lib/bz2.py
@@ -9,7 +9,6 @@ __all__ = ["BZ2File", "BZ2Compressor", "BZ2Decompressor",
__author__ = "Nadeem Vawda <nadeem.vawda@gmail.com>"
-import builtins
import io
import warnings
@@ -28,6 +27,8 @@ _MODE_WRITE = 3
_BUFFER_SIZE = 8192
+_builtin_open = open
+
class BZ2File(io.BufferedIOBase):
@@ -43,16 +44,17 @@ class BZ2File(io.BufferedIOBase):
def __init__(self, filename, mode="r", buffering=None, compresslevel=9):
"""Open a bzip2-compressed file.
- If filename is a str or bytes object, is gives the name of the file to
- be opened. Otherwise, it should be a file object, which will be used to
- read or write the compressed data.
+ If filename is a str or bytes object, it gives the name
+ of the file to be opened. Otherwise, it should be a file object,
+ which will be used to read or write the compressed data.
- mode can be 'r' for reading (default), 'w' for (over)writing, or 'a' for
- appending. These can equivalently be given as 'rb', 'wb', and 'ab'.
+ mode can be 'r' for reading (default), 'w' for (over)writing,
+ 'x' for creating exclusively, or 'a' for appending. These can
+ equivalently be given as 'rb', 'wb', 'xb', and 'ab'.
buffering is ignored. Its use is deprecated.
- If mode is 'w' or 'a', compresslevel can be a number between 1
+ If mode is 'w', 'x' or 'a', compresslevel can be a number between 1
and 9 specifying the level of compression: 1 produces the least
compression, and 9 (default) produces the most compression.
@@ -85,15 +87,19 @@ class BZ2File(io.BufferedIOBase):
mode = "wb"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
+ elif mode in ("x", "xb"):
+ mode = "xb"
+ mode_code = _MODE_WRITE
+ self._compressor = BZ2Compressor(compresslevel)
elif mode in ("a", "ab"):
mode = "ab"
mode_code = _MODE_WRITE
self._compressor = BZ2Compressor(compresslevel)
else:
- raise ValueError("Invalid mode: {!r}".format(mode))
+ raise ValueError("Invalid mode: %r" % (mode,))
if isinstance(filename, (str, bytes)):
- self._fp = builtins.open(filename, mode)
+ self._fp = _builtin_open(filename, mode)
self._closefp = True
self._mode = mode_code
elif hasattr(filename, "read") or hasattr(filename, "write"):
@@ -189,15 +195,17 @@ class BZ2File(io.BufferedIOBase):
if not rawblock:
if self._decompressor.eof:
+ # End-of-stream marker and end of file. We're good.
self._mode = _MODE_READ_EOF
self._size = self._pos
return False
else:
+ # Problem - we were expecting more compressed data.
raise EOFError("Compressed file ended before the "
"end-of-stream marker was reached")
- # Continue to next stream.
if self._decompressor.eof:
+ # Continue to next stream.
self._decompressor = BZ2Decompressor()
try:
self._buffer = self._decompressor.decompress(rawblock)
@@ -419,7 +427,7 @@ class BZ2File(io.BufferedIOBase):
self._read_all(return_data=False)
offset = self._size + offset
else:
- raise ValueError("Invalid value for whence: {}".format(whence))
+ raise ValueError("Invalid value for whence: %s" % (whence,))
# Make it so that offset is the number of bytes to skip forward.
if offset < self._pos:
@@ -443,20 +451,20 @@ def open(filename, mode="rb", compresslevel=9,
encoding=None, errors=None, newline=None):
"""Open a bzip2-compressed file in binary or text mode.
- The filename argument can be an actual filename (a str or bytes object), or
- an existing file object to read from or write to.
+ The filename argument can be an actual filename (a str or bytes
+ object), or an existing file object to read from or write to.
- The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode,
- or "rt", "wt" or "at" for text mode. The default mode is "rb", and the
- default compresslevel is 9.
+ The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or
+ "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode.
+ The default mode is "rb", and the default compresslevel is 9.
- For binary mode, this function is equivalent to the BZ2File constructor:
- BZ2File(filename, mode, compresslevel). In this case, the encoding, errors
- and newline arguments must not be provided.
+ For binary mode, this function is equivalent to the BZ2File
+ constructor: BZ2File(filename, mode, compresslevel). In this case,
+ the encoding, errors and newline arguments must not be provided.
For text mode, a BZ2File object is created, and wrapped in an
- io.TextIOWrapper instance with the specified encoding, error handling
- behavior, and line ending(s).
+ io.TextIOWrapper instance with the specified encoding, error
+ handling behavior, and line ending(s).
"""
if "t" in mode:
diff --git a/Lib/cProfile.py b/Lib/cProfile.py
index c24d45b..1184385 100755
--- a/Lib/cProfile.py
+++ b/Lib/cProfile.py
@@ -7,54 +7,20 @@
__all__ = ["run", "runctx", "Profile"]
import _lsprof
+import profile as _pyprofile
# ____________________________________________________________
# Simple interface
def run(statement, filename=None, sort=-1):
- """Run statement under profiler optionally saving results in filename
-
- This function takes a single argument that can be passed to the
- "exec" statement, and an optional file name. In all cases this
- routine attempts to "exec" its first argument and gather profiling
- statistics from the execution. If no file name is present, then this
- function automatically prints a simple profiling report, sorted by the
- standard name string (file/line/function-name) that is presented in
- each line.
- """
- prof = Profile()
- result = None
- try:
- try:
- prof = prof.run(statement)
- except SystemExit:
- pass
- finally:
- if filename is not None:
- prof.dump_stats(filename)
- else:
- result = prof.print_stats(sort)
- return result
+ return _pyprofile._Utils(Profile).run(statement, filename, sort)
def runctx(statement, globals, locals, filename=None, sort=-1):
- """Run statement under profiler, supplying your own globals and locals,
- optionally saving results in filename.
+ return _pyprofile._Utils(Profile).runctx(statement, globals, locals,
+ filename, sort)
- statement and filename have the same semantics as profile.run
- """
- prof = Profile()
- result = None
- try:
- try:
- prof = prof.runctx(statement, globals, locals)
- except SystemExit:
- pass
- finally:
- if filename is not None:
- prof.dump_stats(filename)
- else:
- result = prof.print_stats(sort)
- return result
+run.__doc__ = _pyprofile.run.__doc__
+runctx.__doc__ = _pyprofile.runctx.__doc__
# ____________________________________________________________
@@ -77,10 +43,9 @@ class Profile(_lsprof.Profiler):
def dump_stats(self, file):
import marshal
- f = open(file, 'wb')
- self.create_stats()
- marshal.dump(self.stats, f)
- f.close()
+ with open(file, 'wb') as f:
+ self.create_stats()
+ marshal.dump(self.stats, f)
def create_stats(self):
self.disable()
diff --git a/Lib/cgi.py b/Lib/cgi.py
index 0f50d0e..a919742 100755
--- a/Lib/cgi.py
+++ b/Lib/cgi.py
@@ -82,7 +82,7 @@ def initlog(*allargs):
if logfile and not logfp:
try:
logfp = open(logfile, "a")
- except IOError:
+ except OSError:
pass
if not logfp:
log = nolog
@@ -560,6 +560,12 @@ class FieldStorage:
else:
self.read_single()
+ def __del__(self):
+ try:
+ self.file.close()
+ except AttributeError:
+ pass
+
def __repr__(self):
"""Return a printable representation."""
return "FieldStorage(%r, %r, %r)" % (
@@ -680,7 +686,6 @@ class FieldStorage:
encoding=self.encoding, errors=self.errors)
for key, value in query:
self.list.append(MiniFieldStorage(key, value))
- FieldStorageClass = None
klass = self.FieldStorageClass or self.__class__
first_line = self.fp.readline() # bytes
@@ -968,8 +973,8 @@ def print_directory():
print("<H3>Current Working Directory:</H3>")
try:
pwd = os.getcwd()
- except os.error as msg:
- print("os.error:", html.escape(str(msg)))
+ except OSError as msg:
+ print("OSError:", html.escape(str(msg)))
else:
print(html.escape(pwd))
print()
diff --git a/Lib/chunk.py b/Lib/chunk.py
index 5863ed0..dc90a75 100644
--- a/Lib/chunk.py
+++ b/Lib/chunk.py
@@ -70,7 +70,7 @@ class Chunk:
self.size_read = 0
try:
self.offset = self.file.tell()
- except (AttributeError, IOError):
+ except (AttributeError, OSError):
self.seekable = False
else:
self.seekable = True
@@ -102,7 +102,7 @@ class Chunk:
if self.closed:
raise ValueError("I/O operation on closed file")
if not self.seekable:
- raise IOError("cannot seek")
+ raise OSError("cannot seek")
if whence == 1:
pos = pos + self.size_read
elif whence == 2:
@@ -158,7 +158,7 @@ class Chunk:
self.file.seek(n, 1)
self.size_read = self.size_read + n
return
- except IOError:
+ except OSError:
pass
while self.size_read < self.chunksize:
n = min(8192, self.chunksize - self.size_read)
diff --git a/Lib/code.py b/Lib/code.py
index 9020aab..f8184b6 100644
--- a/Lib/code.py
+++ b/Lib/code.py
@@ -216,7 +216,7 @@ class InteractiveConsole(InteractiveInterpreter):
self.write("Python %s on %s\n%s\n(%s)\n" %
(sys.version, sys.platform, cprt,
self.__class__.__name__))
- else:
+ elif banner:
self.write("%s\n" % str(banner))
more = 0
while 1:
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
index d737295..d6deb6a 100644
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -3,16 +3,16 @@ __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
# For backwards compatibility, continue to make the collections ABCs
# available through the collections module.
-from collections.abc import *
-import collections.abc
-__all__ += collections.abc.__all__
+from _collections_abc import *
+import _collections_abc
+__all__ += _collections_abc.__all__
from _collections import deque, defaultdict
from operator import itemgetter as _itemgetter, eq as _eq
from keyword import iskeyword as _iskeyword
import sys as _sys
import heapq as _heapq
-from weakref import proxy as _proxy
+from _weakref import proxy as _proxy
from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
from reprlib import recursive_repr as _recursive_repr
@@ -199,13 +199,10 @@ class OrderedDict(dict):
def __reduce__(self):
'Return state information for pickling'
- items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
inst_dict.pop(k, None)
- if inst_dict:
- return (self.__class__, (items,), inst_dict)
- return self.__class__, (items,)
+ return self.__class__, (), inst_dict or None, None, iter(self.items())
def copy(self):
'od.copy() -> a shallow copy of od'
@@ -277,9 +274,7 @@ class {typename}(tuple):
return OrderedDict(zip(self._fields, self))
def _asdict(self):
- '''Return a new OrderedDict which maps field names to their values.
- This method is obsolete. Use vars(nt) or nt.__dict__ instead.
- '''
+ 'Return a new OrderedDict which maps field names to their values.'
return self.__dict__
def __getnewargs__(self):
@@ -822,9 +817,14 @@ class ChainMap(MutableMapping):
__copy__ = copy
- def new_child(self): # like Django's Context.push()
- 'New ChainMap with a new dict followed by all previous maps.'
- return self.__class__({}, *self.maps)
+ def new_child(self, m=None): # like Django's Context.push()
+ '''
+ New ChainMap with a new map followed by all previous maps. If no
+ map is provided, an empty dict is used.
+ '''
+ if m is None:
+ m = {}
+ return self.__class__(m, *self.maps)
@property
def parents(self): # like Django's Context.pop()
diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py
index 7939268..891600d 100644
--- a/Lib/collections/abc.py
+++ b/Lib/collections/abc.py
@@ -1,728 +1,2 @@
-# Copyright 2007 Google, Inc. All Rights Reserved.
-# Licensed to PSF under a Contributor Agreement.
-
-"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
-
-Unit tests are in test_collections.
-"""
-
-from abc import ABCMeta, abstractmethod
-import sys
-
-__all__ = ["Hashable", "Iterable", "Iterator",
- "Sized", "Container", "Callable",
- "Set", "MutableSet",
- "Mapping", "MutableMapping",
- "MappingView", "KeysView", "ItemsView", "ValuesView",
- "Sequence", "MutableSequence",
- "ByteString",
- ]
-
-# Private list of types that we want to register with the various ABCs
-# so that they will pass tests like:
-# it = iter(somebytearray)
-# assert isinstance(it, Iterable)
-# Note: in other implementations, these types many not be distinct
-# and they make have their own implementation specific types that
-# are not included on this list.
-bytes_iterator = type(iter(b''))
-bytearray_iterator = type(iter(bytearray()))
-#callable_iterator = ???
-dict_keyiterator = type(iter({}.keys()))
-dict_valueiterator = type(iter({}.values()))
-dict_itemiterator = type(iter({}.items()))
-list_iterator = type(iter([]))
-list_reverseiterator = type(iter(reversed([])))
-range_iterator = type(iter(range(0)))
-set_iterator = type(iter(set()))
-str_iterator = type(iter(""))
-tuple_iterator = type(iter(()))
-zip_iterator = type(iter(zip()))
-## views ##
-dict_keys = type({}.keys())
-dict_values = type({}.values())
-dict_items = type({}.items())
-## misc ##
-mappingproxy = type(type.__dict__)
-
-
-### ONE-TRICK PONIES ###
-
-class Hashable(metaclass=ABCMeta):
-
- __slots__ = ()
-
- @abstractmethod
- def __hash__(self):
- return 0
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Hashable:
- for B in C.__mro__:
- if "__hash__" in B.__dict__:
- if B.__dict__["__hash__"]:
- return True
- break
- return NotImplemented
-
-
-class Iterable(metaclass=ABCMeta):
-
- __slots__ = ()
-
- @abstractmethod
- def __iter__(self):
- while False:
- yield None
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Iterable:
- if any("__iter__" in B.__dict__ for B in C.__mro__):
- return True
- return NotImplemented
-
-
-class Iterator(Iterable):
-
- __slots__ = ()
-
- @abstractmethod
- def __next__(self):
- 'Return the next item from the iterator. When exhausted, raise StopIteration'
- raise StopIteration
-
- def __iter__(self):
- return self
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Iterator:
- if (any("__next__" in B.__dict__ for B in C.__mro__) and
- any("__iter__" in B.__dict__ for B in C.__mro__)):
- return True
- return NotImplemented
-
-Iterator.register(bytes_iterator)
-Iterator.register(bytearray_iterator)
-#Iterator.register(callable_iterator)
-Iterator.register(dict_keyiterator)
-Iterator.register(dict_valueiterator)
-Iterator.register(dict_itemiterator)
-Iterator.register(list_iterator)
-Iterator.register(list_reverseiterator)
-Iterator.register(range_iterator)
-Iterator.register(set_iterator)
-Iterator.register(str_iterator)
-Iterator.register(tuple_iterator)
-Iterator.register(zip_iterator)
-
-class Sized(metaclass=ABCMeta):
-
- __slots__ = ()
-
- @abstractmethod
- def __len__(self):
- return 0
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Sized:
- if any("__len__" in B.__dict__ for B in C.__mro__):
- return True
- return NotImplemented
-
-
-class Container(metaclass=ABCMeta):
-
- __slots__ = ()
-
- @abstractmethod
- def __contains__(self, x):
- return False
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Container:
- if any("__contains__" in B.__dict__ for B in C.__mro__):
- return True
- return NotImplemented
-
-
-class Callable(metaclass=ABCMeta):
-
- __slots__ = ()
-
- @abstractmethod
- def __call__(self, *args, **kwds):
- return False
-
- @classmethod
- def __subclasshook__(cls, C):
- if cls is Callable:
- if any("__call__" in B.__dict__ for B in C.__mro__):
- return True
- return NotImplemented
-
-
-### SETS ###
-
-
-class Set(Sized, Iterable, Container):
-
- """A set is a finite, iterable container.
-
- This class provides concrete generic implementations of all
- methods except for __contains__, __iter__ and __len__.
-
- To override the comparisons (presumably for speed, as the
- semantics are fixed), all you have to do is redefine __le__ and
- then the other operations will automatically follow suit.
- """
-
- __slots__ = ()
-
- def __le__(self, other):
- if not isinstance(other, Set):
- return NotImplemented
- if len(self) > len(other):
- return False
- for elem in self:
- if elem not in other:
- return False
- return True
-
- def __lt__(self, other):
- if not isinstance(other, Set):
- return NotImplemented
- return len(self) < len(other) and self.__le__(other)
-
- def __gt__(self, other):
- if not isinstance(other, Set):
- return NotImplemented
- return other.__lt__(self)
-
- def __ge__(self, other):
- if not isinstance(other, Set):
- return NotImplemented
- return other.__le__(self)
-
- def __eq__(self, other):
- if not isinstance(other, Set):
- return NotImplemented
- return len(self) == len(other) and self.__le__(other)
-
- def __ne__(self, other):
- return not (self == other)
-
- @classmethod
- def _from_iterable(cls, it):
- '''Construct an instance of the class from any iterable input.
-
- Must override this method if the class constructor signature
- does not accept an iterable for an input.
- '''
- return cls(it)
-
- def __and__(self, other):
- if not isinstance(other, Iterable):
- return NotImplemented
- return self._from_iterable(value for value in other if value in self)
-
- def isdisjoint(self, other):
- 'Return True if two sets have a null intersection.'
- for value in other:
- if value in self:
- return False
- return True
-
- def __or__(self, other):
- if not isinstance(other, Iterable):
- return NotImplemented
- chain = (e for s in (self, other) for e in s)
- return self._from_iterable(chain)
-
- def __sub__(self, other):
- if not isinstance(other, Set):
- if not isinstance(other, Iterable):
- return NotImplemented
- other = self._from_iterable(other)
- return self._from_iterable(value for value in self
- if value not in other)
-
- def __xor__(self, other):
- if not isinstance(other, Set):
- if not isinstance(other, Iterable):
- return NotImplemented
- other = self._from_iterable(other)
- return (self - other) | (other - self)
-
- def _hash(self):
- """Compute the hash value of a set.
-
- Note that we don't define __hash__: not all sets are hashable.
- But if you define a hashable set type, its __hash__ should
- call this function.
-
- This must be compatible __eq__.
-
- All sets ought to compare equal if they contain the same
- elements, regardless of how they are implemented, and
- regardless of the order of the elements; so there's not much
- freedom for __eq__ or __hash__. We match the algorithm used
- by the built-in frozenset type.
- """
- MAX = sys.maxsize
- MASK = 2 * MAX + 1
- n = len(self)
- h = 1927868237 * (n + 1)
- h &= MASK
- for x in self:
- hx = hash(x)
- h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
- h &= MASK
- h = h * 69069 + 907133923
- h &= MASK
- if h > MAX:
- h -= MASK + 1
- if h == -1:
- h = 590923713
- return h
-
-Set.register(frozenset)
-
-
-class MutableSet(Set):
- """A mutable set is a finite, iterable container.
-
- This class provides concrete generic implementations of all
- methods except for __contains__, __iter__, __len__,
- add(), and discard().
-
- To override the comparisons (presumably for speed, as the
- semantics are fixed), all you have to do is redefine __le__ and
- then the other operations will automatically follow suit.
- """
-
- __slots__ = ()
-
- @abstractmethod
- def add(self, value):
- """Add an element."""
- raise NotImplementedError
-
- @abstractmethod
- def discard(self, value):
- """Remove an element. Do not raise an exception if absent."""
- raise NotImplementedError
-
- def remove(self, value):
- """Remove an element. If not a member, raise a KeyError."""
- if value not in self:
- raise KeyError(value)
- self.discard(value)
-
- def pop(self):
- """Return the popped value. Raise KeyError if empty."""
- it = iter(self)
- try:
- value = next(it)
- except StopIteration:
- raise KeyError
- self.discard(value)
- return value
-
- def clear(self):
- """This is slow (creates N new iterators!) but effective."""
- try:
- while True:
- self.pop()
- except KeyError:
- pass
-
- def __ior__(self, it):
- for value in it:
- self.add(value)
- return self
-
- def __iand__(self, it):
- for value in (self - it):
- self.discard(value)
- return self
-
- def __ixor__(self, it):
- if it is self:
- self.clear()
- else:
- if not isinstance(it, Set):
- it = self._from_iterable(it)
- for value in it:
- if value in self:
- self.discard(value)
- else:
- self.add(value)
- return self
-
- def __isub__(self, it):
- if it is self:
- self.clear()
- else:
- for value in it:
- self.discard(value)
- return self
-
-MutableSet.register(set)
-
-
-### MAPPINGS ###
-
-
-class Mapping(Sized, Iterable, Container):
-
- __slots__ = ()
-
- """A Mapping is a generic container for associating key/value
- pairs.
-
- This class provides concrete generic implementations of all
- methods except for __getitem__, __iter__, and __len__.
-
- """
-
- @abstractmethod
- def __getitem__(self, key):
- raise KeyError
-
- def get(self, key, default=None):
- 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.'
- try:
- return self[key]
- except KeyError:
- return default
-
- def __contains__(self, key):
- try:
- self[key]
- except KeyError:
- return False
- else:
- return True
-
- def keys(self):
- "D.keys() -> a set-like object providing a view on D's keys"
- return KeysView(self)
-
- def items(self):
- "D.items() -> a set-like object providing a view on D's items"
- return ItemsView(self)
-
- def values(self):
- "D.values() -> an object providing a view on D's values"
- return ValuesView(self)
-
- def __eq__(self, other):
- if not isinstance(other, Mapping):
- return NotImplemented
- return dict(self.items()) == dict(other.items())
-
- def __ne__(self, other):
- return not (self == other)
-
-Mapping.register(mappingproxy)
-
-
-class MappingView(Sized):
-
- def __init__(self, mapping):
- self._mapping = mapping
-
- def __len__(self):
- return len(self._mapping)
-
- def __repr__(self):
- return '{0.__class__.__name__}({0._mapping!r})'.format(self)
-
-
-class KeysView(MappingView, Set):
-
- @classmethod
- def _from_iterable(self, it):
- return set(it)
-
- def __contains__(self, key):
- return key in self._mapping
-
- def __iter__(self):
- for key in self._mapping:
- yield key
-
-KeysView.register(dict_keys)
-
-
-class ItemsView(MappingView, Set):
-
- @classmethod
- def _from_iterable(self, it):
- return set(it)
-
- def __contains__(self, item):
- key, value = item
- try:
- v = self._mapping[key]
- except KeyError:
- return False
- else:
- return v == value
-
- def __iter__(self):
- for key in self._mapping:
- yield (key, self._mapping[key])
-
-ItemsView.register(dict_items)
-
-
-class ValuesView(MappingView):
-
- def __contains__(self, value):
- for key in self._mapping:
- if value == self._mapping[key]:
- return True
- return False
-
- def __iter__(self):
- for key in self._mapping:
- yield self._mapping[key]
-
-ValuesView.register(dict_values)
-
-
-class MutableMapping(Mapping):
-
- __slots__ = ()
-
- """A MutableMapping is a generic container for associating
- key/value pairs.
-
- This class provides concrete generic implementations of all
- methods except for __getitem__, __setitem__, __delitem__,
- __iter__, and __len__.
-
- """
-
- @abstractmethod
- def __setitem__(self, key, value):
- raise KeyError
-
- @abstractmethod
- def __delitem__(self, key):
- raise KeyError
-
- __marker = object()
-
- def pop(self, key, default=__marker):
- '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
- If key is not found, d is returned if given, otherwise KeyError is raised.
- '''
- try:
- value = self[key]
- except KeyError:
- if default is self.__marker:
- raise
- return default
- else:
- del self[key]
- return value
-
- def popitem(self):
- '''D.popitem() -> (k, v), remove and return some (key, value) pair
- as a 2-tuple; but raise KeyError if D is empty.
- '''
- try:
- key = next(iter(self))
- except StopIteration:
- raise KeyError
- value = self[key]
- del self[key]
- return key, value
-
- def clear(self):
- 'D.clear() -> None. Remove all items from D.'
- try:
- while True:
- self.popitem()
- except KeyError:
- pass
-
- def update(*args, **kwds):
- ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.
- If E present and has a .keys() method, does: for k in E: D[k] = E[k]
- If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v
- In either case, this is followed by: for k, v in F.items(): D[k] = v
- '''
- if len(args) > 2:
- raise TypeError("update() takes at most 2 positional "
- "arguments ({} given)".format(len(args)))
- elif not args:
- raise TypeError("update() takes at least 1 argument (0 given)")
- self = args[0]
- other = args[1] if len(args) >= 2 else ()
-
- if isinstance(other, Mapping):
- for key in other:
- self[key] = other[key]
- elif hasattr(other, "keys"):
- for key in other.keys():
- self[key] = other[key]
- else:
- for key, value in other:
- self[key] = value
- for key, value in kwds.items():
- self[key] = value
-
- def setdefault(self, key, default=None):
- 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D'
- try:
- return self[key]
- except KeyError:
- self[key] = default
- return default
-
-MutableMapping.register(dict)
-
-
-### SEQUENCES ###
-
-
-class Sequence(Sized, Iterable, Container):
-
- """All the operations on a read-only sequence.
-
- Concrete subclasses must override __new__ or __init__,
- __getitem__, and __len__.
- """
-
- __slots__ = ()
-
- @abstractmethod
- def __getitem__(self, index):
- raise IndexError
-
- def __iter__(self):
- i = 0
- try:
- while True:
- v = self[i]
- yield v
- i += 1
- except IndexError:
- return
-
- def __contains__(self, value):
- for v in self:
- if v == value:
- return True
- return False
-
- def __reversed__(self):
- for i in reversed(range(len(self))):
- yield self[i]
-
- def index(self, value):
- '''S.index(value) -> integer -- return first index of value.
- Raises ValueError if the value is not present.
- '''
- for i, v in enumerate(self):
- if v == value:
- return i
- raise ValueError
-
- def count(self, value):
- 'S.count(value) -> integer -- return number of occurrences of value'
- return sum(1 for v in self if v == value)
-
-Sequence.register(tuple)
-Sequence.register(str)
-Sequence.register(range)
-
-
-class ByteString(Sequence):
-
- """This unifies bytes and bytearray.
-
- XXX Should add all their methods.
- """
-
- __slots__ = ()
-
-ByteString.register(bytes)
-ByteString.register(bytearray)
-
-
-class MutableSequence(Sequence):
-
- __slots__ = ()
-
- """All the operations on a read-only sequence.
-
- Concrete subclasses must provide __new__ or __init__,
- __getitem__, __setitem__, __delitem__, __len__, and insert().
-
- """
-
- @abstractmethod
- def __setitem__(self, index, value):
- raise IndexError
-
- @abstractmethod
- def __delitem__(self, index):
- raise IndexError
-
- @abstractmethod
- def insert(self, index, value):
- 'S.insert(index, value) -- insert value before index'
- raise IndexError
-
- def append(self, value):
- 'S.append(value) -- append value to the end of the sequence'
- self.insert(len(self), value)
-
- def clear(self):
- 'S.clear() -> None -- remove all items from S'
- try:
- while True:
- self.pop()
- except IndexError:
- pass
-
- def reverse(self):
- 'S.reverse() -- reverse *IN PLACE*'
- n = len(self)
- for i in range(n//2):
- self[i], self[n-i-1] = self[n-i-1], self[i]
-
- def extend(self, values):
- 'S.extend(iterable) -- extend sequence by appending elements from the iterable'
- for v in values:
- self.append(v)
-
- def pop(self, index=-1):
- '''S.pop([index]) -> item -- remove and return item at index (default last).
- Raise IndexError if list is empty or index is out of range.
- '''
- v = self[index]
- del self[index]
- return v
-
- def remove(self, value):
- '''S.remove(value) -- remove first occurrence of value.
- Raise ValueError if the value is not present.
- '''
- del self[self.index(value)]
-
- def __iadd__(self, values):
- self.extend(values)
- return self
-
-MutableSequence.register(list)
-MutableSequence.register(bytearray) # Multiply inheriting, see ByteString
+from _collections_abc import *
+from _collections_abc import __all__
diff --git a/Lib/colorsys.py b/Lib/colorsys.py
index a6c0cf6..b93e384 100644
--- a/Lib/colorsys.py
+++ b/Lib/colorsys.py
@@ -33,17 +33,25 @@ TWO_THIRD = 2.0/3.0
# YIQ: used by composite video signals (linear combinations of RGB)
# Y: perceived grey level (0.0 == black, 1.0 == white)
# I, Q: color components
+#
+# There are a great many versions of the constants used in these formulae.
+# The ones in this library uses constants from the FCC version of NTSC.
def rgb_to_yiq(r, g, b):
y = 0.30*r + 0.59*g + 0.11*b
- i = 0.60*r - 0.28*g - 0.32*b
- q = 0.21*r - 0.52*g + 0.31*b
+ i = 0.74*(r-y) - 0.27*(b-y)
+ q = 0.48*(r-y) + 0.41*(b-y)
return (y, i, q)
def yiq_to_rgb(y, i, q):
- r = y + 0.948262*i + 0.624013*q
- g = y - 0.276066*i - 0.639810*q
- b = y - 1.105450*i + 1.729860*q
+ # r = y + (0.27*q + 0.41*i) / (0.74*0.41 + 0.27*0.48)
+ # b = y + (0.74*q - 0.48*i) / (0.74*0.41 + 0.27*0.48)
+ # g = y - (0.30*(r-y) + 0.11*(b-y)) / 0.59
+
+ r = y + 0.9468822170900693*i + 0.6235565819861433*q
+ g = y - 0.27478764629897834*i - 0.6356910791873801*q
+ b = y - 1.1085450346420322*i + 1.7090069284064666*q
+
if r < 0.0:
r = 0.0
if g < 0.0:
diff --git a/Lib/compileall.py b/Lib/compileall.py
index 693eda9..d957ee5 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -12,8 +12,7 @@ See module py_compile for details of the actual byte-compilation.
"""
import os
import sys
-import errno
-import imp
+import importlib.util
import py_compile
import struct
@@ -38,7 +37,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
print('Listing {!r}...'.format(dir))
try:
names = os.listdir(dir)
- except os.error:
+ except OSError:
print("Can't list {!r}".format(dir))
names = []
names.sort()
@@ -91,22 +90,23 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
cfile = fullname + ('c' if __debug__ else 'o')
else:
if optimize >= 0:
- cfile = imp.cache_from_source(fullname,
- debug_override=not optimize)
+ cfile = importlib.util.cache_from_source(
+ fullname, debug_override=not optimize)
else:
- cfile = imp.cache_from_source(fullname)
+ cfile = importlib.util.cache_from_source(fullname)
cache_dir = os.path.dirname(cfile)
head, tail = name[:-3], name[-3:]
if tail == '.py':
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
- expect = struct.pack('<4sl', imp.get_magic(), mtime)
+ expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
+ mtime)
with open(cfile, 'rb') as chandle:
actual = chandle.read(8)
if expect == actual:
return success
- except IOError:
+ except OSError:
pass
if not quiet:
print('Compiling {!r}...'.format(fullname))
@@ -124,7 +124,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=False,
msg = msg.decode(sys.stdout.encoding)
print(msg)
success = 0
- except (SyntaxError, UnicodeError, IOError) as e:
+ except (SyntaxError, UnicodeError, OSError) as e:
if quiet:
print('*** Error compiling {!r}...'.format(fullname))
else:
@@ -209,7 +209,7 @@ def main():
with (sys.stdin if args.flist=='-' else open(args.flist)) as f:
for line in f:
compile_dests.append(line.strip())
- except EnvironmentError:
+ except OSError:
print("Error reading file list {}".format(args.flist))
return False
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index d45a404..acd05d0 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -200,8 +200,7 @@ def as_completed(fs, timeout=None):
waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
try:
- for future in finished:
- yield future
+ yield from finished
while pending:
if timeout is None:
@@ -226,7 +225,8 @@ def as_completed(fs, timeout=None):
finally:
for f in fs:
- f._waiters.remove(waiter)
+ with f._condition:
+ f._waiters.remove(waiter)
DoneAndNotDoneFutures = collections.namedtuple(
'DoneAndNotDoneFutures', 'done not_done')
@@ -273,7 +273,8 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
waiter.event.wait(timeout)
for f in fs:
- f._waiters.remove(waiter)
+ with f._condition:
+ f._waiters.remove(waiter)
done.update(waiter.finished_futures)
return DoneAndNotDoneFutures(done, set(fs) - done)
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index adf2ab4..07b5225 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -40,7 +40,7 @@ Local worker thread:
Process #1..n:
- reads _CallItems from "Call Q", executes the calls, and puts the resulting
- _ResultItems in "Request Q"
+ _ResultItems in "Result Q"
"""
__author__ = 'Brian Quinlan (brian@sweetapp.com)'
@@ -49,8 +49,9 @@ import atexit
import os
from concurrent.futures import _base
import queue
+from queue import Full
import multiprocessing
-from multiprocessing.queues import SimpleQueue, Full
+from multiprocessing import SimpleQueue
from multiprocessing.connection import wait
import threading
import weakref
@@ -240,6 +241,8 @@ def _queue_management_worker(executor_reference,
"terminated abruptly while the future was "
"running or pending."
))
+ # Delete references to object. See issue16284
+ del work_item
pending_work_items.clear()
# Terminate remaining workers forcibly: the queues or their
# locks may be in a dirty state and block forever.
@@ -264,6 +267,8 @@ def _queue_management_worker(executor_reference,
work_item.future.set_exception(result_item.exception)
else:
work_item.future.set_result(result_item.result)
+ # Delete references to object. See issue16284
+ del work_item
# Check whether we should start shutting down.
executor = executor_reference()
# No more work items can be added if:
@@ -327,7 +332,7 @@ class ProcessPoolExecutor(_base.Executor):
_check_system_limits()
if max_workers is None:
- self._max_workers = multiprocessing.cpu_count()
+ self._max_workers = os.cpu_count() or 1
else:
self._max_workers = max_workers
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index 95bb682..f9beb0f 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -63,6 +63,8 @@ def _worker(executor_reference, work_queue):
work_item = work_queue.get(block=True)
if work_item is not None:
work_item.run()
+ # Delete references to object. See issue16284
+ del work_item
continue
executor = executor_reference()
# Exit if:
diff --git a/Lib/configparser.py b/Lib/configparser.py
index aebf8a0..4ee8307 100644
--- a/Lib/configparser.py
+++ b/Lib/configparser.py
@@ -670,7 +670,7 @@ class RawConfigParser(MutableMapping):
try:
with open(filename, encoding=encoding) as fp:
self._read(fp, filename)
- except IOError:
+ except OSError:
continue
read_ok.append(filename)
return read_ok
diff --git a/Lib/contextlib.py b/Lib/contextlib.py
index b03f828..82ee955 100644
--- a/Lib/contextlib.py
+++ b/Lib/contextlib.py
@@ -4,7 +4,8 @@ import sys
from collections import deque
from functools import wraps
-__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack"]
+__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack",
+ "redirect_stdout", "suppress"]
class ContextDecorator(object):
@@ -36,6 +37,16 @@ class _GeneratorContextManager(ContextDecorator):
def __init__(self, func, *args, **kwds):
self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = func, args, kwds
+ # Issue 19330: ensure context manager instances have good docstrings
+ doc = getattr(func, "__doc__", None)
+ if doc is None:
+ doc = type(self).__doc__
+ self.__doc__ = doc
+ # Unfortunately, this still doesn't provide good help output when
+ # inspecting the created context manager instances, since pydoc
+ # currently bypasses the instance docstring and shows the docstring
+ # for the class instead.
+ # See http://bugs.python.org/issue19404 for more details.
def _recreate_cm(self):
# _GCM instances are one-shot context managers, so the
@@ -47,7 +58,7 @@ class _GeneratorContextManager(ContextDecorator):
try:
return next(self.gen)
except StopIteration:
- raise RuntimeError("generator didn't yield")
+ raise RuntimeError("generator didn't yield") from None
def __exit__(self, type, value, traceback):
if type is None:
@@ -140,6 +151,62 @@ class closing(object):
def __exit__(self, *exc_info):
self.thing.close()
+class redirect_stdout:
+ """Context manager for temporarily redirecting stdout to another file
+
+ # How to send help() to stderr
+ with redirect_stdout(sys.stderr):
+ help(dir)
+
+ # How to write help() to a file
+ with open('help.txt', 'w') as f:
+ with redirect_stdout(f):
+ help(pow)
+ """
+
+ def __init__(self, new_target):
+ self._new_target = new_target
+ # We use a list of old targets to make this CM re-entrant
+ self._old_targets = []
+
+ def __enter__(self):
+ self._old_targets.append(sys.stdout)
+ sys.stdout = self._new_target
+ return self._new_target
+
+ def __exit__(self, exctype, excinst, exctb):
+ sys.stdout = self._old_targets.pop()
+
+
+class suppress:
+ """Context manager to suppress specified exceptions
+
+ After the exception is suppressed, execution proceeds with the next
+ statement following the with statement.
+
+ with suppress(FileNotFoundError):
+ os.remove(somefile)
+ # Execution still resumes here if the file was already removed
+ """
+
+ def __init__(self, *exceptions):
+ self._exceptions = exceptions
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, exctype, excinst, exctb):
+ # Unlike isinstance and issubclass, CPython exception handling
+ # currently only looks at the concrete type hierarchy (ignoring
+ # the instance and subclass checking hooks). While Guido considers
+ # that a bug rather than a feature, it's a fairly hard one to fix
+ # due to various internal implementation details. suppress provides
+ # the simpler issubclass based semantics, rather than trying to
+ # exactly reproduce the limitations of the CPython interpreter.
+ #
+ # See http://bugs.python.org/issue12029 for more details
+ return exctype is not None and issubclass(exctype, self._exceptions)
+
# Inspired by discussions on http://bugs.python.org/issue13585
class ExitStack(object):
diff --git a/Lib/copyreg.py b/Lib/copyreg.py
index 66c0f8a..67f5bb0 100644
--- a/Lib/copyreg.py
+++ b/Lib/copyreg.py
@@ -87,6 +87,12 @@ def _reduce_ex(self, proto):
def __newobj__(cls, *args):
return cls.__new__(cls, *args)
+def __newobj_ex__(cls, args, kwargs):
+ """Used by pickle protocol 4, instead of __newobj__ to allow classes with
+ keyword-only arguments to be pickled correctly.
+ """
+ return cls.__new__(cls, *args, **kwargs)
+
def _slotnames(cls):
"""Return a list of slot names for a given class.
diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py
index c92e130..e34c646 100644
--- a/Lib/ctypes/__init__.py
+++ b/Lib/ctypes/__init__.py
@@ -34,17 +34,15 @@ from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \
FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \
FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR
-"""
-WINOLEAPI -> HRESULT
-WINOLEAPI_(type)
-
-STDMETHODCALLTYPE
-
-STDMETHOD(name)
-STDMETHOD_(type, name)
-
-STDAPICALLTYPE
-"""
+# WINOLEAPI -> HRESULT
+# WINOLEAPI_(type)
+#
+# STDMETHODCALLTYPE
+#
+# STDMETHOD(name)
+# STDMETHOD_(type, name)
+#
+# STDAPICALLTYPE
def create_string_buffer(init, size=None):
"""create_string_buffer(aBytes) -> character array
@@ -395,7 +393,7 @@ if _os.name in ("nt", "ce"):
_type_ = "l"
# _check_retval_ is called with the function's result when it
# is used as restype. It checks for the FAILED bit, and
- # raises a WindowsError if it is set.
+ # raises an OSError if it is set.
#
# The _check_retval_ method is implemented in C, so that the
# method definition itself is not included in the traceback
@@ -407,7 +405,7 @@ if _os.name in ("nt", "ce"):
class OleDLL(CDLL):
"""This class represents a dll exporting functions using the
Windows stdcall calling convention, and returning HRESULT.
- HRESULT error values are automatically raised as WindowsError
+ HRESULT error values are automatically raised as OSError
exceptions.
"""
_func_flags_ = _FUNCFLAG_STDCALL
@@ -456,7 +454,7 @@ if _os.name in ("nt", "ce"):
code = GetLastError()
if descr is None:
descr = FormatError(code).strip()
- return WindowsError(None, descr, None, code)
+ return OSError(None, descr, None, code)
if sizeof(c_uint) == sizeof(c_void_p):
c_size_t = c_uint
diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py
index cc5fe02..7c72210 100644
--- a/Lib/ctypes/test/__init__.py
+++ b/Lib/ctypes/test/__init__.py
@@ -37,7 +37,7 @@ def requires(resource, msg=None):
def find_package_modules(package, mask):
import fnmatch
- if (hasattr(package, "__loader__") and
+ if (package.__loader__ is not None and
hasattr(package.__loader__, '_files')):
path = package.__name__.replace(".", os.path.sep)
mask = os.path.join(path, mask)
diff --git a/Lib/ctypes/test/test_checkretval.py b/Lib/ctypes/test/test_checkretval.py
index 01ccc57..19bb813 100644
--- a/Lib/ctypes/test/test_checkretval.py
+++ b/Lib/ctypes/test/test_checkretval.py
@@ -31,7 +31,7 @@ class Test(unittest.TestCase):
pass
else:
def test_oledll(self):
- self.assertRaises(WindowsError,
+ self.assertRaises(OSError,
oledll.oleaut32.CreateTypeLib2,
0, None, None)
diff --git a/Lib/ctypes/test/test_internals.py b/Lib/ctypes/test/test_internals.py
index cbf2e05..271e3f5 100644
--- a/Lib/ctypes/test/test_internals.py
+++ b/Lib/ctypes/test/test_internals.py
@@ -5,17 +5,14 @@ from sys import getrefcount as grc
# XXX This test must be reviewed for correctness!!!
-"""
-ctypes' types are container types.
-
-They have an internal memory block, which only consists of some bytes,
-but it has to keep references to other objects as well. This is not
-really needed for trivial C types like int or char, but it is important
-for aggregate types like strings or pointers in particular.
-
-What about pointers?
-
-"""
+# ctypes' types are container types.
+#
+# They have an internal memory block, which only consists of some bytes,
+# but it has to keep references to other objects as well. This is not
+# really needed for trivial C types like int or char, but it is important
+# for aggregate types like strings or pointers in particular.
+#
+# What about pointers?
class ObjectsTestCase(unittest.TestCase):
def assertSame(self, a, b):
diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py
index eda846d..8a2b2c3 100644
--- a/Lib/ctypes/test/test_macholib.py
+++ b/Lib/ctypes/test/test_macholib.py
@@ -3,35 +3,33 @@ import sys
import unittest
# Bob Ippolito:
-"""
-Ok.. the code to find the filename for __getattr__ should look
-something like:
-
-import os
-from macholib.dyld import dyld_find
-
-def find_lib(name):
- possible = ['lib'+name+'.dylib', name+'.dylib',
- name+'.framework/'+name]
- for dylib in possible:
- try:
- return os.path.realpath(dyld_find(dylib))
- except ValueError:
- pass
- raise ValueError, "%s not found" % (name,)
-
-It'll have output like this:
-
- >>> find_lib('pthread')
-'/usr/lib/libSystem.B.dylib'
- >>> find_lib('z')
-'/usr/lib/libz.1.dylib'
- >>> find_lib('IOKit')
-'/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit'
-
--bob
-
-"""
+#
+# Ok.. the code to find the filename for __getattr__ should look
+# something like:
+#
+# import os
+# from macholib.dyld import dyld_find
+#
+# def find_lib(name):
+# possible = ['lib'+name+'.dylib', name+'.dylib',
+# name+'.framework/'+name]
+# for dylib in possible:
+# try:
+# return os.path.realpath(dyld_find(dylib))
+# except ValueError:
+# pass
+# raise ValueError, "%s not found" % (name,)
+#
+# It'll have output like this:
+#
+# >>> find_lib('pthread')
+# '/usr/lib/libSystem.B.dylib'
+# >>> find_lib('z')
+# '/usr/lib/libz.1.dylib'
+# >>> find_lib('IOKit')
+# '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit'
+#
+# -bob
from ctypes.macholib.dyld import dyld_find
@@ -52,8 +50,11 @@ class MachOTest(unittest.TestCase):
'/usr/lib/libSystem.B.dylib')
result = find_lib('z')
- self.assertTrue(result.startswith('/usr/lib/libz.1'))
- self.assertTrue(result.endswith('.dylib'))
+ # Issue #21093: dyld default search path includes $HOME/lib and
+ # /usr/local/lib before /usr/lib, which caused test failures if
+ # a local copy of libz exists in one of them. Now ignore the head
+ # of the path.
+ self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
self.assertEqual(find_lib('IOKit'),
'/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
diff --git a/Lib/ctypes/test/test_win32.py b/Lib/ctypes/test/test_win32.py
index da21336..91ad314 100644
--- a/Lib/ctypes/test/test_win32.py
+++ b/Lib/ctypes/test/test_win32.py
@@ -41,7 +41,7 @@ if sys.platform == "win32":
# Call functions with invalid arguments, and make sure
# that access violations are trapped and raise an
# exception.
- self.assertRaises(WindowsError, windll.kernel32.GetModuleHandleA, 32)
+ self.assertRaises(OSError, windll.kernel32.GetModuleHandleA, 32)
def test_noargs(self):
# This is a special case on win32 x64
diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py
index 054c511..595113b 100644
--- a/Lib/ctypes/util.py
+++ b/Lib/ctypes/util.py
@@ -85,7 +85,7 @@ if os.name == "posix" and sys.platform == "darwin":
elif os.name == "posix":
# Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump
- import re, tempfile, errno
+ import re, tempfile
def _findLib_gcc(name):
expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
@@ -102,9 +102,8 @@ elif os.name == "posix":
finally:
try:
os.unlink(ccout)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
+ except FileNotFoundError:
+ pass
if rv == 10:
raise OSError('gcc or cc command not found')
res = re.search(expr, trace)
@@ -133,8 +132,10 @@ elif os.name == "posix":
cmd = 'if ! type objdump >/dev/null 2>&1; then exit 10; fi;' \
"objdump -p -j .dynamic 2>/dev/null " + f
f = os.popen(cmd)
- dump = f.read()
- rv = f.close()
+ try:
+ dump = f.read()
+ finally:
+ rv = f.close()
if rv == 10:
raise OSError('objdump command not found')
res = re.search(r'\sSONAME\s+([^\s]+)', dump)
@@ -177,10 +178,11 @@ elif os.name == "posix":
else:
cmd = 'env LC_ALL=C /usr/bin/crle 2>/dev/null'
- for line in os.popen(cmd).readlines():
- line = line.strip()
- if line.startswith('Default Library Path (ELF):'):
- paths = line.split()[4]
+ with contextlib.closing(os.popen(cmd)) as f:
+ for line in f.readlines():
+ line = line.strip()
+ if line.startswith('Default Library Path (ELF):'):
+ paths = line.split()[4]
if not paths:
return None
diff --git a/Lib/datetime.py b/Lib/datetime.py
index d1f353b..1789714 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -23,9 +23,10 @@ _MAXORDINAL = 3652059 # date.max.toordinal()
# for all computations. See the book for algorithms for converting between
# proleptic Gregorian ordinals and many other calendar systems.
-_DAYS_IN_MONTH = [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+# -1 is a placeholder for indexing purposes.
+_DAYS_IN_MONTH = [-1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
-_DAYS_BEFORE_MONTH = [None]
+_DAYS_BEFORE_MONTH = [-1] # -1 is a placeholder for indexing purposes.
dbm = 0
for dim in _DAYS_IN_MONTH[1:]:
_DAYS_BEFORE_MONTH.append(dbm)
@@ -1917,203 +1918,203 @@ timezone.utc = timezone._create(timedelta(0))
timezone.min = timezone._create(timezone._minoffset)
timezone.max = timezone._create(timezone._maxoffset)
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
-"""
-Some time zone algebra. For a datetime x, let
- x.n = x stripped of its timezone -- its naive time.
- x.o = x.utcoffset(), and assuming that doesn't raise an exception or
- return None
- x.d = x.dst(), and assuming that doesn't raise an exception or
- return None
- x.s = x's standard offset, x.o - x.d
-
-Now some derived rules, where k is a duration (timedelta).
-
-1. x.o = x.s + x.d
- This follows from the definition of x.s.
-
-2. If x and y have the same tzinfo member, x.s = y.s.
- This is actually a requirement, an assumption we need to make about
- sane tzinfo classes.
-
-3. The naive UTC time corresponding to x is x.n - x.o.
- This is again a requirement for a sane tzinfo class.
-
-4. (x+k).s = x.s
- This follows from #2, and that datimetimetz+timedelta preserves tzinfo.
-
-5. (x+k).n = x.n + k
- Again follows from how arithmetic is defined.
-
-Now we can explain tz.fromutc(x). Let's assume it's an interesting case
-(meaning that the various tzinfo methods exist, and don't blow up or return
-None when called).
-
-The function wants to return a datetime y with timezone tz, equivalent to x.
-x is already in UTC.
-
-By #3, we want
-
- y.n - y.o = x.n [1]
-
-The algorithm starts by attaching tz to x.n, and calling that y. So
-x.n = y.n at the start. Then it wants to add a duration k to y, so that [1]
-becomes true; in effect, we want to solve [2] for k:
-
- (y+k).n - (y+k).o = x.n [2]
-
-By #1, this is the same as
-
- (y+k).n - ((y+k).s + (y+k).d) = x.n [3]
-
-By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start.
-Substituting that into [3],
-
- x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving
- k - (y+k).s - (y+k).d = 0; rearranging,
- k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so
- k = y.s - (y+k).d
-
-On the RHS, (y+k).d can't be computed directly, but y.s can be, and we
-approximate k by ignoring the (y+k).d term at first. Note that k can't be
-very large, since all offset-returning methods return a duration of magnitude
-less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must
-be 0, so ignoring it has no consequence then.
-
-In any case, the new value is
- z = y + y.s [4]
+# Some time zone algebra. For a datetime x, let
+# x.n = x stripped of its timezone -- its naive time.
+# x.o = x.utcoffset(), and assuming that doesn't raise an exception or
+# return None
+# x.d = x.dst(), and assuming that doesn't raise an exception or
+# return None
+# x.s = x's standard offset, x.o - x.d
+#
+# Now some derived rules, where k is a duration (timedelta).
+#
+# 1. x.o = x.s + x.d
+# This follows from the definition of x.s.
+#
+# 2. If x and y have the same tzinfo member, x.s = y.s.
+# This is actually a requirement, an assumption we need to make about
+# sane tzinfo classes.
+#
+# 3. The naive UTC time corresponding to x is x.n - x.o.
+# This is again a requirement for a sane tzinfo class.
+#
+# 4. (x+k).s = x.s
+# This follows from #2, and that datimetimetz+timedelta preserves tzinfo.
+#
+# 5. (x+k).n = x.n + k
+# Again follows from how arithmetic is defined.
+#
+# Now we can explain tz.fromutc(x). Let's assume it's an interesting case
+# (meaning that the various tzinfo methods exist, and don't blow up or return
+# None when called).
+#
+# The function wants to return a datetime y with timezone tz, equivalent to x.
+# x is already in UTC.
+#
+# By #3, we want
+#
+# y.n - y.o = x.n [1]
+#
+# The algorithm starts by attaching tz to x.n, and calling that y. So
+# x.n = y.n at the start. Then it wants to add a duration k to y, so that [1]
+# becomes true; in effect, we want to solve [2] for k:
+#
+# (y+k).n - (y+k).o = x.n [2]
+#
+# By #1, this is the same as
+#
+# (y+k).n - ((y+k).s + (y+k).d) = x.n [3]
+#
+# By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start.
+# Substituting that into [3],
+#
+# x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving
+# k - (y+k).s - (y+k).d = 0; rearranging,
+# k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so
+# k = y.s - (y+k).d
+#
+# On the RHS, (y+k).d can't be computed directly, but y.s can be, and we
+# approximate k by ignoring the (y+k).d term at first. Note that k can't be
+# very large, since all offset-returning methods return a duration of magnitude
+# less than 24 hours. For that reason, if y is firmly in std time, (y+k).d must
+# be 0, so ignoring it has no consequence then.
+#
+# In any case, the new value is
+#
+# z = y + y.s [4]
+#
+# It's helpful to step back at look at [4] from a higher level: it's simply
+# mapping from UTC to tz's standard time.
+#
+# At this point, if
+#
+# z.n - z.o = x.n [5]
+#
+# we have an equivalent time, and are almost done. The insecurity here is
+# at the start of daylight time. Picture US Eastern for concreteness. The wall
+# time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good
+# sense then. The docs ask that an Eastern tzinfo class consider such a time to
+# be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST
+# on the day DST starts. We want to return the 1:MM EST spelling because that's
+# the only spelling that makes sense on the local wall clock.
+#
+# In fact, if [5] holds at this point, we do have the standard-time spelling,
+# but that takes a bit of proof. We first prove a stronger result. What's the
+# difference between the LHS and RHS of [5]? Let
+#
+# diff = x.n - (z.n - z.o) [6]
+#
+# Now
+# z.n = by [4]
+# (y + y.s).n = by #5
+# y.n + y.s = since y.n = x.n
+# x.n + y.s = since z and y are have the same tzinfo member,
+# y.s = z.s by #2
+# x.n + z.s
+#
+# Plugging that back into [6] gives
+#
+# diff =
+# x.n - ((x.n + z.s) - z.o) = expanding
+# x.n - x.n - z.s + z.o = cancelling
+# - z.s + z.o = by #2
+# z.d
+#
+# So diff = z.d.
+#
+# If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time
+# spelling we wanted in the endcase described above. We're done. Contrarily,
+# if z.d = 0, then we have a UTC equivalent, and are also done.
+#
+# If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to
+# add to z (in effect, z is in tz's standard time, and we need to shift the
+# local clock into tz's daylight time).
+#
+# Let
+#
+# z' = z + z.d = z + diff [7]
+#
+# and we can again ask whether
+#
+# z'.n - z'.o = x.n [8]
+#
+# If so, we're done. If not, the tzinfo class is insane, according to the
+# assumptions we've made. This also requires a bit of proof. As before, let's
+# compute the difference between the LHS and RHS of [8] (and skipping some of
+# the justifications for the kinds of substitutions we've done several times
+# already):
+#
+# diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7]
+# x.n - (z.n + diff - z'.o) = replacing diff via [6]
+# x.n - (z.n + x.n - (z.n - z.o) - z'.o) =
+# x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n
+# - z.n + z.n - z.o + z'.o = cancel z.n
+# - z.o + z'.o = #1 twice
+# -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo
+# z'.d - z.d
+#
+# So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal,
+# we've found the UTC-equivalent so are done. In fact, we stop with [7] and
+# return z', not bothering to compute z'.d.
+#
+# How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by
+# a dst() offset, and starting *from* a time already in DST (we know z.d != 0),
+# would have to change the result dst() returns: we start in DST, and moving
+# a little further into it takes us out of DST.
+#
+# There isn't a sane case where this can happen. The closest it gets is at
+# the end of DST, where there's an hour in UTC with no spelling in a hybrid
+# tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During
+# that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM
+# UTC) because the docs insist on that, but 0:MM is taken as being in daylight
+# time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local
+# clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in
+# standard time. Since that's what the local clock *does*, we want to map both
+# UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous
+# in local time, but so it goes -- it's the way the local clock works.
+#
+# When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0,
+# so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going.
+# z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8]
+# (correctly) concludes that z' is not UTC-equivalent to x.
+#
+# Because we know z.d said z was in daylight time (else [5] would have held and
+# we would have stopped then), and we know z.d != z'.d (else [8] would have held
+# and we have stopped then), and there are only 2 possible values dst() can
+# return in Eastern, it follows that z'.d must be 0 (which it is in the example,
+# but the reasoning doesn't depend on the example -- it depends on there being
+# two possible dst() outcomes, one zero and the other non-zero). Therefore
+# z' must be in standard time, and is the spelling we want in this case.
+#
+# Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is
+# concerned (because it takes z' as being in standard time rather than the
+# daylight time we intend here), but returning it gives the real-life "local
+# clock repeats an hour" behavior when mapping the "unspellable" UTC hour into
+# tz.
+#
+# When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with
+# the 1:MM standard time spelling we want.
+#
+# So how can this break? One of the assumptions must be violated. Two
+# possibilities:
+#
+# 1) [2] effectively says that y.s is invariant across all y belong to a given
+# time zone. This isn't true if, for political reasons or continental drift,
+# a region decides to change its base offset from UTC.
+#
+# 2) There may be versions of "double daylight" time where the tail end of
+# the analysis gives up a step too early. I haven't thought about that
+# enough to say.
+#
+# In any case, it's clear that the default fromutc() is strong enough to handle
+# "almost all" time zones: so long as the standard offset is invariant, it
+# doesn't matter if daylight time transition points change from year to year, or
+# if daylight time is skipped in some years; it doesn't matter how large or
+# small dst() may get within its bounds; and it doesn't even matter if some
+# perverse time zone returns a negative dst()). So a breaking case must be
+# pretty bizarre, and a tzinfo subclass can override fromutc() if it is.
-It's helpful to step back at look at [4] from a higher level: it's simply
-mapping from UTC to tz's standard time.
-
-At this point, if
-
- z.n - z.o = x.n [5]
-
-we have an equivalent time, and are almost done. The insecurity here is
-at the start of daylight time. Picture US Eastern for concreteness. The wall
-time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good
-sense then. The docs ask that an Eastern tzinfo class consider such a time to
-be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST
-on the day DST starts. We want to return the 1:MM EST spelling because that's
-the only spelling that makes sense on the local wall clock.
-
-In fact, if [5] holds at this point, we do have the standard-time spelling,
-but that takes a bit of proof. We first prove a stronger result. What's the
-difference between the LHS and RHS of [5]? Let
-
- diff = x.n - (z.n - z.o) [6]
-
-Now
- z.n = by [4]
- (y + y.s).n = by #5
- y.n + y.s = since y.n = x.n
- x.n + y.s = since z and y are have the same tzinfo member,
- y.s = z.s by #2
- x.n + z.s
-
-Plugging that back into [6] gives
-
- diff =
- x.n - ((x.n + z.s) - z.o) = expanding
- x.n - x.n - z.s + z.o = cancelling
- - z.s + z.o = by #2
- z.d
-
-So diff = z.d.
-
-If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time
-spelling we wanted in the endcase described above. We're done. Contrarily,
-if z.d = 0, then we have a UTC equivalent, and are also done.
-
-If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to
-add to z (in effect, z is in tz's standard time, and we need to shift the
-local clock into tz's daylight time).
-
-Let
-
- z' = z + z.d = z + diff [7]
-
-and we can again ask whether
-
- z'.n - z'.o = x.n [8]
-
-If so, we're done. If not, the tzinfo class is insane, according to the
-assumptions we've made. This also requires a bit of proof. As before, let's
-compute the difference between the LHS and RHS of [8] (and skipping some of
-the justifications for the kinds of substitutions we've done several times
-already):
-
- diff' = x.n - (z'.n - z'.o) = replacing z'.n via [7]
- x.n - (z.n + diff - z'.o) = replacing diff via [6]
- x.n - (z.n + x.n - (z.n - z.o) - z'.o) =
- x.n - z.n - x.n + z.n - z.o + z'.o = cancel x.n
- - z.n + z.n - z.o + z'.o = cancel z.n
- - z.o + z'.o = #1 twice
- -z.s - z.d + z'.s + z'.d = z and z' have same tzinfo
- z'.d - z.d
-
-So z' is UTC-equivalent to x iff z'.d = z.d at this point. If they are equal,
-we've found the UTC-equivalent so are done. In fact, we stop with [7] and
-return z', not bothering to compute z'.d.
-
-How could z.d and z'd differ? z' = z + z.d [7], so merely moving z' by
-a dst() offset, and starting *from* a time already in DST (we know z.d != 0),
-would have to change the result dst() returns: we start in DST, and moving
-a little further into it takes us out of DST.
-
-There isn't a sane case where this can happen. The closest it gets is at
-the end of DST, where there's an hour in UTC with no spelling in a hybrid
-tzinfo class. In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT. During
-that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM
-UTC) because the docs insist on that, but 0:MM is taken as being in daylight
-time (4:MM UTC). There is no local time mapping to 5:MM UTC. The local
-clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in
-standard time. Since that's what the local clock *does*, we want to map both
-UTC hours 5:MM and 6:MM to 1:MM Eastern. The result is ambiguous
-in local time, but so it goes -- it's the way the local clock works.
-
-When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0,
-so z=0:MM. z.d=60 (minutes) then, so [5] doesn't hold and we keep going.
-z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8]
-(correctly) concludes that z' is not UTC-equivalent to x.
-
-Because we know z.d said z was in daylight time (else [5] would have held and
-we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we have stopped then), and there are only 2 possible values dst() can
-return in Eastern, it follows that z'.d must be 0 (which it is in the example,
-but the reasoning doesn't depend on the example -- it depends on there being
-two possible dst() outcomes, one zero and the other non-zero). Therefore
-z' must be in standard time, and is the spelling we want in this case.
-
-Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is
-concerned (because it takes z' as being in standard time rather than the
-daylight time we intend here), but returning it gives the real-life "local
-clock repeats an hour" behavior when mapping the "unspellable" UTC hour into
-tz.
-
-When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with
-the 1:MM standard time spelling we want.
-
-So how can this break? One of the assumptions must be violated. Two
-possibilities:
-
-1) [2] effectively says that y.s is invariant across all y belong to a given
- time zone. This isn't true if, for political reasons or continental drift,
- a region decides to change its base offset from UTC.
-
-2) There may be versions of "double daylight" time where the tail end of
- the analysis gives up a step too early. I haven't thought about that
- enough to say.
-
-In any case, it's clear that the default fromutc() is strong enough to handle
-"almost all" time zones: so long as the standard offset is invariant, it
-doesn't matter if daylight time transition points change from year to year, or
-if daylight time is skipped in some years; it doesn't matter how large or
-small dst() may get within its bounds; and it doesn't even matter if some
-perverse time zone returns a negative dst()). So a breaking case must be
-pretty bizarre, and a tzinfo subclass can override fromutc() if it is.
-"""
try:
from _datetime import *
except ImportError:
diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
index a783fde..5f4664a 100644
--- a/Lib/dbm/__init__.py
+++ b/Lib/dbm/__init__.py
@@ -42,7 +42,7 @@ _names = ['dbm.gnu', 'dbm.ndbm', 'dbm.dumb']
_defaultmod = None
_modules = {}
-error = (error, IOError)
+error = (error, OSError)
try:
from dbm import ndbm
@@ -111,12 +111,10 @@ def whichdb(filename):
try:
f = io.open(filename + ".pag", "rb")
f.close()
- # dbm linked with gdbm on OS/2 doesn't have .dir file
- if not (ndbm.library == "GNU gdbm" and sys.platform == "os2emx"):
- f = io.open(filename + ".dir", "rb")
- f.close()
+ f = io.open(filename + ".dir", "rb")
+ f.close()
return "dbm.ndbm"
- except IOError:
+ except OSError:
# some dbm emulations based on Berkeley DB generate a .db file
# some do not, but they should be caught by the bsd checks
try:
@@ -129,7 +127,7 @@ def whichdb(filename):
d = ndbm.open(filename)
d.close()
return "dbm.ndbm"
- except IOError:
+ except OSError:
pass
# Check for dumbdbm next -- this has a .dir and a .dat file
@@ -146,13 +144,13 @@ def whichdb(filename):
return "dbm.dumb"
finally:
f.close()
- except (OSError, IOError):
+ except OSError:
pass
# See if the file exists, return None if not
try:
f = io.open(filename, "rb")
- except IOError:
+ except OSError:
return None
# Read the start of the file -- the magic number
diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py
index cfb9123..ba6a20d 100644
--- a/Lib/dbm/dumb.py
+++ b/Lib/dbm/dumb.py
@@ -29,7 +29,7 @@ __all__ = ["error", "open"]
_BLOCKSIZE = 512
-error = IOError
+error = OSError
class _Database(collections.MutableMapping):
@@ -67,7 +67,7 @@ class _Database(collections.MutableMapping):
# Mod by Jack: create data file if needed
try:
f = _io.open(self._datfile, 'r', encoding="Latin-1")
- except IOError:
+ except OSError:
f = _io.open(self._datfile, 'w', encoding="Latin-1")
self._chmod(self._datfile)
f.close()
@@ -78,7 +78,7 @@ class _Database(collections.MutableMapping):
self._index = {}
try:
f = _io.open(self._dirfile, 'r', encoding="Latin-1")
- except IOError:
+ except OSError:
pass
else:
for line in f:
@@ -100,12 +100,12 @@ class _Database(collections.MutableMapping):
try:
self._os.unlink(self._bakfile)
- except self._os.error:
+ except OSError:
pass
try:
self._os.rename(self._dirfile, self._bakfile)
- except self._os.error:
+ except OSError:
pass
f = self._io.open(self._dirfile, 'w', encoding="Latin-1")
@@ -236,6 +236,12 @@ class _Database(collections.MutableMapping):
if hasattr(self._os, 'chmod'):
self._os.chmod(file, self._mode)
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
def open(file, flag=None, mode=0o666):
"""Open the database file, filename, and return corresponding object.
diff --git a/Lib/decimal.py b/Lib/decimal.py
index 9f37e4f..5b98473 100644
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -142,7 +142,6 @@ __version__ = '1.70' # Highest version of the spec this complies with
# See http://speleotrove.com/decimal/
__libmpdec_version__ = "2.4.0" # compatible libmpdec version
-import copy as _copy
import math as _math
import numbers as _numbers
import sys
@@ -704,8 +703,7 @@ class Decimal(object):
raise TypeError("Cannot convert %r to Decimal" % value)
- # @classmethod, but @decorator is not valid Python 2.3 syntax, so
- # don't use it (see notes on Py2.3 compatibility at top of file)
+ @classmethod
def from_float(cls, f):
"""Converts a float to a decimal number, exactly.
@@ -744,7 +742,6 @@ class Decimal(object):
return result
else:
return cls(result)
- from_float = classmethod(from_float)
def _isnan(self):
"""Returns whether the number is not actually one.
diff --git a/Lib/difflib.py b/Lib/difflib.py
index f0bfcc5..38dfef4 100644
--- a/Lib/difflib.py
+++ b/Lib/difflib.py
@@ -30,7 +30,6 @@ __all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher',
'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff',
'unified_diff', 'HtmlDiff', 'Match']
-import warnings
import heapq
from collections import namedtuple as _namedtuple
@@ -334,20 +333,6 @@ class SequenceMatcher:
for elt in popular: # ditto; as fast for 1% deletion
del b2j[elt]
- def isbjunk(self, item):
- "Deprecated; use 'item in SequenceMatcher().bjunk'."
- warnings.warn("'SequenceMatcher().isbjunk(item)' is deprecated;\n"
- "use 'item in SMinstance.bjunk' instead.",
- DeprecationWarning, 2)
- return item in self.bjunk
-
- def isbpopular(self, item):
- "Deprecated; use 'item in SequenceMatcher().bpopular'."
- warnings.warn("'SequenceMatcher().isbpopular(item)' is deprecated;\n"
- "use 'item in SMinstance.bpopular' instead.",
- DeprecationWarning, 2)
- return item in self.bpopular
-
def find_longest_match(self, alo, ahi, blo, bhi):
"""Find longest matching block in a[alo:ahi] and b[blo:bhi].
@@ -920,8 +905,7 @@ class Differ:
else:
raise ValueError('unknown tag %r' % (tag,))
- for line in g:
- yield line
+ yield from g
def _dump(self, tag, x, lo, hi):
"""Generate comparison results for a same-tagged range."""
@@ -940,8 +924,7 @@ class Differ:
second = self._dump('+', b, blo, bhi)
for g in first, second:
- for line in g:
- yield line
+ yield from g
def _fancy_replace(self, a, alo, ahi, b, blo, bhi):
r"""
@@ -995,8 +978,7 @@ class Differ:
# no non-identical "pretty close" pair
if eqi is None:
# no identical pair either -- treat it as a straight replace
- for line in self._plain_replace(a, alo, ahi, b, blo, bhi):
- yield line
+ yield from self._plain_replace(a, alo, ahi, b, blo, bhi)
return
# no close pair, but an identical pair -- synch up on that
best_i, best_j, best_ratio = eqi, eqj, 1.0
@@ -1008,8 +990,7 @@ class Differ:
# identical
# pump out diffs from before the synch point
- for line in self._fancy_helper(a, alo, best_i, b, blo, best_j):
- yield line
+ yield from self._fancy_helper(a, alo, best_i, b, blo, best_j)
# do intraline marking on the synch pair
aelt, belt = a[best_i], b[best_j]
@@ -1031,15 +1012,13 @@ class Differ:
btags += ' ' * lb
else:
raise ValueError('unknown tag %r' % (tag,))
- for line in self._qformat(aelt, belt, atags, btags):
- yield line
+ yield from self._qformat(aelt, belt, atags, btags)
else:
# the synch pair is identical
yield ' ' + aelt
# pump out diffs from after the synch point
- for line in self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi):
- yield line
+ yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi)
def _fancy_helper(self, a, alo, ahi, b, blo, bhi):
g = []
@@ -1051,8 +1030,7 @@ class Differ:
elif blo < bhi:
g = self._dump('+', b, blo, bhi)
- for line in g:
- yield line
+ yield from g
def _qformat(self, aline, bline, atags, btags):
r"""
diff --git a/Lib/dis.py b/Lib/dis.py
index 543fdc7..81cbe7f 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -2,12 +2,15 @@
import sys
import types
+import collections
+import io
from opcode import *
from opcode import __all__ as _opcodes_all
__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
- "findlinestarts", "findlabels", "show_code"] + _opcodes_all
+ "findlinestarts", "findlabels", "show_code",
+ "get_instructions", "Instruction", "Bytecode"] + _opcodes_all
del _opcodes_all
_have_code = (types.MethodType, types.FunctionType, types.CodeType, type)
@@ -25,14 +28,14 @@ def _try_compile(source, name):
c = compile(source, name, 'exec')
return c
-def dis(x=None):
+def dis(x=None, *, file=None):
"""Disassemble classes, methods, functions, or code.
With no argument, disassemble the last traceback.
"""
if x is None:
- distb()
+ distb(file=file)
return
if hasattr(x, '__func__'): # Method
x = x.__func__
@@ -42,23 +45,23 @@ def dis(x=None):
items = sorted(x.__dict__.items())
for name, x1 in items:
if isinstance(x1, _have_code):
- print("Disassembly of %s:" % name)
+ print("Disassembly of %s:" % name, file=file)
try:
- dis(x1)
+ dis(x1, file=file)
except TypeError as msg:
- print("Sorry:", msg)
- print()
+ print("Sorry:", msg, file=file)
+ print(file=file)
elif hasattr(x, 'co_code'): # Code object
- disassemble(x)
+ disassemble(x, file=file)
elif isinstance(x, (bytes, bytearray)): # Raw bytecode
- _disassemble_bytes(x)
+ _disassemble_bytes(x, file=file)
elif isinstance(x, str): # Source code
- _disassemble_str(x)
+ _disassemble_str(x, file=file)
else:
raise TypeError("don't know how to disassemble %s objects" %
type(x).__name__)
-def distb(tb=None):
+def distb(tb=None, *, file=None):
"""Disassemble a traceback (default: last traceback)."""
if tb is None:
try:
@@ -66,7 +69,7 @@ def distb(tb=None):
except AttributeError:
raise RuntimeError("no last traceback to disassemble")
while tb.tb_next: tb = tb.tb_next
- disassemble(tb.tb_frame.f_code, tb.tb_lasti)
+ disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file)
# The inspect module interrogates this dictionary to build its
# list of CO_* constants. It is also used by pretty_flags to
@@ -95,19 +98,22 @@ def pretty_flags(flags):
names.append(hex(flags))
return ", ".join(names)
-def code_info(x):
- """Formatted details of methods, functions, or code."""
+def _get_code_object(x):
+ """Helper to handle methods, functions, strings and raw code objects"""
if hasattr(x, '__func__'): # Method
x = x.__func__
if hasattr(x, '__code__'): # Function
x = x.__code__
if isinstance(x, str): # Source code
- x = _try_compile(x, "<code_info>")
+ x = _try_compile(x, "<disassembly>")
if hasattr(x, 'co_code'): # Code object
- return _format_code_info(x)
- else:
- raise TypeError("don't know how to disassemble %s objects" %
- type(x).__name__)
+ return x
+ raise TypeError("don't know how to disassemble %s objects" %
+ type(x).__name__)
+
+def code_info(x):
+ """Formatted details of methods, functions, or code."""
+ return _format_code_info(_get_code_object(x))
def _format_code_info(co):
lines = []
@@ -140,106 +146,206 @@ def _format_code_info(co):
lines.append("%4d: %s" % i_n)
return "\n".join(lines)
-def show_code(co):
- """Print details of methods, functions, or code to stdout."""
- print(code_info(co))
+def show_code(co, *, file=None):
+ """Print details of methods, functions, or code to *file*.
-def disassemble(co, lasti=-1):
- """Disassemble a code object."""
- code = co.co_code
- labels = findlabels(code)
+ If *file* is not provided, the output is printed on stdout.
+ """
+ print(code_info(co), file=file)
+
+_Instruction = collections.namedtuple("_Instruction",
+ "opname opcode arg argval argrepr offset starts_line is_jump_target")
+
+class Instruction(_Instruction):
+ """Details for a bytecode operation
+
+ Defined fields:
+ opname - human readable name for operation
+ opcode - numeric code for operation
+ arg - numeric argument to operation (if any), otherwise None
+ argval - resolved arg value (if known), otherwise same as arg
+ argrepr - human readable description of operation argument
+ offset - start index of operation within bytecode sequence
+ starts_line - line started by this opcode (if any), otherwise None
+ is_jump_target - True if other code jumps to here, otherwise False
+ """
+
+ def _disassemble(self, lineno_width=3, mark_as_current=False):
+ """Format instruction details for inclusion in disassembly output
+
+ *lineno_width* sets the width of the line number field (0 omits it)
+ *mark_as_current* inserts a '-->' marker arrow as part of the line
+ """
+ fields = []
+ # Column: Source code line number
+ if lineno_width:
+ if self.starts_line is not None:
+ lineno_fmt = "%%%dd" % lineno_width
+ fields.append(lineno_fmt % self.starts_line)
+ else:
+ fields.append(' ' * lineno_width)
+ # Column: Current instruction indicator
+ if mark_as_current:
+ fields.append('-->')
+ else:
+ fields.append(' ')
+ # Column: Jump target marker
+ if self.is_jump_target:
+ fields.append('>>')
+ else:
+ fields.append(' ')
+ # Column: Instruction offset from start of code sequence
+ fields.append(repr(self.offset).rjust(4))
+ # Column: Opcode name
+ fields.append(self.opname.ljust(20))
+ # Column: Opcode argument
+ if self.arg is not None:
+ fields.append(repr(self.arg).rjust(5))
+ # Column: Opcode argument details
+ if self.argrepr:
+ fields.append('(' + self.argrepr + ')')
+ return ' '.join(fields).rstrip()
+
+
+def get_instructions(x, *, first_line=None):
+ """Iterator for the opcodes in methods, functions or code
+
+ Generates a series of Instruction named tuples giving the details of
+ each operations in the supplied code.
+
+ If *first_line* is not None, it indicates the line number that should
+ be reported for the first source line in the disassembled code.
+ Otherwise, the source line information (if any) is taken directly from
+ the disassembled code object.
+ """
+ co = _get_code_object(x)
+ cell_names = co.co_cellvars + co.co_freevars
linestarts = dict(findlinestarts(co))
- n = len(code)
- i = 0
+ if first_line is not None:
+ line_offset = first_line - co.co_firstlineno
+ else:
+ line_offset = 0
+ return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+ co.co_consts, cell_names, linestarts,
+ line_offset)
+
+def _get_const_info(const_index, const_list):
+ """Helper to get optional details about const references
+
+ Returns the dereferenced constant and its repr if the constant
+ list is defined.
+ Otherwise returns the constant index and its repr().
+ """
+ argval = const_index
+ if const_list is not None:
+ argval = const_list[const_index]
+ return argval, repr(argval)
+
+def _get_name_info(name_index, name_list):
+ """Helper to get optional details about named references
+
+ Returns the dereferenced name as both value and repr if the name
+ list is defined.
+ Otherwise returns the name index and its repr().
+ """
+ argval = name_index
+ if name_list is not None:
+ argval = name_list[name_index]
+ argrepr = argval
+ else:
+ argrepr = repr(argval)
+ return argval, argrepr
+
+
+def _get_instructions_bytes(code, varnames=None, names=None, constants=None,
+ cells=None, linestarts=None, line_offset=0):
+ """Iterate over the instructions in a bytecode string.
+
+ Generates a sequence of Instruction namedtuples giving the details of each
+ opcode. Additional information about the code's runtime environment
+ (e.g. variable names, constants) can be specified using optional
+ arguments.
+
+ """
+ labels = findlabels(code)
extended_arg = 0
+ starts_line = None
free = None
+ # enumerate() is not an option, since we sometimes process
+ # multiple elements on a single pass through the loop
+ n = len(code)
+ i = 0
while i < n:
op = code[i]
- if i in linestarts:
- if i > 0:
- print()
- print("%3d" % linestarts[i], end=' ')
- else:
- print(' ', end=' ')
-
- if i == lasti: print('-->', end=' ')
- else: print(' ', end=' ')
- if i in labels: print('>>', end=' ')
- else: print(' ', end=' ')
- print(repr(i).rjust(4), end=' ')
- print(opname[op].ljust(20), end=' ')
+ offset = i
+ if linestarts is not None:
+ starts_line = linestarts.get(i, None)
+ if starts_line is not None:
+ starts_line += line_offset
+ is_jump_target = i in labels
i = i+1
+ arg = None
+ argval = None
+ argrepr = ''
if op >= HAVE_ARGUMENT:
- oparg = code[i] + code[i+1]*256 + extended_arg
+ arg = code[i] + code[i+1]*256 + extended_arg
extended_arg = 0
i = i+2
if op == EXTENDED_ARG:
- extended_arg = oparg*65536
- print(repr(oparg).rjust(5), end=' ')
+ extended_arg = arg*65536
+ # Set argval to the dereferenced value of the argument when
+ # availabe, and argrepr to the string representation of argval.
+ # _disassemble_bytes needs the string repr of the
+ # raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
+ argval = arg
if op in hasconst:
- print('(' + repr(co.co_consts[oparg]) + ')', end=' ')
+ argval, argrepr = _get_const_info(arg, constants)
elif op in hasname:
- print('(' + co.co_names[oparg] + ')', end=' ')
+ argval, argrepr = _get_name_info(arg, names)
elif op in hasjrel:
- print('(to ' + repr(i + oparg) + ')', end=' ')
+ argval = i + arg
+ argrepr = "to " + repr(argval)
elif op in haslocal:
- print('(' + co.co_varnames[oparg] + ')', end=' ')
+ argval, argrepr = _get_name_info(arg, varnames)
elif op in hascompare:
- print('(' + cmp_op[oparg] + ')', end=' ')
+ argval = cmp_op[arg]
+ argrepr = argval
elif op in hasfree:
- if free is None:
- free = co.co_cellvars + co.co_freevars
- print('(' + free[oparg] + ')', end=' ')
+ argval, argrepr = _get_name_info(arg, cells)
elif op in hasnargs:
- print('(%d positional, %d keyword pair)'
- % (code[i-2], code[i-1]), end=' ')
- print()
+ argrepr = "%d positional, %d keyword pair" % (code[i-2], code[i-1])
+ yield Instruction(opname[op], op,
+ arg, argval, argrepr,
+ offset, starts_line, is_jump_target)
+
+def disassemble(co, lasti=-1, *, file=None):
+ """Disassemble a code object."""
+ cell_names = co.co_cellvars + co.co_freevars
+ linestarts = dict(findlinestarts(co))
+ _disassemble_bytes(co.co_code, lasti, co.co_varnames, co.co_names,
+ co.co_consts, cell_names, linestarts, file=file)
def _disassemble_bytes(code, lasti=-1, varnames=None, names=None,
- constants=None):
- labels = findlabels(code)
- n = len(code)
- i = 0
- while i < n:
- op = code[i]
- if i == lasti: print('-->', end=' ')
- else: print(' ', end=' ')
- if i in labels: print('>>', end=' ')
- else: print(' ', end=' ')
- print(repr(i).rjust(4), end=' ')
- print(opname[op].ljust(15), end=' ')
- i = i+1
- if op >= HAVE_ARGUMENT:
- oparg = code[i] + code[i+1]*256
- i = i+2
- print(repr(oparg).rjust(5), end=' ')
- if op in hasconst:
- if constants:
- print('(' + repr(constants[oparg]) + ')', end=' ')
- else:
- print('(%d)'%oparg, end=' ')
- elif op in hasname:
- if names is not None:
- print('(' + names[oparg] + ')', end=' ')
- else:
- print('(%d)'%oparg, end=' ')
- elif op in hasjrel:
- print('(to ' + repr(i + oparg) + ')', end=' ')
- elif op in haslocal:
- if varnames:
- print('(' + varnames[oparg] + ')', end=' ')
- else:
- print('(%d)' % oparg, end=' ')
- elif op in hascompare:
- print('(' + cmp_op[oparg] + ')', end=' ')
- elif op in hasnargs:
- print('(%d positional, %d keyword pair)'
- % (code[i-2], code[i-1]), end=' ')
- print()
+ constants=None, cells=None, linestarts=None,
+ *, file=None, line_offset=0):
+ # Omit the line number column entirely if we have no line number info
+ show_lineno = linestarts is not None
+ # TODO?: Adjust width upwards if max(linestarts.values()) >= 1000?
+ lineno_width = 3 if show_lineno else 0
+ for instr in _get_instructions_bytes(code, varnames, names,
+ constants, cells, linestarts,
+ line_offset=line_offset):
+ new_source_line = (show_lineno and
+ instr.starts_line is not None and
+ instr.offset > 0)
+ if new_source_line:
+ print(file=file)
+ is_current_instr = instr.offset == lasti
+ print(instr._disassemble(lineno_width, is_current_instr), file=file)
-def _disassemble_str(source):
+def _disassemble_str(source, *, file=None):
"""Compile the source string, then disassemble the code object."""
- disassemble(_try_compile(source, '<dis>'))
+ disassemble(_try_compile(source, '<dis>'), file=file)
disco = disassemble # XXX For backwards compatibility
@@ -250,19 +356,21 @@ def findlabels(code):
"""
labels = []
+ # enumerate() is not an option, since we sometimes process
+ # multiple elements on a single pass through the loop
n = len(code)
i = 0
while i < n:
op = code[i]
i = i+1
if op >= HAVE_ARGUMENT:
- oparg = code[i] + code[i+1]*256
+ arg = code[i] + code[i+1]*256
i = i+2
label = -1
if op in hasjrel:
- label = i+oparg
+ label = i+arg
elif op in hasjabs:
- label = oparg
+ label = arg
if label >= 0:
if label not in labels:
labels.append(label)
@@ -290,27 +398,77 @@ def findlinestarts(code):
if lineno != lastlineno:
yield (addr, lineno)
+class Bytecode:
+ """The bytecode operations of a piece of code
+
+ Instantiate this with a function, method, string of code, or a code object
+ (as returned by compile()).
+
+ Iterating over this yields the bytecode operations as Instruction instances.
+ """
+ def __init__(self, x, *, first_line=None, current_offset=None):
+ self.codeobj = co = _get_code_object(x)
+ if first_line is None:
+ self.first_line = co.co_firstlineno
+ self._line_offset = 0
+ else:
+ self.first_line = first_line
+ self._line_offset = first_line - co.co_firstlineno
+ self._cell_names = co.co_cellvars + co.co_freevars
+ self._linestarts = dict(findlinestarts(co))
+ self._original_object = x
+ self.current_offset = current_offset
+
+ def __iter__(self):
+ co = self.codeobj
+ return _get_instructions_bytes(co.co_code, co.co_varnames, co.co_names,
+ co.co_consts, self._cell_names,
+ self._linestarts,
+ line_offset=self._line_offset)
+
+ def __repr__(self):
+ return "{}({!r})".format(self.__class__.__name__,
+ self._original_object)
+
+ @classmethod
+ def from_traceback(cls, tb):
+ """ Construct a Bytecode from the given traceback """
+ while tb.tb_next:
+ tb = tb.tb_next
+ return cls(tb.tb_frame.f_code, current_offset=tb.tb_lasti)
+
+ def info(self):
+ """Return formatted information about the code object."""
+ return _format_code_info(self.codeobj)
+
+ def dis(self):
+ """Return a formatted view of the bytecode operations."""
+ co = self.codeobj
+ if self.current_offset is not None:
+ offset = self.current_offset
+ else:
+ offset = -1
+ with io.StringIO() as output:
+ _disassemble_bytes(co.co_code, varnames=co.co_varnames,
+ names=co.co_names, constants=co.co_consts,
+ cells=self._cell_names,
+ linestarts=self._linestarts,
+ line_offset=self._line_offset,
+ file=output,
+ lasti=offset)
+ return output.getvalue()
+
+
def _test():
"""Simple test program to disassemble a file."""
- if sys.argv[1:]:
- if sys.argv[2:]:
- sys.stderr.write("usage: python dis.py [-|file]\n")
- sys.exit(2)
- fn = sys.argv[1]
- if not fn or fn == "-":
- fn = None
- else:
- fn = None
- if fn is None:
- f = sys.stdin
- else:
- f = open(fn)
- source = f.read()
- if fn is not None:
- f.close()
- else:
- fn = "<stdin>"
- code = compile(source, fn, "exec")
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
+ args = parser.parse_args()
+ with args.infile as infile:
+ source = infile.read()
+ code = compile(source, args.infile.name, "exec")
dis(code)
if __name__ == "__main__":
diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py
index 2e8719f..04804c1 100644
--- a/Lib/distutils/__init__.py
+++ b/Lib/distutils/__init__.py
@@ -13,5 +13,5 @@ used from a setup script as
# Updated automatically by the Python release process.
#
#--start constants--
-__version__ = "3.3.5"
+__version__ = "3.4.0"
#--end constants--
diff --git a/Lib/distutils/archive_util.py b/Lib/distutils/archive_util.py
index fcda08e..4470bb0 100644
--- a/Lib/distutils/archive_util.py
+++ b/Lib/distutils/archive_util.py
@@ -18,15 +18,55 @@ from distutils.spawn import spawn
from distutils.dir_util import mkpath
from distutils import log
-def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0):
+try:
+ from pwd import getpwnam
+except ImportError:
+ getpwnam = None
+
+try:
+ from grp import getgrnam
+except ImportError:
+ getgrnam = None
+
+def _get_gid(name):
+ """Returns a gid, given a group name."""
+ if getgrnam is None or name is None:
+ return None
+ try:
+ result = getgrnam(name)
+ except KeyError:
+ result = None
+ if result is not None:
+ return result[2]
+ return None
+
+def _get_uid(name):
+ """Returns an uid, given a user name."""
+ if getpwnam is None or name is None:
+ return None
+ try:
+ result = getpwnam(name)
+ except KeyError:
+ result = None
+ if result is not None:
+ return result[2]
+ return None
+
+def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
+ owner=None, group=None):
"""Create a (possibly compressed) tar file from all the files under
'base_dir'.
'compress' must be "gzip" (the default), "compress", "bzip2", or None.
- Both "tar" and the compression utility named by 'compress' must be on
- the default program search path, so this is probably Unix-specific.
+ (compress will be deprecated in Python 3.2)
+
+ 'owner' and 'group' can be used to define an owner and a group for the
+ archive that is being built. If not provided, the current owner and group
+ will be used.
+
The output tar file will be named 'base_dir' + ".tar", possibly plus
the appropriate compression extension (".gz", ".bz2" or ".Z").
+
Returns the output filename.
"""
tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', None: '', 'compress': ''}
@@ -48,10 +88,23 @@ def make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0):
import tarfile # late import so Python build itself doesn't break
log.info('Creating tar archive')
+
+ uid = _get_uid(owner)
+ gid = _get_gid(group)
+
+ def _set_uid_gid(tarinfo):
+ if gid is not None:
+ tarinfo.gid = gid
+ tarinfo.gname = group
+ if uid is not None:
+ tarinfo.uid = uid
+ tarinfo.uname = owner
+ return tarinfo
+
if not dry_run:
tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress])
try:
- tar.add(base_dir)
+ tar.add(base_dir, filter=_set_uid_gid)
finally:
tar.close()
@@ -140,7 +193,7 @@ def check_archive_formats(formats):
return None
def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
- dry_run=0):
+ dry_run=0, owner=None, group=None):
"""Create an archive file (eg. zip or tar).
'base_name' is the name of the file to create, minus any format-specific
@@ -153,6 +206,9 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
ie. 'base_dir' will be the common prefix of all files and
directories in the archive. 'root_dir' and 'base_dir' both default
to the current directory. Returns the name of the archive file.
+
+ 'owner' and 'group' are used when creating a tar archive. By default,
+ uses the current owner and group.
"""
save_cwd = os.getcwd()
if root_dir is not None:
@@ -174,6 +230,11 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
func = format_info[0]
for arg, val in format_info[1]:
kwargs[arg] = val
+
+ if format != 'zip':
+ kwargs['owner'] = owner
+ kwargs['group'] = group
+
try:
filename = func(base_name, base_dir, **kwargs)
finally:
diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py
index c795c95..911e84d 100644
--- a/Lib/distutils/ccompiler.py
+++ b/Lib/distutils/ccompiler.py
@@ -351,7 +351,7 @@ class CCompiler:
return macros, objects, extra, pp_opts, build
def _get_cc_args(self, pp_opts, debug, before):
- # works for unixccompiler, emxccompiler, cygwinccompiler
+ # works for unixccompiler, cygwinccompiler
cc_args = pp_opts + ['-c']
if debug:
cc_args[:0] = ['-g']
@@ -926,7 +926,6 @@ _default_compilers = (
# on a cygwin built python we can use gcc like an ordinary UNIXish
# compiler
('cygwin.*', 'unix'),
- ('os2emx', 'emx'),
# OS name mappings
('posix', 'unix'),
@@ -968,8 +967,6 @@ compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler',
"Mingw32 port of GNU C Compiler for Win32"),
'bcpp': ('bcppcompiler', 'BCPPCompiler',
"Borland C++ Compiler"),
- 'emx': ('emxccompiler', 'EMXCCompiler',
- "EMX port of GNU C Compiler for OS/2"),
}
def show_compilers():
diff --git a/Lib/distutils/cmd.py b/Lib/distutils/cmd.py
index 3ea0810..c89d5ef 100644
--- a/Lib/distutils/cmd.py
+++ b/Lib/distutils/cmd.py
@@ -365,9 +365,11 @@ class Command:
from distutils.spawn import spawn
spawn(cmd, search_path, dry_run=self.dry_run)
- def make_archive(self, base_name, format, root_dir=None, base_dir=None):
+ def make_archive(self, base_name, format, root_dir=None, base_dir=None,
+ owner=None, group=None):
return archive_util.make_archive(base_name, format, root_dir, base_dir,
- dry_run=self.dry_run)
+ dry_run=self.dry_run,
+ owner=owner, group=group)
def make_file(self, infiles, outfile, func, args,
exec_msg=None, skip_msg=None, level=1):
diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py
index c5188eb..6814a1c 100644
--- a/Lib/distutils/command/bdist.py
+++ b/Lib/distutils/command/bdist.py
@@ -37,6 +37,12 @@ class bdist(Command):
"[default: dist]"),
('skip-build', None,
"skip rebuilding everything (for testing/debugging)"),
+ ('owner=', 'u',
+ "Owner name used when creating a tar file"
+ " [default: current user]"),
+ ('group=', 'g',
+ "Group name used when creating a tar file"
+ " [default: current group]"),
]
boolean_options = ['skip-build']
@@ -52,8 +58,7 @@ class bdist(Command):
# This won't do in reality: will need to distinguish RPM-ish Linux,
# Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS.
default_format = {'posix': 'gztar',
- 'nt': 'zip',
- 'os2': 'zip'}
+ 'nt': 'zip'}
# Establish the preferred order (for the --help-formats option).
format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar',
@@ -78,6 +83,8 @@ class bdist(Command):
self.formats = None
self.dist_dir = None
self.skip_build = 0
+ self.group = None
+ self.owner = None
def finalize_options(self):
# have to finalize 'plat_name' before 'bdist_base'
@@ -123,6 +130,11 @@ class bdist(Command):
if cmd_name not in self.no_format_option:
sub_cmd.format = self.formats[i]
+ # passing the owner and group names for tar archiving
+ if cmd_name == 'bdist_dumb':
+ sub_cmd.owner = self.owner
+ sub_cmd.group = self.group
+
# If we're going to need to run this command again, tell it to
# keep its temporary files around so subsequent runs go faster.
if cmd_name in commands[i+1:]:
diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py
index 1ab09d1..4405d12 100644
--- a/Lib/distutils/command/bdist_dumb.py
+++ b/Lib/distutils/command/bdist_dumb.py
@@ -33,13 +33,18 @@ class bdist_dumb(Command):
('relative', None,
"build the archive using relative paths"
"(default: false)"),
+ ('owner=', 'u',
+ "Owner name used when creating a tar file"
+ " [default: current user]"),
+ ('group=', 'g',
+ "Group name used when creating a tar file"
+ " [default: current group]"),
]
boolean_options = ['keep-temp', 'skip-build', 'relative']
default_format = { 'posix': 'gztar',
- 'nt': 'zip',
- 'os2': 'zip' }
+ 'nt': 'zip' }
def initialize_options(self):
self.bdist_dir = None
@@ -49,6 +54,8 @@ class bdist_dumb(Command):
self.dist_dir = None
self.skip_build = None
self.relative = 0
+ self.owner = None
+ self.group = None
def finalize_options(self):
if self.bdist_dir is None:
@@ -85,11 +92,6 @@ class bdist_dumb(Command):
archive_basename = "%s.%s" % (self.distribution.get_fullname(),
self.plat_name)
- # OS/2 objects to any ":" characters in a filename (such as when
- # a timestamp is used in a version) so change them to hyphens.
- if os.name == "os2":
- archive_basename = archive_basename.replace(":", "-")
-
pseudoinstall_root = os.path.join(self.dist_dir, archive_basename)
if not self.relative:
archive_root = self.bdist_dir
@@ -107,7 +109,8 @@ class bdist_dumb(Command):
# Make the archive
filename = self.make_archive(pseudoinstall_root,
- self.format, root_dir=archive_root)
+ self.format, root_dir=archive_root,
+ owner=self.owner, group=self.group)
if self.distribution.has_ext_modules():
pyversion = get_python_version()
else:
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index bc6a23f..80689b6 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -230,11 +230,6 @@ class build_ext(Command):
self.library_dirs.append(os.path.join(sys.exec_prefix,
'PC', 'VC6'))
- # OS/2 (EMX) doesn't support Debug vs Release builds, but has the
- # import libraries in its "Config" subdirectory
- if os.name == 'os2':
- self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config'))
-
# for extensions under Cygwin and AtheOS Python's library directory must be
# appended to library_dirs
if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos':
@@ -619,9 +614,6 @@ class build_ext(Command):
return fn
else:
return "swig.exe"
- elif os.name == "os2":
- # assume swig available in the PATH.
- return "swig.exe"
else:
raise DistutilsPlatformError(
"I don't know how to find (much less run) SWIG "
@@ -672,9 +664,6 @@ class build_ext(Command):
"""
from distutils.sysconfig import get_config_var
ext_path = ext_name.split('.')
- # OS/2 has an 8 character module (extension) limit :-(
- if os.name == "os2":
- ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8]
# extensions in debug_mode are named 'module_d.pyd' under windows
ext_suffix = get_config_var('EXT_SUFFIX')
if os.name == 'nt' and self.debug:
@@ -695,7 +684,7 @@ class build_ext(Command):
def get_libraries(self, ext):
"""Return the list of libraries to link against when building a
shared extension. On most platforms, this is just 'ext.libraries';
- on Windows and OS/2, we add the Python library (eg. python20.dll).
+ on Windows, we add the Python library (eg. python20.dll).
"""
# The python library is always needed on Windows. For MSVC, this
# is redundant, since the library is mentioned in a pragma in
@@ -715,19 +704,6 @@ class build_ext(Command):
return ext.libraries + [pythonlib]
else:
return ext.libraries
- elif sys.platform == "os2emx":
- # EMX/GCC requires the python library explicitly, and I
- # believe VACPP does as well (though not confirmed) - AIM Apr01
- template = "python%d%d"
- # debug versions of the main DLL aren't supported, at least
- # not at this time - AIM Apr01
- #if self.debug:
- # template = template + '_d'
- pythonlib = (template %
- (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
- # don't extend ext.libraries, it may be shared with other
- # extensions, it is a reference to the original list
- return ext.libraries + [pythonlib]
elif sys.platform[:6] == "cygwin":
template = "python%d.%d"
pythonlib = (template %
diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py
index d48eb69..9100b96 100644
--- a/Lib/distutils/command/build_py.py
+++ b/Lib/distutils/command/build_py.py
@@ -3,7 +3,7 @@
Implements the Distutils 'build_py' command."""
import os
-import imp
+import importlib.util
import sys
from glob import glob
@@ -313,11 +313,11 @@ class build_py (Command):
outputs.append(filename)
if include_bytecode:
if self.compile:
- outputs.append(imp.cache_from_source(filename,
- debug_override=True))
+ outputs.append(importlib.util.cache_from_source(
+ filename, debug_override=True))
if self.optimize > 0:
- outputs.append(imp.cache_from_source(filename,
- debug_override=False))
+ outputs.append(importlib.util.cache_from_source(
+ filename, debug_override=False))
outputs += [
os.path.join(build_dir, filename)
diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py
index 4b5b22e..90a8380 100644
--- a/Lib/distutils/command/build_scripts.py
+++ b/Lib/distutils/command/build_scripts.py
@@ -74,7 +74,7 @@ class build_scripts(Command):
# script.
try:
f = open(script, "rb")
- except IOError:
+ except OSError:
if not self.dry_run:
raise
f = None
diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py
index 3c675d1..456511c 100644
--- a/Lib/distutils/command/install.py
+++ b/Lib/distutils/command/install.py
@@ -58,13 +58,6 @@ INSTALL_SCHEMES = {
'data' : '$base',
},
'nt': WINDOWS_SCHEME,
- 'os2': {
- 'purelib': '$base/Lib/site-packages',
- 'platlib': '$base/Lib/site-packages',
- 'headers': '$base/Include/$dist_name',
- 'scripts': '$base/Scripts',
- 'data' : '$base',
- },
}
# user site schemes
@@ -86,14 +79,6 @@ if HAS_USER_SITE:
'data' : '$userbase',
}
- INSTALL_SCHEMES['os2_home'] = {
- 'purelib': '$usersite',
- 'platlib': '$usersite',
- 'headers': '$userbase/include/python$py_version_short/$dist_name',
- 'scripts': '$userbase/bin',
- 'data' : '$userbase',
- }
-
# The keys to an installation scheme; if any new types of files are to be
# installed, be sure to add an entry to every installation scheme above,
# and to SCHEME_KEYS here.
diff --git a/Lib/distutils/command/install_lib.py b/Lib/distutils/command/install_lib.py
index 15c08f1..215813b 100644
--- a/Lib/distutils/command/install_lib.py
+++ b/Lib/distutils/command/install_lib.py
@@ -4,7 +4,7 @@ Implements the Distutils 'install_lib' command
(install all Python modules)."""
import os
-import imp
+import importlib.util
import sys
from distutils.core import Command
@@ -165,10 +165,10 @@ class install_lib(Command):
if ext != PYTHON_SOURCE_EXTENSION:
continue
if self.compile:
- bytecode_files.append(imp.cache_from_source(
+ bytecode_files.append(importlib.util.cache_from_source(
py_file, debug_override=True))
if self.optimize > 0:
- bytecode_files.append(imp.cache_from_source(
+ bytecode_files.append(importlib.util.cache_from_source(
py_file, debug_override=False))
return bytecode_files
diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py
index 116f67e..7ea3d5f 100644
--- a/Lib/distutils/command/sdist.py
+++ b/Lib/distutils/command/sdist.py
@@ -74,6 +74,10 @@ class sdist(Command):
('metadata-check', None,
"Ensure that all required elements of meta-data "
"are supplied. Warn if any missing. [default]"),
+ ('owner=', 'u',
+ "Owner name used when creating a tar file [default: current user]"),
+ ('group=', 'g',
+ "Group name used when creating a tar file [default: current group]"),
]
boolean_options = ['use-defaults', 'prune',
@@ -113,6 +117,8 @@ class sdist(Command):
self.archive_files = None
self.metadata_check = 1
+ self.owner = None
+ self.group = None
def finalize_options(self):
if self.manifest is None:
@@ -444,7 +450,8 @@ class sdist(Command):
self.formats.append(self.formats.pop(self.formats.index('tar')))
for fmt in self.formats:
- file = self.make_archive(base_name, fmt, base_dir=base_dir)
+ file = self.make_archive(base_name, fmt, base_dir=base_dir,
+ owner=self.owner, group=self.group)
archive_files.append(file)
self.distribution.dist_files.append(('sdist', '', file))
diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py
index e30c189..d6762e4 100644
--- a/Lib/distutils/command/upload.py
+++ b/Lib/distutils/command/upload.py
@@ -182,7 +182,7 @@ class upload(PyPIRCCommand):
result = urlopen(request)
status = result.getcode()
reason = result.msg
- except socket.error as e:
+ except OSError as e:
self.announce(str(e), log.ERROR)
return
except HTTPError as e:
diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py
index 106e146..382aca8 100644
--- a/Lib/distutils/config.py
+++ b/Lib/distutils/config.py
@@ -83,6 +83,15 @@ class PyPIRCCommand(Command):
current[key] = config.get(server, key)
else:
current[key] = default
+
+ # work around people having "repository" for the "pypi"
+ # section of their config set to the HTTP (rather than
+ # HTTPS) URL
+ if (server == 'pypi' and
+ repository in (self.DEFAULT_REPOSITORY, 'pypi')):
+ current['repository'] = self.DEFAULT_REPOSITORY
+ return current
+
if (current['server'] == repository or
current['repository'] == repository):
return current
diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py
index 25d91ba..2bfe66a 100644
--- a/Lib/distutils/core.py
+++ b/Lib/distutils/core.py
@@ -127,8 +127,9 @@ def setup (**attrs):
if _setup_stop_after == "config":
return dist
- # Parse the command line; any command-line errors are the end user's
- # fault, so turn them into SystemExit to suppress tracebacks.
+ # Parse the command line and override config files; any
+ # command-line errors are the end user's fault, so turn them into
+ # SystemExit to suppress tracebacks.
try:
ok = dist.parse_command_line()
except DistutilsArgError as msg:
@@ -147,7 +148,7 @@ def setup (**attrs):
dist.run_commands()
except KeyboardInterrupt:
raise SystemExit("interrupted")
- except (IOError, os.error) as exc:
+ except OSError as exc:
if DEBUG:
sys.stderr.write("error: %s\n" % (exc,))
raise
diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py
index e0074a1..d28b1b3 100644
--- a/Lib/distutils/cygwinccompiler.py
+++ b/Lib/distutils/cygwinccompiler.py
@@ -54,7 +54,8 @@ import re
from distutils.ccompiler import gen_preprocess_options, gen_lib_options
from distutils.unixccompiler import UnixCCompiler
from distutils.file_util import write_file
-from distutils.errors import DistutilsExecError, CompileError, UnknownFileError
+from distutils.errors import (DistutilsExecError, CCompilerError,
+ CompileError, UnknownFileError)
from distutils import log
from distutils.version import LooseVersion
from distutils.spawn import find_executable
@@ -294,18 +295,17 @@ class Mingw32CCompiler(CygwinCCompiler):
else:
entry_point = ''
- if self.gcc_version < '4' or is_cygwingcc():
- no_cygwin = ' -mno-cygwin'
- else:
- no_cygwin = ''
-
- self.set_executables(compiler='gcc%s -O -Wall' % no_cygwin,
- compiler_so='gcc%s -mdll -O -Wall' % no_cygwin,
- compiler_cxx='g++%s -O -Wall' % no_cygwin,
- linker_exe='gcc%s' % no_cygwin,
- linker_so='%s%s %s %s'
- % (self.linker_dll, no_cygwin,
- shared_option, entry_point))
+ if is_cygwingcc():
+ raise CCompilerError(
+ 'Cygwin gcc cannot be used with --compiler=mingw32')
+
+ self.set_executables(compiler='gcc -O -Wall',
+ compiler_so='gcc -mdll -O -Wall',
+ compiler_cxx='g++ -O -Wall',
+ linker_exe='gcc',
+ linker_so='%s %s %s'
+ % (self.linker_dll, shared_option,
+ entry_point))
# Maybe we should also append -mthreads, but then the finished
# dlls need another dll (mingwm10.dll see Mingw32 docs)
# (-mthreads: Support thread-safe exception handling on `Mingw32')
@@ -364,7 +364,7 @@ def check_config_h():
return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn
finally:
config_h.close()
- except IOError as exc:
+ except OSError as exc:
return (CONFIG_H_UNCERTAIN,
"couldn't read '%s': %s" % (fn, exc.strerror))
diff --git a/Lib/distutils/dir_util.py b/Lib/distutils/dir_util.py
index 6a72bdd..9879b0d 100644
--- a/Lib/distutils/dir_util.py
+++ b/Lib/distutils/dir_util.py
@@ -124,7 +124,7 @@ def copy_tree(src, dst, preserve_mode=1, preserve_times=1,
"cannot copy tree '%s': not a directory" % src)
try:
names = os.listdir(src)
- except os.error as e:
+ except OSError as e:
(errno, errstr) = e
if dry_run:
names = []
@@ -197,7 +197,7 @@ def remove_tree(directory, verbose=1, dry_run=0):
abspath = os.path.abspath(cmd[1])
if abspath in _path_created:
del _path_created[abspath]
- except (IOError, OSError) as exc:
+ except OSError as exc:
log.warn("error removing %s: %s", directory, exc)
def ensure_relative(path):
diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py
index 11a2102..7eb04bc 100644
--- a/Lib/distutils/dist.py
+++ b/Lib/distutils/dist.py
@@ -52,7 +52,9 @@ class Distribution:
('quiet', 'q', "run quietly (turns verbosity off)"),
('dry-run', 'n', "don't actually do anything"),
('help', 'h', "show detailed help message"),
- ]
+ ('no-user-cfg', None,
+ 'ignore pydistutils.cfg in your home directory'),
+ ]
# 'common_usage' is a short (2-3 line) string describing the common
# usage of the setup script.
@@ -259,6 +261,22 @@ Common commands: (see '--help-commands' for more)
else:
sys.stderr.write(msg + "\n")
+ # no-user-cfg is handled before other command line args
+ # because other args override the config files, and this
+ # one is needed before we can load the config files.
+ # If attrs['script_args'] wasn't passed, assume false.
+ #
+ # This also make sure we just look at the global options
+ self.want_user_cfg = True
+
+ if self.script_args is not None:
+ for arg in self.script_args:
+ if not arg.startswith('-'):
+ break
+ if arg == '--no-user-cfg':
+ self.want_user_cfg = False
+ break
+
self.finalize_options()
def get_option_dict(self, command):
@@ -310,7 +328,10 @@ Common commands: (see '--help-commands' for more)
Distutils installation directory (ie. where the top-level
Distutils __inst__.py file lives), a file in the user's home
directory named .pydistutils.cfg on Unix and pydistutils.cfg
- on Windows/Mac, and setup.cfg in the current directory.
+ on Windows/Mac; and setup.cfg in the current directory.
+
+ The file in the user's home directory can be disabled with the
+ --no-user-cfg option.
"""
files = []
check_environ()
@@ -330,15 +351,19 @@ Common commands: (see '--help-commands' for more)
user_filename = "pydistutils.cfg"
# And look for the user config file
- user_file = os.path.join(os.path.expanduser('~'), user_filename)
- if os.path.isfile(user_file):
- files.append(user_file)
+ if self.want_user_cfg:
+ user_file = os.path.join(os.path.expanduser('~'), user_filename)
+ if os.path.isfile(user_file):
+ files.append(user_file)
# All platforms support local setup.cfg
local_file = "setup.cfg"
if os.path.isfile(local_file):
files.append(local_file)
+ if DEBUG:
+ self.announce("using config files: %s" % ', '.join(files))
+
return files
def parse_config_files(self, filenames=None):
diff --git a/Lib/distutils/emxccompiler.py b/Lib/distutils/emxccompiler.py
deleted file mode 100644
index 3675f8d..0000000
--- a/Lib/distutils/emxccompiler.py
+++ /dev/null
@@ -1,315 +0,0 @@
-"""distutils.emxccompiler
-
-Provides the EMXCCompiler class, a subclass of UnixCCompiler that
-handles the EMX port of the GNU C compiler to OS/2.
-"""
-
-# issues:
-#
-# * OS/2 insists that DLLs can have names no longer than 8 characters
-# We put export_symbols in a def-file, as though the DLL can have
-# an arbitrary length name, but truncate the output filename.
-#
-# * only use OMF objects and use LINK386 as the linker (-Zomf)
-#
-# * always build for multithreading (-Zmt) as the accompanying OS/2 port
-# of Python is only distributed with threads enabled.
-#
-# tested configurations:
-#
-# * EMX gcc 2.81/EMX 0.9d fix03
-
-import os,sys,copy
-from distutils.ccompiler import gen_preprocess_options, gen_lib_options
-from distutils.unixccompiler import UnixCCompiler
-from distutils.file_util import write_file
-from distutils.errors import DistutilsExecError, CompileError, UnknownFileError
-from distutils import log
-
-class EMXCCompiler (UnixCCompiler):
-
- compiler_type = 'emx'
- obj_extension = ".obj"
- static_lib_extension = ".lib"
- shared_lib_extension = ".dll"
- static_lib_format = "%s%s"
- shared_lib_format = "%s%s"
- res_extension = ".res" # compiled resource file
- exe_extension = ".exe"
-
- def __init__ (self,
- verbose=0,
- dry_run=0,
- force=0):
-
- UnixCCompiler.__init__ (self, verbose, dry_run, force)
-
- (status, details) = check_config_h()
- self.debug_print("Python's GCC status: %s (details: %s)" %
- (status, details))
- if status is not CONFIG_H_OK:
- self.warn(
- "Python's pyconfig.h doesn't seem to support your compiler. " +
- ("Reason: %s." % details) +
- "Compiling may fail because of undefined preprocessor macros.")
-
- (self.gcc_version, self.ld_version) = \
- get_versions()
- self.debug_print(self.compiler_type + ": gcc %s, ld %s\n" %
- (self.gcc_version,
- self.ld_version) )
-
- # Hard-code GCC because that's what this is all about.
- # XXX optimization, warnings etc. should be customizable.
- self.set_executables(compiler='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall',
- compiler_so='gcc -Zomf -Zmt -O3 -fomit-frame-pointer -mprobe -Wall',
- linker_exe='gcc -Zomf -Zmt -Zcrtdll',
- linker_so='gcc -Zomf -Zmt -Zcrtdll -Zdll')
-
- # want the gcc library statically linked (so that we don't have
- # to distribute a version dependent on the compiler we have)
- self.dll_libraries=["gcc"]
-
- # __init__ ()
-
- def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts):
- if ext == '.rc':
- # gcc requires '.rc' compiled to binary ('.res') files !!!
- try:
- self.spawn(["rc", "-r", src])
- except DistutilsExecError as msg:
- raise CompileError(msg)
- else: # for other files use the C-compiler
- try:
- self.spawn(self.compiler_so + cc_args + [src, '-o', obj] +
- extra_postargs)
- except DistutilsExecError as msg:
- raise CompileError(msg)
-
- def link (self,
- target_desc,
- objects,
- output_filename,
- output_dir=None,
- libraries=None,
- library_dirs=None,
- runtime_library_dirs=None,
- export_symbols=None,
- debug=0,
- extra_preargs=None,
- extra_postargs=None,
- build_temp=None,
- target_lang=None):
-
- # use separate copies, so we can modify the lists
- extra_preargs = copy.copy(extra_preargs or [])
- libraries = copy.copy(libraries or [])
- objects = copy.copy(objects or [])
-
- # Additional libraries
- libraries.extend(self.dll_libraries)
-
- # handle export symbols by creating a def-file
- # with executables this only works with gcc/ld as linker
- if ((export_symbols is not None) and
- (target_desc != self.EXECUTABLE)):
- # (The linker doesn't do anything if output is up-to-date.
- # So it would probably better to check if we really need this,
- # but for this we had to insert some unchanged parts of
- # UnixCCompiler, and this is not what we want.)
-
- # we want to put some files in the same directory as the
- # object files are, build_temp doesn't help much
- # where are the object files
- temp_dir = os.path.dirname(objects[0])
- # name of dll to give the helper files the same base name
- (dll_name, dll_extension) = os.path.splitext(
- os.path.basename(output_filename))
-
- # generate the filenames for these files
- def_file = os.path.join(temp_dir, dll_name + ".def")
-
- # Generate .def file
- contents = [
- "LIBRARY %s INITINSTANCE TERMINSTANCE" % \
- os.path.splitext(os.path.basename(output_filename))[0],
- "DATA MULTIPLE NONSHARED",
- "EXPORTS"]
- for sym in export_symbols:
- contents.append(' "%s"' % sym)
- self.execute(write_file, (def_file, contents),
- "writing %s" % def_file)
-
- # next add options for def-file and to creating import libraries
- # for gcc/ld the def-file is specified as any other object files
- objects.append(def_file)
-
- #end: if ((export_symbols is not None) and
- # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")):
-
- # who wants symbols and a many times larger output file
- # should explicitly switch the debug mode on
- # otherwise we let dllwrap/ld strip the output file
- # (On my machine: 10KB < stripped_file < ??100KB
- # unstripped_file = stripped_file + XXX KB
- # ( XXX=254 for a typical python extension))
- if not debug:
- extra_preargs.append("-s")
-
- UnixCCompiler.link(self,
- target_desc,
- objects,
- output_filename,
- output_dir,
- libraries,
- library_dirs,
- runtime_library_dirs,
- None, # export_symbols, we do this in our def-file
- debug,
- extra_preargs,
- extra_postargs,
- build_temp,
- target_lang)
-
- # link ()
-
- # -- Miscellaneous methods -----------------------------------------
-
- # override the object_filenames method from CCompiler to
- # support rc and res-files
- def object_filenames (self,
- source_filenames,
- strip_dir=0,
- output_dir=''):
- if output_dir is None: output_dir = ''
- obj_names = []
- for src_name in source_filenames:
- # use normcase to make sure '.rc' is really '.rc' and not '.RC'
- (base, ext) = os.path.splitext (os.path.normcase(src_name))
- if ext not in (self.src_extensions + ['.rc']):
- raise UnknownFileError("unknown file type '%s' (from '%s')" % \
- (ext, src_name))
- if strip_dir:
- base = os.path.basename (base)
- if ext == '.rc':
- # these need to be compiled to object files
- obj_names.append (os.path.join (output_dir,
- base + self.res_extension))
- else:
- obj_names.append (os.path.join (output_dir,
- base + self.obj_extension))
- return obj_names
-
- # object_filenames ()
-
- # override the find_library_file method from UnixCCompiler
- # to deal with file naming/searching differences
- def find_library_file(self, dirs, lib, debug=0):
- shortlib = '%s.lib' % lib
- longlib = 'lib%s.lib' % lib # this form very rare
-
- # get EMX's default library directory search path
- try:
- emx_dirs = os.environ['LIBRARY_PATH'].split(';')
- except KeyError:
- emx_dirs = []
-
- for dir in dirs + emx_dirs:
- shortlibp = os.path.join(dir, shortlib)
- longlibp = os.path.join(dir, longlib)
- if os.path.exists(shortlibp):
- return shortlibp
- elif os.path.exists(longlibp):
- return longlibp
-
- # Oops, didn't find it in *any* of 'dirs'
- return None
-
-# class EMXCCompiler
-
-
-# Because these compilers aren't configured in Python's pyconfig.h file by
-# default, we should at least warn the user if he is using a unmodified
-# version.
-
-CONFIG_H_OK = "ok"
-CONFIG_H_NOTOK = "not ok"
-CONFIG_H_UNCERTAIN = "uncertain"
-
-def check_config_h():
-
- """Check if the current Python installation (specifically, pyconfig.h)
- appears amenable to building extensions with GCC. Returns a tuple
- (status, details), where 'status' is one of the following constants:
- CONFIG_H_OK
- all is well, go ahead and compile
- CONFIG_H_NOTOK
- doesn't look good
- CONFIG_H_UNCERTAIN
- not sure -- unable to read pyconfig.h
- 'details' is a human-readable string explaining the situation.
-
- Note there are two ways to conclude "OK": either 'sys.version' contains
- the string "GCC" (implying that this Python was built with GCC), or the
- installed "pyconfig.h" contains the string "__GNUC__".
- """
-
- # XXX since this function also checks sys.version, it's not strictly a
- # "pyconfig.h" check -- should probably be renamed...
-
- from distutils import sysconfig
- # if sys.version contains GCC then python was compiled with
- # GCC, and the pyconfig.h file should be OK
- if sys.version.find("GCC") >= 0:
- return (CONFIG_H_OK, "sys.version mentions 'GCC'")
-
- fn = sysconfig.get_config_h_filename()
- try:
- # It would probably better to read single lines to search.
- # But we do this only once, and it is fast enough
- f = open(fn)
- try:
- s = f.read()
- finally:
- f.close()
-
- except IOError as exc:
- # if we can't read this file, we cannot say it is wrong
- # the compiler will complain later about this file as missing
- return (CONFIG_H_UNCERTAIN,
- "couldn't read '%s': %s" % (fn, exc.strerror))
-
- else:
- # "pyconfig.h" contains an "#ifdef __GNUC__" or something similar
- if s.find("__GNUC__") >= 0:
- return (CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn)
- else:
- return (CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn)
-
-
-def get_versions():
- """ Try to find out the versions of gcc and ld.
- If not possible it returns None for it.
- """
- from distutils.version import StrictVersion
- from distutils.spawn import find_executable
- import re
-
- gcc_exe = find_executable('gcc')
- if gcc_exe:
- out = os.popen(gcc_exe + ' -dumpversion','r')
- try:
- out_string = out.read()
- finally:
- out.close()
- result = re.search('(\d+\.\d+\.\d+)', out_string, re.ASCII)
- if result:
- gcc_version = StrictVersion(result.group(1))
- else:
- gcc_version = None
- else:
- gcc_version = None
- # EMX ld has no way of reporting version number, and we use GCC
- # anyway - so we can link OMF DLLs
- ld_version = None
- return (gcc_version, ld_version)
diff --git a/Lib/distutils/errors.py b/Lib/distutils/errors.py
index eb13c98..8b93059 100644
--- a/Lib/distutils/errors.py
+++ b/Lib/distutils/errors.py
@@ -35,8 +35,8 @@ class DistutilsArgError (DistutilsError):
class DistutilsFileError (DistutilsError):
"""Any problems in the filesystem: expected file not found, etc.
- Typically this is for problems that we detect before IOError or
- OSError could be raised."""
+ Typically this is for problems that we detect before OSError
+ could be raised."""
pass
class DistutilsOptionError (DistutilsError):
diff --git a/Lib/distutils/file_util.py b/Lib/distutils/file_util.py
index 9bdd14e..f6ed290 100644
--- a/Lib/distutils/file_util.py
+++ b/Lib/distutils/file_util.py
@@ -27,26 +27,26 @@ def _copy_file_contents(src, dst, buffer_size=16*1024):
try:
try:
fsrc = open(src, 'rb')
- except os.error as e:
+ except OSError as e:
raise DistutilsFileError("could not open '%s': %s" % (src, e.strerror))
if os.path.exists(dst):
try:
os.unlink(dst)
- except os.error as e:
+ except OSError as e:
raise DistutilsFileError(
"could not delete '%s': %s" % (dst, e.strerror))
try:
fdst = open(dst, 'wb')
- except os.error as e:
+ except OSError as e:
raise DistutilsFileError(
"could not create '%s': %s" % (dst, e.strerror))
while True:
try:
buf = fsrc.read(buffer_size)
- except os.error as e:
+ except OSError as e:
raise DistutilsFileError(
"could not read from '%s': %s" % (src, e.strerror))
@@ -55,7 +55,7 @@ def _copy_file_contents(src, dst, buffer_size=16*1024):
try:
fdst.write(buf)
- except os.error as e:
+ except OSError as e:
raise DistutilsFileError(
"could not write to '%s': %s" % (dst, e.strerror))
finally:
@@ -193,7 +193,7 @@ def move_file (src, dst,
copy_it = False
try:
os.rename(src, dst)
- except os.error as e:
+ except OSError as e:
(num, msg) = e
if num == errno.EXDEV:
copy_it = True
@@ -205,11 +205,11 @@ def move_file (src, dst,
copy_file(src, dst, verbose=verbose)
try:
os.unlink(src)
- except os.error as e:
+ except OSError as e:
(num, msg) = e
try:
os.unlink(dst)
- except os.error:
+ except OSError:
pass
raise DistutilsFileError(
"couldn't move '%s' to '%s' by copy/delete: "
diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py
index b3f6ce1..9688f20 100644
--- a/Lib/distutils/msvc9compiler.py
+++ b/Lib/distutils/msvc9compiler.py
@@ -729,7 +729,7 @@ class MSVCCompiler(CCompiler) :
return manifest_file
finally:
manifest_f.close()
- except IOError:
+ except OSError:
pass
# -- Miscellaneous methods -----------------------------------------
diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py
index f66ff93..22e87e8 100644
--- a/Lib/distutils/spawn.py
+++ b/Lib/distutils/spawn.py
@@ -36,8 +36,6 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0):
_spawn_posix(cmd, search_path, dry_run=dry_run)
elif os.name == 'nt':
_spawn_nt(cmd, search_path, dry_run=dry_run)
- elif os.name == 'os2':
- _spawn_os2(cmd, search_path, dry_run=dry_run)
else:
raise DistutilsPlatformError(
"don't know how to spawn programs on platform '%s'" % os.name)
@@ -82,30 +80,6 @@ def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0):
raise DistutilsExecError(
"command %r failed with exit status %d" % (cmd, rc))
-def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0):
- executable = cmd[0]
- if search_path:
- # either we find one or it stays the same
- executable = find_executable(executable) or executable
- log.info(' '.join([executable] + cmd[1:]))
- if not dry_run:
- # spawnv for OS/2 EMX requires a full path to the .exe
- try:
- rc = os.spawnv(os.P_WAIT, executable, cmd)
- except OSError as exc:
- # this seems to happen when the command isn't found
- if not DEBUG:
- cmd = executable
- raise DistutilsExecError(
- "command %r failed: %s" % (cmd, exc.args[-1]))
- if rc != 0:
- # and this reflects the command running but failing
- if not DEBUG:
- cmd = executable
- log.debug("command %r failed with exit status %d" % (cmd, rc))
- raise DistutilsExecError(
- "command %r failed with exit status %d" % (cmd, rc))
-
if sys.platform == 'darwin':
from distutils import sysconfig
_cfg_target = None
@@ -207,7 +181,7 @@ def find_executable(executable, path=None):
paths = path.split(os.pathsep)
base, ext = os.path.splitext(executable)
- if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'):
+ if (sys.platform == 'win32') and (ext != '.exe'):
executable = executable + '.exe'
if not os.path.isfile(executable):
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
index b947988..75537db 100644
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -114,8 +114,6 @@ def get_python_inc(plat_specific=0, prefix=None):
return os.path.join(prefix, "include", python_dir)
elif os.name == "nt":
return os.path.join(prefix, "include")
- elif os.name == "os2":
- return os.path.join(prefix, "Include")
else:
raise DistutilsPlatformError(
"I don't know where Python installs its C header files "
@@ -157,11 +155,6 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
return prefix
else:
return os.path.join(prefix, "Lib", "site-packages")
- elif os.name == "os2":
- if standard_lib:
- return os.path.join(prefix, "Lib")
- else:
- return os.path.join(prefix, "Lib", "site-packages")
else:
raise DistutilsPlatformError(
"I don't know where Python installs its library "
@@ -442,7 +435,7 @@ def _init_posix():
try:
filename = get_makefile_filename()
parse_makefile(filename, g)
- except IOError as msg:
+ except OSError as msg:
my_msg = "invalid Python installation: unable to open %s" % filename
if hasattr(msg, "strerror"):
my_msg = my_msg + " (%s)" % msg.strerror
@@ -454,7 +447,7 @@ def _init_posix():
filename = get_config_h_filename()
with open(filename) as file:
parse_config_h(file, g)
- except IOError as msg:
+ except OSError as msg:
my_msg = "invalid Python installation: unable to open %s" % filename
if hasattr(msg, "strerror"):
my_msg = my_msg + " (%s)" % msg.strerror
@@ -492,7 +485,6 @@ def _init_nt():
# XXX hmmm.. a normal install puts include files here
g['INCLUDEPY'] = get_python_inc(plat_specific=0)
- g['SO'] = '.pyd'
g['EXT_SUFFIX'] = '.pyd'
g['EXE'] = ".exe"
g['VERSION'] = get_python_version().replace(".", "")
@@ -502,24 +494,6 @@ def _init_nt():
_config_vars = g
-def _init_os2():
- """Initialize the module as appropriate for OS/2"""
- g = {}
- # set basic install directories
- g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
- g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
- # XXX hmmm.. a normal install puts include files here
- g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
- g['SO'] = '.pyd'
- g['EXT_SUFFIX'] = '.pyd'
- g['EXE'] = ".exe"
-
- global _config_vars
- _config_vars = g
-
-
def get_config_vars(*args):
"""With no arguments, return a dictionary of all configuration
variables relevant for the current platform. Generally this includes
@@ -544,6 +518,11 @@ def get_config_vars(*args):
_config_vars['prefix'] = PREFIX
_config_vars['exec_prefix'] = EXEC_PREFIX
+ # For backward compatibility, see issue19555
+ SO = _config_vars.get('EXT_SUFFIX')
+ if SO is not None:
+ _config_vars['SO'] = SO
+
# Always convert srcdir to an absolute path
srcdir = _config_vars.get('srcdir', project_base)
if os.name == 'posix':
@@ -594,4 +573,7 @@ def get_config_var(name):
returned by 'get_config_vars()'. Equivalent to
get_config_vars().get(name)
"""
+ if name == 'SO':
+ import warnings
+ warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
return get_config_vars().get(name)
diff --git a/Lib/distutils/tests/test_archive_util.py b/Lib/distutils/tests/test_archive_util.py
index d3fb24a..2d72af4 100644
--- a/Lib/distutils/tests/test_archive_util.py
+++ b/Lib/distutils/tests/test_archive_util.py
@@ -16,6 +16,13 @@ from distutils.tests import support
from test.support import check_warnings, run_unittest, patch
try:
+ import grp
+ import pwd
+ UID_GID_SUPPORT = True
+except ImportError:
+ UID_GID_SUPPORT = False
+
+try:
import zipfile
ZIP_SUPPORT = True
except ImportError:
@@ -77,7 +84,7 @@ class ArchiveUtilTestCase(support.TempdirManager,
tmpdir2 = self.mkdtemp()
unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0],
- "Source and target should be on same drive")
+ "source and target should be on same drive")
base_name = os.path.join(tmpdir2, target_name)
@@ -275,6 +282,58 @@ class ArchiveUtilTestCase(support.TempdirManager,
finally:
del ARCHIVE_FORMATS['xxx']
+ def test_make_archive_owner_group(self):
+ # testing make_archive with owner and group, with various combinations
+ # this works even if there's not gid/uid support
+ if UID_GID_SUPPORT:
+ group = grp.getgrgid(0)[0]
+ owner = pwd.getpwuid(0)[0]
+ else:
+ group = owner = 'root'
+
+ base_dir, root_dir, base_name = self._create_files()
+ base_name = os.path.join(self.mkdtemp() , 'archive')
+ res = make_archive(base_name, 'zip', root_dir, base_dir, owner=owner,
+ group=group)
+ self.assertTrue(os.path.exists(res))
+
+ res = make_archive(base_name, 'zip', root_dir, base_dir)
+ self.assertTrue(os.path.exists(res))
+
+ res = make_archive(base_name, 'tar', root_dir, base_dir,
+ owner=owner, group=group)
+ self.assertTrue(os.path.exists(res))
+
+ res = make_archive(base_name, 'tar', root_dir, base_dir,
+ owner='kjhkjhkjg', group='oihohoh')
+ self.assertTrue(os.path.exists(res))
+
+ @unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib")
+ @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
+ def test_tarfile_root_owner(self):
+ tmpdir, tmpdir2, base_name = self._create_files()
+ old_dir = os.getcwd()
+ os.chdir(tmpdir)
+ group = grp.getgrgid(0)[0]
+ owner = pwd.getpwuid(0)[0]
+ try:
+ archive_name = make_tarball(base_name, 'dist', compress=None,
+ owner=owner, group=group)
+ finally:
+ os.chdir(old_dir)
+
+ # check if the compressed tarball was created
+ self.assertTrue(os.path.exists(archive_name))
+
+ # now checks the rights
+ archive = tarfile.open(archive_name)
+ try:
+ for member in archive.getmembers():
+ self.assertEqual(member.uid, 0)
+ self.assertEqual(member.gid, 0)
+ finally:
+ archive.close()
+
def test_suite():
return unittest.makeSuite(ArchiveUtilTestCase)
diff --git a/Lib/distutils/tests/test_bdist_dumb.py b/Lib/distutils/tests/test_bdist_dumb.py
index 0ad32d4..c8ccdc2 100644
--- a/Lib/distutils/tests/test_bdist_dumb.py
+++ b/Lib/distutils/tests/test_bdist_dumb.py
@@ -1,7 +1,6 @@
"""Tests for distutils.command.bdist_dumb."""
import os
-import imp
import sys
import zipfile
import unittest
@@ -75,8 +74,6 @@ class BuildDumbTestCase(support.TempdirManager,
# see what we have
dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name)
- if os.name == 'os2':
- base = base.replace(':', '-')
self.assertEqual(dist_created, [base])
@@ -90,7 +87,7 @@ class BuildDumbTestCase(support.TempdirManager,
contents = sorted(os.path.basename(fn) for fn in contents)
wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py']
if not sys.dont_write_bytecode:
- wanted.append('foo.%s.pyc' % imp.get_tag())
+ wanted.append('foo.%s.pyc' % sys.implementation.cache_tag)
self.assertEqual(contents, sorted(wanted))
def test_suite():
diff --git a/Lib/distutils/tests/test_build_py.py b/Lib/distutils/tests/test_build_py.py
index 2ce9d44..c8f6b89 100644
--- a/Lib/distutils/tests/test_build_py.py
+++ b/Lib/distutils/tests/test_build_py.py
@@ -2,7 +2,6 @@
import os
import sys
-import imp
import unittest
from distutils.command.build_py import build_py
@@ -63,7 +62,8 @@ class BuildPyTestCase(support.TempdirManager,
self.assertFalse(os.path.exists(pycache_dir))
else:
pyc_files = os.listdir(pycache_dir)
- self.assertIn("__init__.%s.pyc" % imp.get_tag(), pyc_files)
+ self.assertIn("__init__.%s.pyc" % sys.implementation.cache_tag,
+ pyc_files)
def test_empty_package_dir(self):
# See bugs #1668596/#1720897
@@ -102,7 +102,8 @@ class BuildPyTestCase(support.TempdirManager,
found = os.listdir(cmd.build_lib)
self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
- self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()])
+ self.assertEqual(found,
+ ['boiledeggs.%s.pyc' % sys.implementation.cache_tag])
@unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
def test_byte_compile_optimized(self):
@@ -119,7 +120,8 @@ class BuildPyTestCase(support.TempdirManager,
found = os.listdir(cmd.build_lib)
self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
- self.assertEqual(sorted(found), ['boiledeggs.%s.pyo' % imp.get_tag()])
+ self.assertEqual(sorted(found),
+ ['boiledeggs.%s.pyo' % sys.implementation.cache_tag])
def test_dir_in_package_data(self):
"""
diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py
index 61ac57d..b7fd3fb 100644
--- a/Lib/distutils/tests/test_dist.py
+++ b/Lib/distutils/tests/test_dist.py
@@ -39,6 +39,7 @@ class TestDistribution(Distribution):
class DistributionTestCase(support.LoggingSilencer,
+ support.TempdirManager,
support.EnvironGuard,
unittest.TestCase):
@@ -213,6 +214,34 @@ class DistributionTestCase(support.LoggingSilencer,
self.assertRaises(ValueError, dist.announce, args, kwargs)
+ def test_find_config_files_disable(self):
+ # Ticket #1180: Allow user to disable their home config file.
+ temp_home = self.mkdtemp()
+ if os.name == 'posix':
+ user_filename = os.path.join(temp_home, ".pydistutils.cfg")
+ else:
+ user_filename = os.path.join(temp_home, "pydistutils.cfg")
+
+ with open(user_filename, 'w') as f:
+ f.write('[distutils]\n')
+
+ def _expander(path):
+ return temp_home
+
+ old_expander = os.path.expanduser
+ os.path.expanduser = _expander
+ try:
+ d = Distribution()
+ all_files = d.find_config_files()
+
+ d = Distribution(attrs={'script_args': ['--no-user-cfg']})
+ files = d.find_config_files()
+ finally:
+ os.path.expanduser = old_expander
+
+ # make sure --no-user-cfg disables the user cfg file
+ self.assertEqual(len(all_files)-1, len(files))
+
class MetadataTestCase(support.TempdirManager, support.EnvironGuard,
unittest.TestCase):
diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py
index ede88e5..18e1e57 100644
--- a/Lib/distutils/tests/test_install.py
+++ b/Lib/distutils/tests/test_install.py
@@ -1,7 +1,6 @@
"""Tests for distutils.command.install."""
import os
-import imp
import sys
import unittest
import site
@@ -94,7 +93,7 @@ class InstallTestCase(support.TempdirManager,
self.addCleanup(cleanup)
- for key in ('nt_user', 'unix_user', 'os2_home'):
+ for key in ('nt_user', 'unix_user'):
self.assertIn(key, INSTALL_SCHEMES)
dist = Distribution({'name': 'xx'})
@@ -193,7 +192,8 @@ class InstallTestCase(support.TempdirManager,
f.close()
found = [os.path.basename(line) for line in content.splitlines()]
- expected = ['hello.py', 'hello.%s.pyc' % imp.get_tag(), 'sayhi',
+ expected = ['hello.py', 'hello.%s.pyc' % sys.implementation.cache_tag,
+ 'sayhi',
'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]]
self.assertEqual(found, expected)
diff --git a/Lib/distutils/tests/test_install_lib.py b/Lib/distutils/tests/test_install_lib.py
index d0dfca0..40dd1a9 100644
--- a/Lib/distutils/tests/test_install_lib.py
+++ b/Lib/distutils/tests/test_install_lib.py
@@ -1,7 +1,7 @@
"""Tests for distutils.command.install_data."""
import sys
import os
-import imp
+import importlib.util
import unittest
from distutils.command.install_lib import install_lib
@@ -44,8 +44,10 @@ class InstallLibTestCase(support.TempdirManager,
f = os.path.join(project_dir, 'foo.py')
self.write_file(f, '# python file')
cmd.byte_compile([f])
- pyc_file = imp.cache_from_source('foo.py', debug_override=True)
- pyo_file = imp.cache_from_source('foo.py', debug_override=False)
+ pyc_file = importlib.util.cache_from_source('foo.py',
+ debug_override=True)
+ pyo_file = importlib.util.cache_from_source('foo.py',
+ debug_override=False)
self.assertTrue(os.path.exists(pyc_file))
self.assertTrue(os.path.exists(pyo_file))
diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py
index c952406..5a04e0d 100644
--- a/Lib/distutils/tests/test_sdist.py
+++ b/Lib/distutils/tests/test_sdist.py
@@ -14,6 +14,12 @@ try:
except ImportError:
ZLIB_SUPPORT = False
+try:
+ import grp
+ import pwd
+ UID_GID_SUPPORT = True
+except ImportError:
+ UID_GID_SUPPORT = False
from distutils.command.sdist import sdist, show_formats
from distutils.core import Distribution
@@ -423,6 +429,54 @@ class SDistTestCase(PyPIRCCommandTestCase):
self.assertEqual(sorted(filenames), ['fake-1.0', 'fake-1.0/PKG-INFO',
'fake-1.0/README.manual'])
+ @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib")
+ @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support")
+ @unittest.skipIf(find_executable('tar') is None,
+ "The tar command is not found")
+ @unittest.skipIf(find_executable('gzip') is None,
+ "The gzip command is not found")
+ def test_make_distribution_owner_group(self):
+ # now building a sdist
+ dist, cmd = self.get_cmd()
+
+ # creating a gztar and specifying the owner+group
+ cmd.formats = ['gztar']
+ cmd.owner = pwd.getpwuid(0)[0]
+ cmd.group = grp.getgrgid(0)[0]
+ cmd.ensure_finalized()
+ cmd.run()
+
+ # making sure we have the good rights
+ archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
+ archive = tarfile.open(archive_name)
+ try:
+ for member in archive.getmembers():
+ self.assertEqual(member.uid, 0)
+ self.assertEqual(member.gid, 0)
+ finally:
+ archive.close()
+
+ # building a sdist again
+ dist, cmd = self.get_cmd()
+
+ # creating a gztar
+ cmd.formats = ['gztar']
+ cmd.ensure_finalized()
+ cmd.run()
+
+ # making sure we have the good rights
+ archive_name = join(self.tmp_dir, 'dist', 'fake-1.0.tar.gz')
+ archive = tarfile.open(archive_name)
+
+ # note that we are not testing the group ownership here
+ # because, depending on the platforms and the container
+ # rights (see #7408)
+ try:
+ for member in archive.getmembers():
+ self.assertEqual(member.uid, os.getuid())
+ finally:
+ archive.close()
+
def test_suite():
return unittest.makeSuite(SDistTestCase)
diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py
index a1cb47d..95fa9dc 100644
--- a/Lib/distutils/tests/test_sysconfig.py
+++ b/Lib/distutils/tests/test_sysconfig.py
@@ -1,16 +1,14 @@
"""Tests for distutils.sysconfig."""
import os
import shutil
-import test
import unittest
from distutils import sysconfig
from distutils.ccompiler import get_default_compiler
from distutils.tests import support
-from test.support import TESTFN, run_unittest
+from test.support import TESTFN, run_unittest, check_warnings
-class SysconfigTestCase(support.EnvironGuard,
- unittest.TestCase):
+class SysconfigTestCase(support.EnvironGuard, unittest.TestCase):
def setUp(self):
super(SysconfigTestCase, self).setUp()
self.makefile = None
@@ -32,7 +30,6 @@ class SysconfigTestCase(support.EnvironGuard,
self.assertTrue(os.path.isfile(config_h), config_h)
def test_get_python_lib(self):
- lib_dir = sysconfig.get_python_lib()
# XXX doesn't work on Linux when Python was never installed before
#self.assertTrue(os.path.isdir(lib_dir), lib_dir)
# test for pythonxx.lib?
@@ -67,8 +64,9 @@ class SysconfigTestCase(support.EnvironGuard,
self.assertTrue(os.path.exists(Python_h), Python_h)
self.assertTrue(sysconfig._is_python_source_dir(srcdir))
elif os.name == 'posix':
- self.assertEqual(os.path.dirname(sysconfig.get_makefile_filename()),
- srcdir)
+ self.assertEqual(
+ os.path.dirname(sysconfig.get_makefile_filename()),
+ srcdir)
def test_srcdir_independent_of_cwd(self):
# srcdir should be independent of the current working directory
@@ -126,10 +124,13 @@ class SysconfigTestCase(support.EnvironGuard,
def test_sysconfig_module(self):
import sysconfig as global_sysconfig
- self.assertEqual(global_sysconfig.get_config_var('CFLAGS'), sysconfig.get_config_var('CFLAGS'))
- self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'), sysconfig.get_config_var('LDFLAGS'))
+ self.assertEqual(global_sysconfig.get_config_var('CFLAGS'),
+ sysconfig.get_config_var('CFLAGS'))
+ self.assertEqual(global_sysconfig.get_config_var('LDFLAGS'),
+ sysconfig.get_config_var('LDFLAGS'))
- @unittest.skipIf(sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'),'compiler flags customized')
+ @unittest.skipIf(sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'),
+ 'compiler flags customized')
def test_sysconfig_compiler_vars(self):
# On OS X, binary installers support extension module building on
# various levels of the operating system with differing Xcode
@@ -148,9 +149,30 @@ class SysconfigTestCase(support.EnvironGuard,
import sysconfig as global_sysconfig
if sysconfig.get_config_var('CUSTOMIZED_OSX_COMPILER'):
self.skipTest('compiler flags customized')
- self.assertEqual(global_sysconfig.get_config_var('LDSHARED'), sysconfig.get_config_var('LDSHARED'))
- self.assertEqual(global_sysconfig.get_config_var('CC'), sysconfig.get_config_var('CC'))
-
+ self.assertEqual(global_sysconfig.get_config_var('LDSHARED'),
+ sysconfig.get_config_var('LDSHARED'))
+ self.assertEqual(global_sysconfig.get_config_var('CC'),
+ sysconfig.get_config_var('CC'))
+
+ @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
+ 'EXT_SUFFIX required for this test')
+ def test_SO_deprecation(self):
+ self.assertWarns(DeprecationWarning,
+ sysconfig.get_config_var, 'SO')
+
+ @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
+ 'EXT_SUFFIX required for this test')
+ def test_SO_value(self):
+ with check_warnings(('', DeprecationWarning)):
+ self.assertEqual(sysconfig.get_config_var('SO'),
+ sysconfig.get_config_var('EXT_SUFFIX'))
+
+ @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None,
+ 'EXT_SUFFIX required for this test')
+ def test_SO_in_vars(self):
+ vars = sysconfig.get_config_vars()
+ self.assertIsNotNone(vars['SO'])
+ self.assertEqual(vars['SO'], vars['EXT_SUFFIX'])
def test_suite():
diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py
index 8532369..f53eb26 100644
--- a/Lib/distutils/tests/test_upload.py
+++ b/Lib/distutils/tests/test_upload.py
@@ -124,11 +124,12 @@ class uploadTestCase(PyPIRCCommandTestCase):
# what did we send ?
headers = dict(self.last_open.req.headers)
self.assertEqual(headers['Content-length'], '2087')
- self.assertTrue(headers['Content-type'].startswith('multipart/form-data'))
+ content_type = headers['Content-type']
+ self.assertTrue(content_type.startswith('multipart/form-data'))
self.assertEqual(self.last_open.req.get_method(), 'POST')
- self.assertEqual(self.last_open.req.get_full_url(),
- 'https://pypi.python.org/pypi')
- self.assertIn(b'xxx', self.last_open.req.data)
+ expected_url = 'https://pypi.python.org/pypi'
+ self.assertEqual(self.last_open.req.get_full_url(), expected_url)
+ self.assertTrue(b'xxx' in self.last_open.req.data)
# The PyPI response body was echoed
results = self.get_logs(INFO)
diff --git a/Lib/distutils/tests/test_util.py b/Lib/distutils/tests/test_util.py
index a1abf8f..4e9d79b 100644
--- a/Lib/distutils/tests/test_util.py
+++ b/Lib/distutils/tests/test_util.py
@@ -237,7 +237,7 @@ class UtilTestCase(support.EnvironGuard, unittest.TestCase):
self.assertRaises(DistutilsPlatformError,
change_root, 'c:\\root', 'its\\here')
- # XXX platforms to be covered: os2, mac
+ # XXX platforms to be covered: mac
def test_check_environ(self):
util._environ_checked = 0
diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py
index b558957..5adcac5 100644
--- a/Lib/distutils/util.py
+++ b/Lib/distutils/util.py
@@ -6,7 +6,7 @@ one of the other *util.py modules.
import os
import re
-import imp
+import importlib.util
import sys
import string
from distutils.errors import DistutilsPlatformError
@@ -154,12 +154,6 @@ def change_root (new_root, pathname):
path = path[1:]
return os.path.join(new_root, path)
- elif os.name == 'os2':
- (drive, path) = os.path.splitdrive(pathname)
- if path[0] == os.sep:
- path = path[1:]
- return os.path.join(new_root, path)
-
else:
raise DistutilsPlatformError("nothing known about platform '%s'" % os.name)
@@ -444,9 +438,10 @@ byte_compile(files, optimize=%r, force=%r,
# cfile - byte-compiled file
# dfile - purported source filename (same as 'file' by default)
if optimize >= 0:
- cfile = imp.cache_from_source(file, debug_override=not optimize)
+ cfile = importlib.util.cache_from_source(
+ file, debug_override=not optimize)
else:
- cfile = imp.cache_from_source(file)
+ cfile = importlib.util.cache_from_source(file)
dfile = file
if prefix:
if file[:len(prefix)] != prefix:
diff --git a/Lib/doctest.py b/Lib/doctest.py
index 3f0d9d9..d212ad6 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -62,6 +62,7 @@ __all__ = [
'REPORT_NDIFF',
'REPORT_ONLY_FIRST_FAILURE',
'REPORTING_FLAGS',
+ 'FAIL_FAST',
# 1. Utility Functions
# 2. Example & DocTest
'Example',
@@ -92,6 +93,7 @@ __all__ = [
]
import __future__
+import argparse
import difflib
import inspect
import linecache
@@ -150,11 +152,13 @@ REPORT_UDIFF = register_optionflag('REPORT_UDIFF')
REPORT_CDIFF = register_optionflag('REPORT_CDIFF')
REPORT_NDIFF = register_optionflag('REPORT_NDIFF')
REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE')
+FAIL_FAST = register_optionflag('FAIL_FAST')
REPORTING_FLAGS = (REPORT_UDIFF |
REPORT_CDIFF |
REPORT_NDIFF |
- REPORT_ONLY_FIRST_FAILURE)
+ REPORT_ONLY_FIRST_FAILURE |
+ FAIL_FAST)
# Special string markers for use in `want` strings:
BLANKLINE_MARKER = '<BLANKLINE>'
@@ -212,7 +216,7 @@ def _load_testfile(filename, package, module_relative, encoding):
if module_relative:
package = _normalize_module(package, 3)
filename = _module_relative_path(package, filename)
- if hasattr(package, '__loader__'):
+ if getattr(package, '__loader__', None) is not None:
if hasattr(package.__loader__, 'get_data'):
file_contents = package.__loader__.get_data(filename)
file_contents = file_contents.decode(encoding)
@@ -940,6 +944,14 @@ class DocTestFinder:
return module is inspect.getmodule(object)
elif inspect.isfunction(object):
return module.__dict__ is object.__globals__
+ elif inspect.ismethoddescriptor(object):
+ if hasattr(object, '__objclass__'):
+ obj_mod = object.__objclass__.__module__
+ elif hasattr(object, '__module__'):
+ obj_mod = object.__module__
+ else:
+ return True # [XX] no easy way to tell otherwise
+ return module.__name__ == obj_mod
elif inspect.isclass(object):
return module.__name__ == object.__module__
elif hasattr(object, '__module__'):
@@ -972,7 +984,7 @@ class DocTestFinder:
for valname, val in obj.__dict__.items():
valname = '%s.%s' % (name, valname)
# Recurse to functions & classes.
- if ((inspect.isfunction(val) or inspect.isclass(val)) and
+ if ((inspect.isroutine(val) or inspect.isclass(val)) and
self._from_module(module, val)):
self._find(tests, val, valname, module, source_lines,
globs, seen)
@@ -984,9 +996,8 @@ class DocTestFinder:
raise ValueError("DocTestFinder.find: __test__ keys "
"must be strings: %r" %
(type(valname),))
- if not (inspect.isfunction(val) or inspect.isclass(val) or
- inspect.ismethod(val) or inspect.ismodule(val) or
- isinstance(val, str)):
+ if not (inspect.isroutine(val) or inspect.isclass(val) or
+ inspect.ismodule(val) or isinstance(val, str)):
raise ValueError("DocTestFinder.find: __test__ values "
"must be strings, functions, methods, "
"classes, or modules: %r" %
@@ -1005,7 +1016,7 @@ class DocTestFinder:
val = getattr(obj, valname).__func__
# Recurse to methods, properties, and nested classes.
- if ((inspect.isfunction(val) or inspect.isclass(val) or
+ if ((inspect.isroutine(val) or inspect.isclass(val) or
isinstance(val, property)) and
self._from_module(module, val)):
valname = '%s.%s' % (name, valname)
@@ -1367,6 +1378,9 @@ class DocTestRunner:
else:
assert False, ("unknown outcome", outcome)
+ if failures and self.optionflags & FAIL_FAST:
+ break
+
# Restore the option flags (in case they were modified)
self.optionflags = original_optionflags
@@ -2308,6 +2322,12 @@ class SkipDocTestCase(DocTestCase):
__str__ = shortDescription
+class _DocTestSuite(unittest.TestSuite):
+
+ def _removeTestAtIndex(self, index):
+ pass
+
+
def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
**options):
"""
@@ -2353,7 +2373,7 @@ def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
if not tests and sys.flags.optimize >=2:
# Skip doctests when running with -O2
- suite = unittest.TestSuite()
+ suite = _DocTestSuite()
suite.addTest(SkipDocTestCase(module))
return suite
elif not tests:
@@ -2367,7 +2387,7 @@ def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None,
raise ValueError(module, "has no docstrings")
tests.sort()
- suite = unittest.TestSuite()
+ suite = _DocTestSuite()
for test in tests:
if len(test.examples) == 0:
@@ -2477,7 +2497,7 @@ def DocFileSuite(*paths, **kw):
encoding
An encoding that will be used to convert the files to unicode.
"""
- suite = unittest.TestSuite()
+ suite = _DocTestSuite()
# We do this here so that _normalize_module is called at the right
# level. If it were called in DocFileTest, then this function
@@ -2727,13 +2747,30 @@ __test__ = {"_TestClass": _TestClass,
def _test():
- testfiles = [arg for arg in sys.argv[1:] if arg and arg[0] != '-']
- if not testfiles:
- name = os.path.basename(sys.argv[0])
- if '__loader__' in globals(): # python -m
- name, _ = os.path.splitext(name)
- print("usage: {0} [-v] file ...".format(name))
- return 2
+ parser = argparse.ArgumentParser(description="doctest runner")
+ parser.add_argument('-v', '--verbose', action='store_true', default=False,
+ help='print very verbose output for all tests')
+ parser.add_argument('-o', '--option', action='append',
+ choices=OPTIONFLAGS_BY_NAME.keys(), default=[],
+ help=('specify a doctest option flag to apply'
+ ' to the test run; may be specified more'
+ ' than once to apply multiple options'))
+ parser.add_argument('-f', '--fail-fast', action='store_true',
+ help=('stop running tests after first failure (this'
+ ' is a shorthand for -o FAIL_FAST, and is'
+ ' in addition to any other -o options)'))
+ parser.add_argument('file', nargs='+',
+ help='file containing the tests to run')
+ args = parser.parse_args()
+ testfiles = args.file
+ # Verbose used to be handled by the "inspect argv" magic in DocTestRunner,
+ # but since we are using argparse we are passing it manually now.
+ verbose = args.verbose
+ options = 0
+ for option in args.option:
+ options |= OPTIONFLAGS_BY_NAME[option]
+ if args.fail_fast:
+ options |= FAIL_FAST
for filename in testfiles:
if filename.endswith(".py"):
# It is a module -- insert its dir into sys.path and try to
@@ -2743,9 +2780,10 @@ def _test():
sys.path.insert(0, dirname)
m = __import__(filename[:-3])
del sys.path[0]
- failures, _ = testmod(m)
+ failures, _ = testmod(m, verbose=verbose, optionflags=options)
else:
- failures, _ = testfile(filename, module_relative=False)
+ failures, _ = testfile(filename, module_relative=False,
+ verbose=verbose, optionflags=options)
if failures:
return 1
return 0
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 0369e01..3dc5502 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -70,7 +70,7 @@ XXX: provide complete list of token types.
import re
import urllib # For urllib.parse.unquote
from string import hexdigits
-from collections import namedtuple, OrderedDict
+from collections import OrderedDict
from email import _encoded_words as _ew
from email import errors
from email import utils
@@ -368,8 +368,7 @@ class TokenList(list):
yield (indent + ' !! invalid element in token '
'list: {!r}'.format(token))
else:
- for line in token._pp(indent+' '):
- yield line
+ yield from token._pp(indent+' ')
if self.defects:
extra = ' Defects: {}'.format(self.defects)
else:
@@ -1315,24 +1314,22 @@ RouteComponentMarker = ValueTerminal('@', 'route-component-marker')
# Parser
#
-"""Parse strings according to RFC822/2047/2822/5322 rules.
-
-This is a stateless parser. Each get_XXX function accepts a string and
-returns either a Terminal or a TokenList representing the RFC object named
-by the method and a string containing the remaining unparsed characters
-from the input. Thus a parser method consumes the next syntactic construct
-of a given type and returns a token representing the construct plus the
-unparsed remainder of the input string.
-
-For example, if the first element of a structured header is a 'phrase',
-then:
-
- phrase, value = get_phrase(value)
-
-returns the complete phrase from the start of the string value, plus any
-characters left in the string after the phrase is removed.
-
-"""
+# Parse strings according to RFC822/2047/2822/5322 rules.
+#
+# This is a stateless parser. Each get_XXX function accepts a string and
+# returns either a Terminal or a TokenList representing the RFC object named
+# by the method and a string containing the remaining unparsed characters
+# from the input. Thus a parser method consumes the next syntactic construct
+# of a given type and returns a token representing the construct plus the
+# unparsed remainder of the input string.
+#
+# For example, if the first element of a structured header is a 'phrase',
+# then:
+#
+# phrase, value = get_phrase(value)
+#
+# returns the complete phrase from the start of the string value, plus any
+# characters left in the string after the phrase is removed.
_wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split
_non_atom_end_matcher = re.compile(r"[^{}]+".format(
diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py
new file mode 100644
index 0000000..d363652
--- /dev/null
+++ b/Lib/email/contentmanager.py
@@ -0,0 +1,249 @@
+import binascii
+import email.charset
+import email.message
+import email.errors
+from email import quoprimime
+
+class ContentManager:
+
+ def __init__(self):
+ self.get_handlers = {}
+ self.set_handlers = {}
+
+ def add_get_handler(self, key, handler):
+ self.get_handlers[key] = handler
+
+ def get_content(self, msg, *args, **kw):
+ content_type = msg.get_content_type()
+ if content_type in self.get_handlers:
+ return self.get_handlers[content_type](msg, *args, **kw)
+ maintype = msg.get_content_maintype()
+ if maintype in self.get_handlers:
+ return self.get_handlers[maintype](msg, *args, **kw)
+ if '' in self.get_handlers:
+ return self.get_handlers[''](msg, *args, **kw)
+ raise KeyError(content_type)
+
+ def add_set_handler(self, typekey, handler):
+ self.set_handlers[typekey] = handler
+
+ def set_content(self, msg, obj, *args, **kw):
+ if msg.get_content_maintype() == 'multipart':
+ # XXX: is this error a good idea or not? We can remove it later,
+ # but we can't add it later, so do it for now.
+ raise TypeError("set_content not valid on multipart")
+ handler = self._find_set_handler(msg, obj)
+ msg.clear_content()
+ handler(msg, obj, *args, **kw)
+
+ def _find_set_handler(self, msg, obj):
+ full_path_for_error = None
+ for typ in type(obj).__mro__:
+ if typ in self.set_handlers:
+ return self.set_handlers[typ]
+ qname = typ.__qualname__
+ modname = getattr(typ, '__module__', '')
+ full_path = '.'.join((modname, qname)) if modname else qname
+ if full_path_for_error is None:
+ full_path_for_error = full_path
+ if full_path in self.set_handlers:
+ return self.set_handlers[full_path]
+ if qname in self.set_handlers:
+ return self.set_handlers[qname]
+ name = typ.__name__
+ if name in self.set_handlers:
+ return self.set_handlers[name]
+ if None in self.set_handlers:
+ return self.set_handlers[None]
+ raise KeyError(full_path_for_error)
+
+
+raw_data_manager = ContentManager()
+
+
+def get_text_content(msg, errors='replace'):
+ content = msg.get_payload(decode=True)
+ charset = msg.get_param('charset', 'ASCII')
+ return content.decode(charset, errors=errors)
+raw_data_manager.add_get_handler('text', get_text_content)
+
+
+def get_non_text_content(msg):
+ return msg.get_payload(decode=True)
+for maintype in 'audio image video application'.split():
+ raw_data_manager.add_get_handler(maintype, get_non_text_content)
+
+
+def get_message_content(msg):
+ return msg.get_payload(0)
+for subtype in 'rfc822 external-body'.split():
+ raw_data_manager.add_get_handler('message/'+subtype, get_message_content)
+
+
+def get_and_fixup_unknown_message_content(msg):
+ # If we don't understand a message subtype, we are supposed to treat it as
+ # if it were application/octet-stream, per
+ # tools.ietf.org/html/rfc2046#section-5.2.4. Feedparser doesn't do that,
+ # so do our best to fix things up. Note that it is *not* appropriate to
+ # model message/partial content as Message objects, so they are handled
+ # here as well. (How to reassemble them is out of scope for this comment :)
+ return bytes(msg.get_payload(0))
+raw_data_manager.add_get_handler('message',
+ get_and_fixup_unknown_message_content)
+
+
+def _prepare_set(msg, maintype, subtype, headers):
+ msg['Content-Type'] = '/'.join((maintype, subtype))
+ if headers:
+ if not hasattr(headers[0], 'name'):
+ mp = msg.policy
+ headers = [mp.header_factory(*mp.header_source_parse([header]))
+ for header in headers]
+ try:
+ for header in headers:
+ if header.defects:
+ raise header.defects[0]
+ msg[header.name] = header
+ except email.errors.HeaderDefect as exc:
+ raise ValueError("Invalid header: {}".format(
+ header.fold(policy=msg.policy))) from exc
+
+
+def _finalize_set(msg, disposition, filename, cid, params):
+ if disposition is None and filename is not None:
+ disposition = 'attachment'
+ if disposition is not None:
+ msg['Content-Disposition'] = disposition
+ if filename is not None:
+ msg.set_param('filename',
+ filename,
+ header='Content-Disposition',
+ replace=True)
+ if cid is not None:
+ msg['Content-ID'] = cid
+ if params is not None:
+ for key, value in params.items():
+ msg.set_param(key, value)
+
+
+# XXX: This is a cleaned-up version of base64mime.body_encode. It would
+# be nice to drop both this and quoprimime.body_encode in favor of
+# enhanced binascii routines that accepted a max_line_length parameter.
+def _encode_base64(data, max_line_length):
+ encoded_lines = []
+ unencoded_bytes_per_line = max_line_length * 3 // 4
+ for i in range(0, len(data), unencoded_bytes_per_line):
+ thisline = data[i:i+unencoded_bytes_per_line]
+ encoded_lines.append(binascii.b2a_base64(thisline).decode('ascii'))
+ return ''.join(encoded_lines)
+
+
+def _encode_text(string, charset, cte, policy):
+ lines = string.encode(charset).splitlines()
+ linesep = policy.linesep.encode('ascii')
+ def embeded_body(lines): return linesep.join(lines) + linesep
+ def normal_body(lines): return b'\n'.join(lines) + b'\n'
+ if cte==None:
+ # Use heuristics to decide on the "best" encoding.
+ try:
+ return '7bit', normal_body(lines).decode('ascii')
+ except UnicodeDecodeError:
+ pass
+ if (policy.cte_type == '8bit' and
+ max(len(x) for x in lines) <= policy.max_line_length):
+ return '8bit', normal_body(lines).decode('ascii', 'surrogateescape')
+ sniff = embeded_body(lines[:10])
+ sniff_qp = quoprimime.body_encode(sniff.decode('latin-1'),
+ policy.max_line_length)
+ sniff_base64 = binascii.b2a_base64(sniff)
+ # This is a little unfair to qp; it includes lineseps, base64 doesn't.
+ if len(sniff_qp) > len(sniff_base64):
+ cte = 'base64'
+ else:
+ cte = 'quoted-printable'
+ if len(lines) <= 10:
+ return cte, sniff_qp
+ if cte == '7bit':
+ data = normal_body(lines).decode('ascii')
+ elif cte == '8bit':
+ data = normal_body(lines).decode('ascii', 'surrogateescape')
+ elif cte == 'quoted-printable':
+ data = quoprimime.body_encode(normal_body(lines).decode('latin-1'),
+ policy.max_line_length)
+ elif cte == 'base64':
+ data = _encode_base64(embeded_body(lines), policy.max_line_length)
+ else:
+ raise ValueError("Unknown content transfer encoding {}".format(cte))
+ return cte, data
+
+
+def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None,
+ disposition=None, filename=None, cid=None,
+ params=None, headers=None):
+ _prepare_set(msg, 'text', subtype, headers)
+ cte, payload = _encode_text(string, charset, cte, msg.policy)
+ msg.set_payload(payload)
+ msg.set_param('charset',
+ email.charset.ALIASES.get(charset, charset),
+ replace=True)
+ msg['Content-Transfer-Encoding'] = cte
+ _finalize_set(msg, disposition, filename, cid, params)
+raw_data_manager.add_set_handler(str, set_text_content)
+
+
+def set_message_content(msg, message, subtype="rfc822", cte=None,
+ disposition=None, filename=None, cid=None,
+ params=None, headers=None):
+ if subtype == 'partial':
+ raise ValueError("message/partial is not supported for Message objects")
+ if subtype == 'rfc822':
+ if cte not in (None, '7bit', '8bit', 'binary'):
+ # http://tools.ietf.org/html/rfc2046#section-5.2.1 mandate.
+ raise ValueError(
+ "message/rfc822 parts do not support cte={}".format(cte))
+ # 8bit will get coerced on serialization if policy.cte_type='7bit'. We
+ # may end up claiming 8bit when it isn't needed, but the only negative
+ # result of that should be a gateway that needs to coerce to 7bit
+ # having to look through the whole embedded message to discover whether
+ # or not it actually has to do anything.
+ cte = '8bit' if cte is None else cte
+ elif subtype == 'external-body':
+ if cte not in (None, '7bit'):
+ # http://tools.ietf.org/html/rfc2046#section-5.2.3 mandate.
+ raise ValueError(
+ "message/external-body parts do not support cte={}".format(cte))
+ cte = '7bit'
+ elif cte is None:
+ # http://tools.ietf.org/html/rfc2046#section-5.2.4 says all future
+ # subtypes should be restricted to 7bit, so assume that.
+ cte = '7bit'
+ _prepare_set(msg, 'message', subtype, headers)
+ msg.set_payload([message])
+ msg['Content-Transfer-Encoding'] = cte
+ _finalize_set(msg, disposition, filename, cid, params)
+raw_data_manager.add_set_handler(email.message.Message, set_message_content)
+
+
+def set_bytes_content(msg, data, maintype, subtype, cte='base64',
+ disposition=None, filename=None, cid=None,
+ params=None, headers=None):
+ _prepare_set(msg, maintype, subtype, headers)
+ if cte == 'base64':
+ data = _encode_base64(data, max_line_length=msg.policy.max_line_length)
+ elif cte == 'quoted-printable':
+ # XXX: quoprimime.body_encode won't encode newline characters in data,
+ # so we can't use it. This means max_line_length is ignored. Another
+ # bug to fix later. (Note: encoders.quopri is broken on line ends.)
+ data = binascii.b2a_qp(data, istext=False, header=False, quotetabs=True)
+ data = data.decode('ascii')
+ elif cte == '7bit':
+ # Make sure it really is only ASCII. The early warning here seems
+ # worth the overhead...if you care write your own content manager :).
+ data.encode('ascii')
+ elif cte in ('8bit', 'binary'):
+ data = data.decode('ascii', 'surrogateescape')
+ msg.set_payload(data)
+ msg['Content-Transfer-Encoding'] = cte
+ _finalize_set(msg, disposition, filename, cid, params)
+for typ in (bytes, bytearray, memoryview):
+ raw_data_manager.add_set_handler(typ, set_bytes_content)
diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py
index f9657f0..0a66acb 100644
--- a/Lib/email/encoders.py
+++ b/Lib/email/encoders.py
@@ -54,21 +54,12 @@ def encode_7or8bit(msg):
# There's no payload. For backwards compatibility we use 7bit
msg['Content-Transfer-Encoding'] = '7bit'
return
- # We play a trick to make this go fast. If encoding/decode to ASCII
- # succeeds, we know the data must be 7bit, otherwise treat it as 8bit.
+ # We play a trick to make this go fast. If decoding from ASCII succeeds,
+ # we know the data must be 7bit, otherwise treat it as 8bit.
try:
- if isinstance(orig, str):
- orig.encode('ascii')
- else:
- orig.decode('ascii')
+ orig.decode('ascii')
except UnicodeError:
- charset = msg.get_charset()
- output_cset = charset and charset.output_charset
- # iso-2022-* is non-ASCII but encodes to a 7-bit representation
- if output_cset and output_cset.lower().startswith('iso-2022-'):
- msg['Content-Transfer-Encoding'] = '7bit'
- else:
- msg['Content-Transfer-Encoding'] = '8bit'
+ msg['Content-Transfer-Encoding'] = '8bit'
else:
msg['Content-Transfer-Encoding'] = '7bit'
diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index ea41e95..6cf9b91 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -98,24 +98,15 @@ class BufferedSubFile(object):
"""Push some new data into this object."""
# Handle any previous leftovers
data, self._partial = self._partial + data, ''
- # Crack into lines, but preserve the newlines on the end of each
- parts = NLCRE_crack.split(data)
- # The *ahem* interesting behaviour of re.split when supplied grouping
- # parentheses is that the last element of the resulting list is the
- # data after the final RE. In the case of a NL/CR terminated string,
- # this is the empty string.
- self._partial = parts.pop()
- #GAN 29Mar09 bugs 1555570, 1721862 Confusion at 8K boundary ending with \r:
- # is there a \n to follow later?
- if not self._partial and parts and parts[-1].endswith('\r'):
- self._partial = parts.pop(-2)+parts.pop()
- # parts is a list of strings, alternating between the line contents
- # and the eol character(s). Gather up a list of lines after
- # re-attaching the newlines.
- lines = []
- for i in range(len(parts) // 2):
- lines.append(parts[i*2] + parts[i*2+1])
- self.pushlines(lines)
+ # Crack into lines, but preserve the linesep characters on the end of each
+ parts = data.splitlines(True)
+ # If the last element of the list does not end in a newline, then treat
+ # it as a partial line. We only check for '\n' here because a line
+ # ending with '\r' might be a line that was split in the middle of a
+ # '\r\n' sequence (see bugs 1555570 and 1721862).
+ if parts and not parts[-1].endswith('\n'):
+ self._partial = parts.pop()
+ self.pushlines(parts)
def pushlines(self, lines):
# Reverse and insert at the front of the lines.
@@ -135,7 +126,7 @@ class BufferedSubFile(object):
class FeedParser:
"""A feed-style parser of email."""
- def __init__(self, _factory=message.Message, *, policy=compat32):
+ def __init__(self, _factory=None, *, policy=compat32):
"""_factory is called with no arguments to create a new message obj
The policy keyword specifies a policy object that controls a number of
@@ -143,14 +134,23 @@ class FeedParser:
backward compatibility.
"""
- self._factory = _factory
self.policy = policy
- try:
- _factory(policy=self.policy)
- self._factory_kwds = lambda: {'policy': self.policy}
- except TypeError:
- # Assume this is an old-style factory
- self._factory_kwds = lambda: {}
+ self._factory_kwds = lambda: {'policy': self.policy}
+ if _factory is None:
+ # What this should be:
+ #self._factory = policy.default_message_factory
+ # but, because we are post 3.4 feature freeze, fix with temp hack:
+ if self.policy is compat32:
+ self._factory = message.Message
+ else:
+ self._factory = message.EmailMessage
+ else:
+ self._factory = _factory
+ try:
+ _factory(policy=self.policy)
+ except TypeError:
+ # Assume this is an old-style factory
+ self._factory_kwds = lambda: {}
self._input = BufferedSubFile()
self._msgstack = []
self._parse = self._parsegen().__next__
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index e4a86d4..1535210 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -10,14 +10,10 @@ import re
import sys
import time
import random
-import warnings
from copy import deepcopy
from io import StringIO, BytesIO
-from email._policybase import compat32
-from email.header import Header
from email.utils import _has_surrogates
-import email.charset as _charset
UNDERSCORE = '_'
NL = '\n' # XXX: no longer used by the code below.
diff --git a/Lib/email/header.py b/Lib/email/header.py
index 5bd0638..9c89589 100644
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -100,7 +100,6 @@ def decode_header(header):
words.append((encoded, encoding, charset))
# Now loop over words and remove words that consist of whitespace
# between two encoded strings.
- import sys
droplist = []
for n, w in enumerate(words):
if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace():
@@ -362,7 +361,6 @@ class Header:
for string, charset in self._chunks:
if hasspace is not None:
hasspace = string and self._nonctext(string[0])
- import sys
if lastcs not in (None, 'us-ascii'):
if not hasspace or charset not in (None, 'us-ascii'):
formatter.add_transition()
diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py
index 3adc4a0..b5502ee 100644
--- a/Lib/email/iterators.py
+++ b/Lib/email/iterators.py
@@ -26,8 +26,7 @@ def walk(self):
yield self
if self.is_multipart():
for subpart in self.get_payload():
- for subsubpart in subpart.walk():
- yield subsubpart
+ yield from subpart.walk()
@@ -40,8 +39,7 @@ def body_line_iterator(msg, decode=False):
for subpart in msg.walk():
payload = subpart.get_payload(decode=decode)
if isinstance(payload, str):
- for line in StringIO(payload):
- yield line
+ yield from StringIO(payload)
def typed_subpart_iterator(msg, maintype='text', subtype=None):
diff --git a/Lib/email/message.py b/Lib/email/message.py
index afe350c..aa46deb 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -8,8 +8,7 @@ __all__ = ['Message']
import re
import uu
-import base64
-import binascii
+import quopri
from io import BytesIO, StringIO
# Intrapackage imports
@@ -132,22 +131,50 @@ class Message:
def __str__(self):
"""Return the entire formatted message as a string.
- This includes the headers, body, and envelope header.
"""
return self.as_string()
- def as_string(self, unixfrom=False, maxheaderlen=0):
+ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None):
"""Return the entire formatted message as a string.
- Optional `unixfrom' when True, means include the Unix From_ envelope
- header.
- This is a convenience method and may not generate the message exactly
- as you intend. For more flexibility, use the flatten() method of a
- Generator instance.
+ Optional 'unixfrom', when true, means include the Unix From_ envelope
+ header. For backward compatibility reasons, if maxheaderlen is
+ not specified it defaults to 0, so you must override it explicitly
+ if you want a different maxheaderlen. 'policy' is passed to the
+ Generator instance used to serialize the mesasge; if it is not
+ specified the policy associated with the message instance is used.
+
+ If the message object contains binary data that is not encoded
+ according to RFC standards, the non-compliant data will be replaced by
+ unicode "unknown character" code points.
"""
from email.generator import Generator
+ policy = self.policy if policy is None else policy
fp = StringIO()
- g = Generator(fp, mangle_from_=False, maxheaderlen=maxheaderlen)
+ g = Generator(fp,
+ mangle_from_=False,
+ maxheaderlen=maxheaderlen,
+ policy=policy)
+ g.flatten(self, unixfrom=unixfrom)
+ return fp.getvalue()
+
+ def __bytes__(self):
+ """Return the entire formatted message as a bytes object.
+ """
+ return self.as_bytes()
+
+ def as_bytes(self, unixfrom=False, policy=None):
+ """Return the entire formatted message as a bytes object.
+
+ Optional 'unixfrom', when true, means include the Unix From_ envelope
+ header. 'policy' is passed to the BytesGenerator instance used to
+ serialize the message; if not specified the policy associated with
+ the message instance is used.
+ """
+ from email.generator import BytesGenerator
+ policy = self.policy if policy is None else policy
+ fp = BytesIO()
+ g = BytesGenerator(fp, mangle_from_=False, policy=policy)
g.flatten(self, unixfrom=unixfrom)
return fp.getvalue()
@@ -177,7 +204,11 @@ class Message:
if self._payload is None:
self._payload = [payload]
else:
- self._payload.append(payload)
+ try:
+ self._payload.append(payload)
+ except AttributeError:
+ raise TypeError("Attach is not valid on a message with a"
+ " non-multipart payload")
def get_payload(self, i=None, decode=False):
"""Return a reference to the payload.
@@ -248,7 +279,7 @@ class Message:
if not decode:
return payload
if cte == 'quoted-printable':
- return utils._qdecode(bpayload)
+ return quopri.decodestring(bpayload)
elif cte == 'base64':
# XXX: this is a bit of a hack; decode_b should probably be factored
# out somewhere, but I haven't figured out where yet.
@@ -668,7 +699,7 @@ class Message:
return failobj
def set_param(self, param, value, header='Content-Type', requote=True,
- charset=None, language=''):
+ charset=None, language='', replace=False):
"""Set a parameter in the Content-Type header.
If the parameter already exists in the header, its value will be
@@ -712,8 +743,11 @@ class Message:
else:
ctype = SEMISPACE.join([ctype, append_param])
if ctype != self.get(header):
- del self[header]
- self[header] = ctype
+ if replace:
+ self.replace_header(header, ctype)
+ else:
+ del self[header]
+ self[header] = ctype
def del_param(self, param, header='content-type', requote=True):
"""Remove the given parameter completely from the Content-Type header.
@@ -894,3 +928,208 @@ class Message:
# I.e. def walk(self): ...
from email.iterators import walk
+
+
+class MIMEPart(Message):
+
+ def __init__(self, policy=None):
+ if policy is None:
+ from email.policy import default
+ policy = default
+ Message.__init__(self, policy)
+
+ @property
+ def is_attachment(self):
+ c_d = self.get('content-disposition')
+ if c_d is None:
+ return False
+ return c_d.lower() == 'attachment'
+
+ def _find_body(self, part, preferencelist):
+ if part.is_attachment:
+ return
+ maintype, subtype = part.get_content_type().split('/')
+ if maintype == 'text':
+ if subtype in preferencelist:
+ yield (preferencelist.index(subtype), part)
+ return
+ if maintype != 'multipart':
+ return
+ if subtype != 'related':
+ for subpart in part.iter_parts():
+ yield from self._find_body(subpart, preferencelist)
+ return
+ if 'related' in preferencelist:
+ yield (preferencelist.index('related'), part)
+ candidate = None
+ start = part.get_param('start')
+ if start:
+ for subpart in part.iter_parts():
+ if subpart['content-id'] == start:
+ candidate = subpart
+ break
+ if candidate is None:
+ subparts = part.get_payload()
+ candidate = subparts[0] if subparts else None
+ if candidate is not None:
+ yield from self._find_body(candidate, preferencelist)
+
+ def get_body(self, preferencelist=('related', 'html', 'plain')):
+ """Return best candidate mime part for display as 'body' of message.
+
+ Do a depth first search, starting with self, looking for the first part
+ matching each of the items in preferencelist, and return the part
+ corresponding to the first item that has a match, or None if no items
+ have a match. If 'related' is not included in preferencelist, consider
+ the root part of any multipart/related encountered as a candidate
+ match. Ignore parts with 'Content-Disposition: attachment'.
+ """
+ best_prio = len(preferencelist)
+ body = None
+ for prio, part in self._find_body(self, preferencelist):
+ if prio < best_prio:
+ best_prio = prio
+ body = part
+ if prio == 0:
+ break
+ return body
+
+ _body_types = {('text', 'plain'),
+ ('text', 'html'),
+ ('multipart', 'related'),
+ ('multipart', 'alternative')}
+ def iter_attachments(self):
+ """Return an iterator over the non-main parts of a multipart.
+
+ Skip the first of each occurrence of text/plain, text/html,
+ multipart/related, or multipart/alternative in the multipart (unless
+ they have a 'Content-Disposition: attachment' header) and include all
+ remaining subparts in the returned iterator. When applied to a
+ multipart/related, return all parts except the root part. Return an
+ empty iterator when applied to a multipart/alternative or a
+ non-multipart.
+ """
+ maintype, subtype = self.get_content_type().split('/')
+ if maintype != 'multipart' or subtype == 'alternative':
+ return
+ parts = self.get_payload()
+ if maintype == 'multipart' and subtype == 'related':
+ # For related, we treat everything but the root as an attachment.
+ # The root may be indicated by 'start'; if there's no start or we
+ # can't find the named start, treat the first subpart as the root.
+ start = self.get_param('start')
+ if start:
+ found = False
+ attachments = []
+ for part in parts:
+ if part.get('content-id') == start:
+ found = True
+ else:
+ attachments.append(part)
+ if found:
+ yield from attachments
+ return
+ parts.pop(0)
+ yield from parts
+ return
+ # Otherwise we more or less invert the remaining logic in get_body.
+ # This only really works in edge cases (ex: non-text relateds or
+ # alternatives) if the sending agent sets content-disposition.
+ seen = [] # Only skip the first example of each candidate type.
+ for part in parts:
+ maintype, subtype = part.get_content_type().split('/')
+ if ((maintype, subtype) in self._body_types and
+ not part.is_attachment and subtype not in seen):
+ seen.append(subtype)
+ continue
+ yield part
+
+ def iter_parts(self):
+ """Return an iterator over all immediate subparts of a multipart.
+
+ Return an empty iterator for a non-multipart.
+ """
+ if self.get_content_maintype() == 'multipart':
+ yield from self.get_payload()
+
+ def get_content(self, *args, content_manager=None, **kw):
+ if content_manager is None:
+ content_manager = self.policy.content_manager
+ return content_manager.get_content(self, *args, **kw)
+
+ def set_content(self, *args, content_manager=None, **kw):
+ if content_manager is None:
+ content_manager = self.policy.content_manager
+ content_manager.set_content(self, *args, **kw)
+
+ def _make_multipart(self, subtype, disallowed_subtypes, boundary):
+ if self.get_content_maintype() == 'multipart':
+ existing_subtype = self.get_content_subtype()
+ disallowed_subtypes = disallowed_subtypes + (subtype,)
+ if existing_subtype in disallowed_subtypes:
+ raise ValueError("Cannot convert {} to {}".format(
+ existing_subtype, subtype))
+ keep_headers = []
+ part_headers = []
+ for name, value in self._headers:
+ if name.lower().startswith('content-'):
+ part_headers.append((name, value))
+ else:
+ keep_headers.append((name, value))
+ if part_headers:
+ # There is existing content, move it to the first subpart.
+ part = type(self)(policy=self.policy)
+ part._headers = part_headers
+ part._payload = self._payload
+ self._payload = [part]
+ else:
+ self._payload = []
+ self._headers = keep_headers
+ self['Content-Type'] = 'multipart/' + subtype
+ if boundary is not None:
+ self.set_param('boundary', boundary)
+
+ def make_related(self, boundary=None):
+ self._make_multipart('related', ('alternative', 'mixed'), boundary)
+
+ def make_alternative(self, boundary=None):
+ self._make_multipart('alternative', ('mixed',), boundary)
+
+ def make_mixed(self, boundary=None):
+ self._make_multipart('mixed', (), boundary)
+
+ def _add_multipart(self, _subtype, *args, _disp=None, **kw):
+ if (self.get_content_maintype() != 'multipart' or
+ self.get_content_subtype() != _subtype):
+ getattr(self, 'make_' + _subtype)()
+ part = type(self)(policy=self.policy)
+ part.set_content(*args, **kw)
+ if _disp and 'content-disposition' not in part:
+ part['Content-Disposition'] = _disp
+ self.attach(part)
+
+ def add_related(self, *args, **kw):
+ self._add_multipart('related', *args, _disp='inline', **kw)
+
+ def add_alternative(self, *args, **kw):
+ self._add_multipart('alternative', *args, **kw)
+
+ def add_attachment(self, *args, **kw):
+ self._add_multipart('mixed', *args, _disp='attachment', **kw)
+
+ def clear(self):
+ self._headers = []
+ self._payload = None
+
+ def clear_content(self):
+ self._headers = [(n, v) for n, v in self._headers
+ if not n.lower().startswith('content-')]
+ self._payload = None
+
+
+class EmailMessage(MIMEPart):
+
+ def set_content(self, *args, **kw):
+ super().set_content(*args, **kw)
+ if 'MIME-Version' not in self:
+ self['MIME-Version'] = '1.0'
diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py
index 3b5b09f..ec18b85 100644
--- a/Lib/email/mime/text.py
+++ b/Lib/email/mime/text.py
@@ -6,7 +6,6 @@
__all__ = ['MIMEText']
-from email.encoders import encode_7or8bit
from email.mime.nonmultipart import MIMENonMultipart
diff --git a/Lib/email/parser.py b/Lib/email/parser.py
index 752bf35..9f5f95d 100644
--- a/Lib/email/parser.py
+++ b/Lib/email/parser.py
@@ -4,19 +4,18 @@
"""A parser of RFC 2822 and MIME email messages."""
-__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser']
+__all__ = ['Parser', 'HeaderParser', 'BytesParser', 'BytesHeaderParser',
+ 'FeedParser', 'BytesFeedParser']
-import warnings
from io import StringIO, TextIOWrapper
from email.feedparser import FeedParser, BytesFeedParser
-from email.message import Message
from email._policybase import compat32
class Parser:
- def __init__(self, _class=Message, *, policy=compat32):
+ def __init__(self, _class=None, *, policy=compat32):
"""Parser of RFC 2822 and MIME email messages.
Creates an in-memory object tree representing the email message, which
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
index 38e88af..f0b20f4 100644
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -5,6 +5,7 @@ code that adds all the email6 features.
from email._policybase import Policy, Compat32, compat32, _extend_docstrings
from email.utils import _has_surrogates
from email.headerregistry import HeaderRegistry as HeaderRegistry
+from email.contentmanager import raw_data_manager
__all__ = [
'Compat32',
@@ -58,10 +59,22 @@ class EmailPolicy(Policy):
special treatment, while all other fields are
treated as unstructured. This list will be
completed before the extension is marked stable.)
+
+ content_manager -- an object with at least two methods: get_content
+ and set_content. When the get_content or
+ set_content method of a Message object is called,
+ it calls the corresponding method of this object,
+ passing it the message object as its first argument,
+ and any arguments or keywords that were passed to
+ it as additional arguments. The default
+ content_manager is
+ :data:`~email.contentmanager.raw_data_manager`.
+
"""
refold_source = 'long'
header_factory = HeaderRegistry()
+ content_manager = raw_data_manager
def __init__(self, **kw):
# Ensure that each new instance gets a unique header factory
diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py
index 30bf916..c1fe2b4 100644
--- a/Lib/email/quoprimime.py
+++ b/Lib/email/quoprimime.py
@@ -40,7 +40,6 @@ __all__ = [
]
import re
-import io
from string import ascii_letters, digits, hexdigits
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index f76c21e..cacb9b1 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -25,13 +25,10 @@ __all__ = [
import os
import re
import time
-import base64
import random
import socket
import datetime
import urllib.parse
-import warnings
-from io import StringIO
from email._parseaddr import quote
from email._parseaddr import AddressList as _AddressList
@@ -39,10 +36,7 @@ from email._parseaddr import mktime_tz
from email._parseaddr import parsedate, parsedate_tz, _parsedate_tz
-from quopri import decodestring as _qdecode
-
# Intrapackage imports
-from email.encoders import _bencode, _qencode
from email.charset import Charset
COMMASPACE = ', '
@@ -54,17 +48,27 @@ TICK = "'"
specialsre = re.compile(r'[][\\()<>@,:;".]')
escapesre = re.compile(r'[\\"]')
-# How to figure out if we are processing strings that come from a byte
-# source with undecodable characters.
-_has_surrogates = re.compile(
- '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
+def _has_surrogates(s):
+ """Return True if s contains surrogate-escaped binary data."""
+ # This check is based on the fact that unless there are surrogates, utf8
+ # (Python's default encoding) can encode any string. This is the fastest
+ # way to check for surrogates, see issue 11454 for timings.
+ try:
+ s.encode()
+ return False
+ except UnicodeEncodeError:
+ return True
# How to deal with a string containing bytes before handing it to the
# application through the 'normal' interface.
def _sanitize(string):
- # Turn any escaped bytes into unicode 'unknown' char.
- original_bytes = string.encode('ascii', 'surrogateescape')
- return original_bytes.decode('ascii', 'replace')
+ # Turn any escaped bytes into unicode 'unknown' char. If the escaped
+ # bytes happen to be utf-8 they will instead get decoded, even if they
+ # were invalid in the charset the source was supposed to be in. This
+ # seems like it is not a bad thing; a defect was still registered.
+ original_bytes = string.encode('utf-8', 'surrogateescape')
+ return original_bytes.decode('utf-8', 'replace')
+
# Helpers
diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index 235deb5..4cbaade 100644
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -33,9 +33,9 @@ aliases = {
'us' : 'ascii',
'us_ascii' : 'ascii',
- ## base64_codec codec
- #'base64' : 'base64_codec',
- #'base_64' : 'base64_codec',
+ # base64_codec codec
+ 'base64' : 'base64_codec',
+ 'base_64' : 'base64_codec',
# big5 codec
'big5_tw' : 'big5',
@@ -45,8 +45,8 @@ aliases = {
'big5_hkscs' : 'big5hkscs',
'hkscs' : 'big5hkscs',
- ## bz2_codec codec
- #'bz2' : 'bz2_codec',
+ # bz2_codec codec
+ 'bz2' : 'bz2_codec',
# cp037 codec
'037' : 'cp037',
@@ -63,6 +63,12 @@ aliases = {
'csibm1026' : 'cp1026',
'ibm1026' : 'cp1026',
+ # cp1125 codec
+ '1125' : 'cp1125',
+ 'ibm1125' : 'cp1125',
+ 'cp866u' : 'cp1125',
+ 'ruscii' : 'cp1125',
+
# cp1140 codec
'1140' : 'cp1140',
'ibm1140' : 'cp1140',
@@ -103,6 +109,11 @@ aliases = {
'1258' : 'cp1258',
'windows_1258' : 'cp1258',
+ # cp273 codec
+ '273' : 'cp273',
+ 'ibm273' : 'cp273',
+ 'csibm273' : 'cp273',
+
# cp424 codec
'424' : 'cp424',
'csibm424' : 'cp424',
@@ -248,8 +259,8 @@ aliases = {
'cp936' : 'gbk',
'ms936' : 'gbk',
- ## hex_codec codec
- #'hex' : 'hex_codec',
+ # hex_codec codec
+ 'hex' : 'hex_codec',
# hp_roman8 codec
'roman8' : 'hp_roman8',
@@ -450,13 +461,13 @@ aliases = {
'cp154' : 'ptcp154',
'cyrillic_asian' : 'ptcp154',
- ## quopri_codec codec
- #'quopri' : 'quopri_codec',
- #'quoted_printable' : 'quopri_codec',
- #'quotedprintable' : 'quopri_codec',
+ # quopri_codec codec
+ 'quopri' : 'quopri_codec',
+ 'quoted_printable' : 'quopri_codec',
+ 'quotedprintable' : 'quopri_codec',
- ## rot_13 codec
- #'rot13' : 'rot_13',
+ # rot_13 codec
+ 'rot13' : 'rot_13',
# shift_jis codec
'csshiftjis' : 'shift_jis',
@@ -518,12 +529,12 @@ aliases = {
'utf8_ucs2' : 'utf_8',
'utf8_ucs4' : 'utf_8',
- ## uu_codec codec
- #'uu' : 'uu_codec',
+ # uu_codec codec
+ 'uu' : 'uu_codec',
- ## zlib_codec codec
- #'zip' : 'zlib_codec',
- #'zlib' : 'zlib_codec',
+ # zlib_codec codec
+ 'zip' : 'zlib_codec',
+ 'zlib' : 'zlib_codec',
# temporary mac CJK aliases, will be replaced by proper codecs in 3.1
'x_mac_japanese' : 'shift_jis',
diff --git a/Lib/encodings/cp037.py b/Lib/encodings/cp037.py
index bfe2c1e..4edd708 100644
--- a/Lib/encodings/cp037.py
+++ b/Lib/encodings/cp037.py
@@ -301,7 +301,6 @@ decoding_table = (
'\xd9' # 0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
'\xda' # 0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
'\x9f' # 0xFF -> CONTROL
- '\ufffe' ## Widen to UCS2 for optimization
)
### Encoding table
diff --git a/Lib/encodings/cp1125.py b/Lib/encodings/cp1125.py
new file mode 100644
index 0000000..b1fd69d
--- /dev/null
+++ b/Lib/encodings/cp1125.py
@@ -0,0 +1,698 @@
+""" Python Character Mapping Codec for CP1125
+
+"""#"
+
+import codecs
+
+### Codec APIs
+
+class Codec(codecs.Codec):
+
+ def encode(self,input,errors='strict'):
+ return codecs.charmap_encode(input,errors,encoding_map)
+
+ def decode(self,input,errors='strict'):
+ return codecs.charmap_decode(input,errors,decoding_table)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.charmap_encode(input,self.errors,encoding_map)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return codecs.charmap_decode(input,self.errors,decoding_table)[0]
+
+class StreamWriter(Codec,codecs.StreamWriter):
+ pass
+
+class StreamReader(Codec,codecs.StreamReader):
+ pass
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='cp1125',
+ encode=Codec().encode,
+ decode=Codec().decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamreader=StreamReader,
+ streamwriter=StreamWriter,
+ )
+
+### Decoding Map
+
+decoding_map = codecs.make_identity_dict(range(256))
+decoding_map.update({
+ 0x0080: 0x0410, # CYRILLIC CAPITAL LETTER A
+ 0x0081: 0x0411, # CYRILLIC CAPITAL LETTER BE
+ 0x0082: 0x0412, # CYRILLIC CAPITAL LETTER VE
+ 0x0083: 0x0413, # CYRILLIC CAPITAL LETTER GHE
+ 0x0084: 0x0414, # CYRILLIC CAPITAL LETTER DE
+ 0x0085: 0x0415, # CYRILLIC CAPITAL LETTER IE
+ 0x0086: 0x0416, # CYRILLIC CAPITAL LETTER ZHE
+ 0x0087: 0x0417, # CYRILLIC CAPITAL LETTER ZE
+ 0x0088: 0x0418, # CYRILLIC CAPITAL LETTER I
+ 0x0089: 0x0419, # CYRILLIC CAPITAL LETTER SHORT I
+ 0x008a: 0x041a, # CYRILLIC CAPITAL LETTER KA
+ 0x008b: 0x041b, # CYRILLIC CAPITAL LETTER EL
+ 0x008c: 0x041c, # CYRILLIC CAPITAL LETTER EM
+ 0x008d: 0x041d, # CYRILLIC CAPITAL LETTER EN
+ 0x008e: 0x041e, # CYRILLIC CAPITAL LETTER O
+ 0x008f: 0x041f, # CYRILLIC CAPITAL LETTER PE
+ 0x0090: 0x0420, # CYRILLIC CAPITAL LETTER ER
+ 0x0091: 0x0421, # CYRILLIC CAPITAL LETTER ES
+ 0x0092: 0x0422, # CYRILLIC CAPITAL LETTER TE
+ 0x0093: 0x0423, # CYRILLIC CAPITAL LETTER U
+ 0x0094: 0x0424, # CYRILLIC CAPITAL LETTER EF
+ 0x0095: 0x0425, # CYRILLIC CAPITAL LETTER HA
+ 0x0096: 0x0426, # CYRILLIC CAPITAL LETTER TSE
+ 0x0097: 0x0427, # CYRILLIC CAPITAL LETTER CHE
+ 0x0098: 0x0428, # CYRILLIC CAPITAL LETTER SHA
+ 0x0099: 0x0429, # CYRILLIC CAPITAL LETTER SHCHA
+ 0x009a: 0x042a, # CYRILLIC CAPITAL LETTER HARD SIGN
+ 0x009b: 0x042b, # CYRILLIC CAPITAL LETTER YERU
+ 0x009c: 0x042c, # CYRILLIC CAPITAL LETTER SOFT SIGN
+ 0x009d: 0x042d, # CYRILLIC CAPITAL LETTER E
+ 0x009e: 0x042e, # CYRILLIC CAPITAL LETTER YU
+ 0x009f: 0x042f, # CYRILLIC CAPITAL LETTER YA
+ 0x00a0: 0x0430, # CYRILLIC SMALL LETTER A
+ 0x00a1: 0x0431, # CYRILLIC SMALL LETTER BE
+ 0x00a2: 0x0432, # CYRILLIC SMALL LETTER VE
+ 0x00a3: 0x0433, # CYRILLIC SMALL LETTER GHE
+ 0x00a4: 0x0434, # CYRILLIC SMALL LETTER DE
+ 0x00a5: 0x0435, # CYRILLIC SMALL LETTER IE
+ 0x00a6: 0x0436, # CYRILLIC SMALL LETTER ZHE
+ 0x00a7: 0x0437, # CYRILLIC SMALL LETTER ZE
+ 0x00a8: 0x0438, # CYRILLIC SMALL LETTER I
+ 0x00a9: 0x0439, # CYRILLIC SMALL LETTER SHORT I
+ 0x00aa: 0x043a, # CYRILLIC SMALL LETTER KA
+ 0x00ab: 0x043b, # CYRILLIC SMALL LETTER EL
+ 0x00ac: 0x043c, # CYRILLIC SMALL LETTER EM
+ 0x00ad: 0x043d, # CYRILLIC SMALL LETTER EN
+ 0x00ae: 0x043e, # CYRILLIC SMALL LETTER O
+ 0x00af: 0x043f, # CYRILLIC SMALL LETTER PE
+ 0x00b0: 0x2591, # LIGHT SHADE
+ 0x00b1: 0x2592, # MEDIUM SHADE
+ 0x00b2: 0x2593, # DARK SHADE
+ 0x00b3: 0x2502, # BOX DRAWINGS LIGHT VERTICAL
+ 0x00b4: 0x2524, # BOX DRAWINGS LIGHT VERTICAL AND LEFT
+ 0x00b5: 0x2561, # BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE
+ 0x00b6: 0x2562, # BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE
+ 0x00b7: 0x2556, # BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE
+ 0x00b8: 0x2555, # BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE
+ 0x00b9: 0x2563, # BOX DRAWINGS DOUBLE VERTICAL AND LEFT
+ 0x00ba: 0x2551, # BOX DRAWINGS DOUBLE VERTICAL
+ 0x00bb: 0x2557, # BOX DRAWINGS DOUBLE DOWN AND LEFT
+ 0x00bc: 0x255d, # BOX DRAWINGS DOUBLE UP AND LEFT
+ 0x00bd: 0x255c, # BOX DRAWINGS UP DOUBLE AND LEFT SINGLE
+ 0x00be: 0x255b, # BOX DRAWINGS UP SINGLE AND LEFT DOUBLE
+ 0x00bf: 0x2510, # BOX DRAWINGS LIGHT DOWN AND LEFT
+ 0x00c0: 0x2514, # BOX DRAWINGS LIGHT UP AND RIGHT
+ 0x00c1: 0x2534, # BOX DRAWINGS LIGHT UP AND HORIZONTAL
+ 0x00c2: 0x252c, # BOX DRAWINGS LIGHT DOWN AND HORIZONTAL
+ 0x00c3: 0x251c, # BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+ 0x00c4: 0x2500, # BOX DRAWINGS LIGHT HORIZONTAL
+ 0x00c5: 0x253c, # BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL
+ 0x00c6: 0x255e, # BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE
+ 0x00c7: 0x255f, # BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE
+ 0x00c8: 0x255a, # BOX DRAWINGS DOUBLE UP AND RIGHT
+ 0x00c9: 0x2554, # BOX DRAWINGS DOUBLE DOWN AND RIGHT
+ 0x00ca: 0x2569, # BOX DRAWINGS DOUBLE UP AND HORIZONTAL
+ 0x00cb: 0x2566, # BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL
+ 0x00cc: 0x2560, # BOX DRAWINGS DOUBLE VERTICAL AND RIGHT
+ 0x00cd: 0x2550, # BOX DRAWINGS DOUBLE HORIZONTAL
+ 0x00ce: 0x256c, # BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL
+ 0x00cf: 0x2567, # BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE
+ 0x00d0: 0x2568, # BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE
+ 0x00d1: 0x2564, # BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE
+ 0x00d2: 0x2565, # BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE
+ 0x00d3: 0x2559, # BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE
+ 0x00d4: 0x2558, # BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE
+ 0x00d5: 0x2552, # BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE
+ 0x00d6: 0x2553, # BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE
+ 0x00d7: 0x256b, # BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE
+ 0x00d8: 0x256a, # BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE
+ 0x00d9: 0x2518, # BOX DRAWINGS LIGHT UP AND LEFT
+ 0x00da: 0x250c, # BOX DRAWINGS LIGHT DOWN AND RIGHT
+ 0x00db: 0x2588, # FULL BLOCK
+ 0x00dc: 0x2584, # LOWER HALF BLOCK
+ 0x00dd: 0x258c, # LEFT HALF BLOCK
+ 0x00de: 0x2590, # RIGHT HALF BLOCK
+ 0x00df: 0x2580, # UPPER HALF BLOCK
+ 0x00e0: 0x0440, # CYRILLIC SMALL LETTER ER
+ 0x00e1: 0x0441, # CYRILLIC SMALL LETTER ES
+ 0x00e2: 0x0442, # CYRILLIC SMALL LETTER TE
+ 0x00e3: 0x0443, # CYRILLIC SMALL LETTER U
+ 0x00e4: 0x0444, # CYRILLIC SMALL LETTER EF
+ 0x00e5: 0x0445, # CYRILLIC SMALL LETTER HA
+ 0x00e6: 0x0446, # CYRILLIC SMALL LETTER TSE
+ 0x00e7: 0x0447, # CYRILLIC SMALL LETTER CHE
+ 0x00e8: 0x0448, # CYRILLIC SMALL LETTER SHA
+ 0x00e9: 0x0449, # CYRILLIC SMALL LETTER SHCHA
+ 0x00ea: 0x044a, # CYRILLIC SMALL LETTER HARD SIGN
+ 0x00eb: 0x044b, # CYRILLIC SMALL LETTER YERU
+ 0x00ec: 0x044c, # CYRILLIC SMALL LETTER SOFT SIGN
+ 0x00ed: 0x044d, # CYRILLIC SMALL LETTER E
+ 0x00ee: 0x044e, # CYRILLIC SMALL LETTER YU
+ 0x00ef: 0x044f, # CYRILLIC SMALL LETTER YA
+ 0x00f0: 0x0401, # CYRILLIC CAPITAL LETTER IO
+ 0x00f1: 0x0451, # CYRILLIC SMALL LETTER IO
+ 0x00f2: 0x0490, # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+ 0x00f3: 0x0491, # CYRILLIC SMALL LETTER GHE WITH UPTURN
+ 0x00f4: 0x0404, # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+ 0x00f5: 0x0454, # CYRILLIC SMALL LETTER UKRAINIAN IE
+ 0x00f6: 0x0406, # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+ 0x00f7: 0x0456, # CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
+ 0x00f8: 0x0407, # CYRILLIC CAPITAL LETTER YI
+ 0x00f9: 0x0457, # CYRILLIC SMALL LETTER YI
+ 0x00fa: 0x00b7, # MIDDLE DOT
+ 0x00fb: 0x221a, # SQUARE ROOT
+ 0x00fc: 0x2116, # NUMERO SIGN
+ 0x00fd: 0x00a4, # CURRENCY SIGN
+ 0x00fe: 0x25a0, # BLACK SQUARE
+ 0x00ff: 0x00a0, # NO-BREAK SPACE
+})
+
+### Decoding Table
+
+decoding_table = (
+ '\x00' # 0x0000 -> NULL
+ '\x01' # 0x0001 -> START OF HEADING
+ '\x02' # 0x0002 -> START OF TEXT
+ '\x03' # 0x0003 -> END OF TEXT
+ '\x04' # 0x0004 -> END OF TRANSMISSION
+ '\x05' # 0x0005 -> ENQUIRY
+ '\x06' # 0x0006 -> ACKNOWLEDGE
+ '\x07' # 0x0007 -> BELL
+ '\x08' # 0x0008 -> BACKSPACE
+ '\t' # 0x0009 -> HORIZONTAL TABULATION
+ '\n' # 0x000a -> LINE FEED
+ '\x0b' # 0x000b -> VERTICAL TABULATION
+ '\x0c' # 0x000c -> FORM FEED
+ '\r' # 0x000d -> CARRIAGE RETURN
+ '\x0e' # 0x000e -> SHIFT OUT
+ '\x0f' # 0x000f -> SHIFT IN
+ '\x10' # 0x0010 -> DATA LINK ESCAPE
+ '\x11' # 0x0011 -> DEVICE CONTROL ONE
+ '\x12' # 0x0012 -> DEVICE CONTROL TWO
+ '\x13' # 0x0013 -> DEVICE CONTROL THREE
+ '\x14' # 0x0014 -> DEVICE CONTROL FOUR
+ '\x15' # 0x0015 -> NEGATIVE ACKNOWLEDGE
+ '\x16' # 0x0016 -> SYNCHRONOUS IDLE
+ '\x17' # 0x0017 -> END OF TRANSMISSION BLOCK
+ '\x18' # 0x0018 -> CANCEL
+ '\x19' # 0x0019 -> END OF MEDIUM
+ '\x1a' # 0x001a -> SUBSTITUTE
+ '\x1b' # 0x001b -> ESCAPE
+ '\x1c' # 0x001c -> FILE SEPARATOR
+ '\x1d' # 0x001d -> GROUP SEPARATOR
+ '\x1e' # 0x001e -> RECORD SEPARATOR
+ '\x1f' # 0x001f -> UNIT SEPARATOR
+ ' ' # 0x0020 -> SPACE
+ '!' # 0x0021 -> EXCLAMATION MARK
+ '"' # 0x0022 -> QUOTATION MARK
+ '#' # 0x0023 -> NUMBER SIGN
+ '$' # 0x0024 -> DOLLAR SIGN
+ '%' # 0x0025 -> PERCENT SIGN
+ '&' # 0x0026 -> AMPERSAND
+ "'" # 0x0027 -> APOSTROPHE
+ '(' # 0x0028 -> LEFT PARENTHESIS
+ ')' # 0x0029 -> RIGHT PARENTHESIS
+ '*' # 0x002a -> ASTERISK
+ '+' # 0x002b -> PLUS SIGN
+ ',' # 0x002c -> COMMA
+ '-' # 0x002d -> HYPHEN-MINUS
+ '.' # 0x002e -> FULL STOP
+ '/' # 0x002f -> SOLIDUS
+ '0' # 0x0030 -> DIGIT ZERO
+ '1' # 0x0031 -> DIGIT ONE
+ '2' # 0x0032 -> DIGIT TWO
+ '3' # 0x0033 -> DIGIT THREE
+ '4' # 0x0034 -> DIGIT FOUR
+ '5' # 0x0035 -> DIGIT FIVE
+ '6' # 0x0036 -> DIGIT SIX
+ '7' # 0x0037 -> DIGIT SEVEN
+ '8' # 0x0038 -> DIGIT EIGHT
+ '9' # 0x0039 -> DIGIT NINE
+ ':' # 0x003a -> COLON
+ ';' # 0x003b -> SEMICOLON
+ '<' # 0x003c -> LESS-THAN SIGN
+ '=' # 0x003d -> EQUALS SIGN
+ '>' # 0x003e -> GREATER-THAN SIGN
+ '?' # 0x003f -> QUESTION MARK
+ '@' # 0x0040 -> COMMERCIAL AT
+ 'A' # 0x0041 -> LATIN CAPITAL LETTER A
+ 'B' # 0x0042 -> LATIN CAPITAL LETTER B
+ 'C' # 0x0043 -> LATIN CAPITAL LETTER C
+ 'D' # 0x0044 -> LATIN CAPITAL LETTER D
+ 'E' # 0x0045 -> LATIN CAPITAL LETTER E
+ 'F' # 0x0046 -> LATIN CAPITAL LETTER F
+ 'G' # 0x0047 -> LATIN CAPITAL LETTER G
+ 'H' # 0x0048 -> LATIN CAPITAL LETTER H
+ 'I' # 0x0049 -> LATIN CAPITAL LETTER I
+ 'J' # 0x004a -> LATIN CAPITAL LETTER J
+ 'K' # 0x004b -> LATIN CAPITAL LETTER K
+ 'L' # 0x004c -> LATIN CAPITAL LETTER L
+ 'M' # 0x004d -> LATIN CAPITAL LETTER M
+ 'N' # 0x004e -> LATIN CAPITAL LETTER N
+ 'O' # 0x004f -> LATIN CAPITAL LETTER O
+ 'P' # 0x0050 -> LATIN CAPITAL LETTER P
+ 'Q' # 0x0051 -> LATIN CAPITAL LETTER Q
+ 'R' # 0x0052 -> LATIN CAPITAL LETTER R
+ 'S' # 0x0053 -> LATIN CAPITAL LETTER S
+ 'T' # 0x0054 -> LATIN CAPITAL LETTER T
+ 'U' # 0x0055 -> LATIN CAPITAL LETTER U
+ 'V' # 0x0056 -> LATIN CAPITAL LETTER V
+ 'W' # 0x0057 -> LATIN CAPITAL LETTER W
+ 'X' # 0x0058 -> LATIN CAPITAL LETTER X
+ 'Y' # 0x0059 -> LATIN CAPITAL LETTER Y
+ 'Z' # 0x005a -> LATIN CAPITAL LETTER Z
+ '[' # 0x005b -> LEFT SQUARE BRACKET
+ '\\' # 0x005c -> REVERSE SOLIDUS
+ ']' # 0x005d -> RIGHT SQUARE BRACKET
+ '^' # 0x005e -> CIRCUMFLEX ACCENT
+ '_' # 0x005f -> LOW LINE
+ '`' # 0x0060 -> GRAVE ACCENT
+ 'a' # 0x0061 -> LATIN SMALL LETTER A
+ 'b' # 0x0062 -> LATIN SMALL LETTER B
+ 'c' # 0x0063 -> LATIN SMALL LETTER C
+ 'd' # 0x0064 -> LATIN SMALL LETTER D
+ 'e' # 0x0065 -> LATIN SMALL LETTER E
+ 'f' # 0x0066 -> LATIN SMALL LETTER F
+ 'g' # 0x0067 -> LATIN SMALL LETTER G
+ 'h' # 0x0068 -> LATIN SMALL LETTER H
+ 'i' # 0x0069 -> LATIN SMALL LETTER I
+ 'j' # 0x006a -> LATIN SMALL LETTER J
+ 'k' # 0x006b -> LATIN SMALL LETTER K
+ 'l' # 0x006c -> LATIN SMALL LETTER L
+ 'm' # 0x006d -> LATIN SMALL LETTER M
+ 'n' # 0x006e -> LATIN SMALL LETTER N
+ 'o' # 0x006f -> LATIN SMALL LETTER O
+ 'p' # 0x0070 -> LATIN SMALL LETTER P
+ 'q' # 0x0071 -> LATIN SMALL LETTER Q
+ 'r' # 0x0072 -> LATIN SMALL LETTER R
+ 's' # 0x0073 -> LATIN SMALL LETTER S
+ 't' # 0x0074 -> LATIN SMALL LETTER T
+ 'u' # 0x0075 -> LATIN SMALL LETTER U
+ 'v' # 0x0076 -> LATIN SMALL LETTER V
+ 'w' # 0x0077 -> LATIN SMALL LETTER W
+ 'x' # 0x0078 -> LATIN SMALL LETTER X
+ 'y' # 0x0079 -> LATIN SMALL LETTER Y
+ 'z' # 0x007a -> LATIN SMALL LETTER Z
+ '{' # 0x007b -> LEFT CURLY BRACKET
+ '|' # 0x007c -> VERTICAL LINE
+ '}' # 0x007d -> RIGHT CURLY BRACKET
+ '~' # 0x007e -> TILDE
+ '\x7f' # 0x007f -> DELETE
+ '\u0410' # 0x0080 -> CYRILLIC CAPITAL LETTER A
+ '\u0411' # 0x0081 -> CYRILLIC CAPITAL LETTER BE
+ '\u0412' # 0x0082 -> CYRILLIC CAPITAL LETTER VE
+ '\u0413' # 0x0083 -> CYRILLIC CAPITAL LETTER GHE
+ '\u0414' # 0x0084 -> CYRILLIC CAPITAL LETTER DE
+ '\u0415' # 0x0085 -> CYRILLIC CAPITAL LETTER IE
+ '\u0416' # 0x0086 -> CYRILLIC CAPITAL LETTER ZHE
+ '\u0417' # 0x0087 -> CYRILLIC CAPITAL LETTER ZE
+ '\u0418' # 0x0088 -> CYRILLIC CAPITAL LETTER I
+ '\u0419' # 0x0089 -> CYRILLIC CAPITAL LETTER SHORT I
+ '\u041a' # 0x008a -> CYRILLIC CAPITAL LETTER KA
+ '\u041b' # 0x008b -> CYRILLIC CAPITAL LETTER EL
+ '\u041c' # 0x008c -> CYRILLIC CAPITAL LETTER EM
+ '\u041d' # 0x008d -> CYRILLIC CAPITAL LETTER EN
+ '\u041e' # 0x008e -> CYRILLIC CAPITAL LETTER O
+ '\u041f' # 0x008f -> CYRILLIC CAPITAL LETTER PE
+ '\u0420' # 0x0090 -> CYRILLIC CAPITAL LETTER ER
+ '\u0421' # 0x0091 -> CYRILLIC CAPITAL LETTER ES
+ '\u0422' # 0x0092 -> CYRILLIC CAPITAL LETTER TE
+ '\u0423' # 0x0093 -> CYRILLIC CAPITAL LETTER U
+ '\u0424' # 0x0094 -> CYRILLIC CAPITAL LETTER EF
+ '\u0425' # 0x0095 -> CYRILLIC CAPITAL LETTER HA
+ '\u0426' # 0x0096 -> CYRILLIC CAPITAL LETTER TSE
+ '\u0427' # 0x0097 -> CYRILLIC CAPITAL LETTER CHE
+ '\u0428' # 0x0098 -> CYRILLIC CAPITAL LETTER SHA
+ '\u0429' # 0x0099 -> CYRILLIC CAPITAL LETTER SHCHA
+ '\u042a' # 0x009a -> CYRILLIC CAPITAL LETTER HARD SIGN
+ '\u042b' # 0x009b -> CYRILLIC CAPITAL LETTER YERU
+ '\u042c' # 0x009c -> CYRILLIC CAPITAL LETTER SOFT SIGN
+ '\u042d' # 0x009d -> CYRILLIC CAPITAL LETTER E
+ '\u042e' # 0x009e -> CYRILLIC CAPITAL LETTER YU
+ '\u042f' # 0x009f -> CYRILLIC CAPITAL LETTER YA
+ '\u0430' # 0x00a0 -> CYRILLIC SMALL LETTER A
+ '\u0431' # 0x00a1 -> CYRILLIC SMALL LETTER BE
+ '\u0432' # 0x00a2 -> CYRILLIC SMALL LETTER VE
+ '\u0433' # 0x00a3 -> CYRILLIC SMALL LETTER GHE
+ '\u0434' # 0x00a4 -> CYRILLIC SMALL LETTER DE
+ '\u0435' # 0x00a5 -> CYRILLIC SMALL LETTER IE
+ '\u0436' # 0x00a6 -> CYRILLIC SMALL LETTER ZHE
+ '\u0437' # 0x00a7 -> CYRILLIC SMALL LETTER ZE
+ '\u0438' # 0x00a8 -> CYRILLIC SMALL LETTER I
+ '\u0439' # 0x00a9 -> CYRILLIC SMALL LETTER SHORT I
+ '\u043a' # 0x00aa -> CYRILLIC SMALL LETTER KA
+ '\u043b' # 0x00ab -> CYRILLIC SMALL LETTER EL
+ '\u043c' # 0x00ac -> CYRILLIC SMALL LETTER EM
+ '\u043d' # 0x00ad -> CYRILLIC SMALL LETTER EN
+ '\u043e' # 0x00ae -> CYRILLIC SMALL LETTER O
+ '\u043f' # 0x00af -> CYRILLIC SMALL LETTER PE
+ '\u2591' # 0x00b0 -> LIGHT SHADE
+ '\u2592' # 0x00b1 -> MEDIUM SHADE
+ '\u2593' # 0x00b2 -> DARK SHADE
+ '\u2502' # 0x00b3 -> BOX DRAWINGS LIGHT VERTICAL
+ '\u2524' # 0x00b4 -> BOX DRAWINGS LIGHT VERTICAL AND LEFT
+ '\u2561' # 0x00b5 -> BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE
+ '\u2562' # 0x00b6 -> BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE
+ '\u2556' # 0x00b7 -> BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE
+ '\u2555' # 0x00b8 -> BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE
+ '\u2563' # 0x00b9 -> BOX DRAWINGS DOUBLE VERTICAL AND LEFT
+ '\u2551' # 0x00ba -> BOX DRAWINGS DOUBLE VERTICAL
+ '\u2557' # 0x00bb -> BOX DRAWINGS DOUBLE DOWN AND LEFT
+ '\u255d' # 0x00bc -> BOX DRAWINGS DOUBLE UP AND LEFT
+ '\u255c' # 0x00bd -> BOX DRAWINGS UP DOUBLE AND LEFT SINGLE
+ '\u255b' # 0x00be -> BOX DRAWINGS UP SINGLE AND LEFT DOUBLE
+ '\u2510' # 0x00bf -> BOX DRAWINGS LIGHT DOWN AND LEFT
+ '\u2514' # 0x00c0 -> BOX DRAWINGS LIGHT UP AND RIGHT
+ '\u2534' # 0x00c1 -> BOX DRAWINGS LIGHT UP AND HORIZONTAL
+ '\u252c' # 0x00c2 -> BOX DRAWINGS LIGHT DOWN AND HORIZONTAL
+ '\u251c' # 0x00c3 -> BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+ '\u2500' # 0x00c4 -> BOX DRAWINGS LIGHT HORIZONTAL
+ '\u253c' # 0x00c5 -> BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL
+ '\u255e' # 0x00c6 -> BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE
+ '\u255f' # 0x00c7 -> BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE
+ '\u255a' # 0x00c8 -> BOX DRAWINGS DOUBLE UP AND RIGHT
+ '\u2554' # 0x00c9 -> BOX DRAWINGS DOUBLE DOWN AND RIGHT
+ '\u2569' # 0x00ca -> BOX DRAWINGS DOUBLE UP AND HORIZONTAL
+ '\u2566' # 0x00cb -> BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL
+ '\u2560' # 0x00cc -> BOX DRAWINGS DOUBLE VERTICAL AND RIGHT
+ '\u2550' # 0x00cd -> BOX DRAWINGS DOUBLE HORIZONTAL
+ '\u256c' # 0x00ce -> BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL
+ '\u2567' # 0x00cf -> BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE
+ '\u2568' # 0x00d0 -> BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE
+ '\u2564' # 0x00d1 -> BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE
+ '\u2565' # 0x00d2 -> BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE
+ '\u2559' # 0x00d3 -> BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE
+ '\u2558' # 0x00d4 -> BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE
+ '\u2552' # 0x00d5 -> BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE
+ '\u2553' # 0x00d6 -> BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE
+ '\u256b' # 0x00d7 -> BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE
+ '\u256a' # 0x00d8 -> BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE
+ '\u2518' # 0x00d9 -> BOX DRAWINGS LIGHT UP AND LEFT
+ '\u250c' # 0x00da -> BOX DRAWINGS LIGHT DOWN AND RIGHT
+ '\u2588' # 0x00db -> FULL BLOCK
+ '\u2584' # 0x00dc -> LOWER HALF BLOCK
+ '\u258c' # 0x00dd -> LEFT HALF BLOCK
+ '\u2590' # 0x00de -> RIGHT HALF BLOCK
+ '\u2580' # 0x00df -> UPPER HALF BLOCK
+ '\u0440' # 0x00e0 -> CYRILLIC SMALL LETTER ER
+ '\u0441' # 0x00e1 -> CYRILLIC SMALL LETTER ES
+ '\u0442' # 0x00e2 -> CYRILLIC SMALL LETTER TE
+ '\u0443' # 0x00e3 -> CYRILLIC SMALL LETTER U
+ '\u0444' # 0x00e4 -> CYRILLIC SMALL LETTER EF
+ '\u0445' # 0x00e5 -> CYRILLIC SMALL LETTER HA
+ '\u0446' # 0x00e6 -> CYRILLIC SMALL LETTER TSE
+ '\u0447' # 0x00e7 -> CYRILLIC SMALL LETTER CHE
+ '\u0448' # 0x00e8 -> CYRILLIC SMALL LETTER SHA
+ '\u0449' # 0x00e9 -> CYRILLIC SMALL LETTER SHCHA
+ '\u044a' # 0x00ea -> CYRILLIC SMALL LETTER HARD SIGN
+ '\u044b' # 0x00eb -> CYRILLIC SMALL LETTER YERU
+ '\u044c' # 0x00ec -> CYRILLIC SMALL LETTER SOFT SIGN
+ '\u044d' # 0x00ed -> CYRILLIC SMALL LETTER E
+ '\u044e' # 0x00ee -> CYRILLIC SMALL LETTER YU
+ '\u044f' # 0x00ef -> CYRILLIC SMALL LETTER YA
+ '\u0401' # 0x00f0 -> CYRILLIC CAPITAL LETTER IO
+ '\u0451' # 0x00f1 -> CYRILLIC SMALL LETTER IO
+ '\u0490' # 0x00f2 -> CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+ '\u0491' # 0x00f3 -> CYRILLIC SMALL LETTER GHE WITH UPTURN
+ '\u0404' # 0x00f4 -> CYRILLIC CAPITAL LETTER UKRAINIAN IE
+ '\u0454' # 0x00f5 -> CYRILLIC SMALL LETTER UKRAINIAN IE
+ '\u0406' # 0x00f6 -> CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+ '\u0456' # 0x00f7 -> CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
+ '\u0407' # 0x00f8 -> CYRILLIC CAPITAL LETTER YI
+ '\u0457' # 0x00f9 -> CYRILLIC SMALL LETTER YI
+ '\xb7' # 0x00fa -> MIDDLE DOT
+ '\u221a' # 0x00fb -> SQUARE ROOT
+ '\u2116' # 0x00fc -> NUMERO SIGN
+ '\xa4' # 0x00fd -> CURRENCY SIGN
+ '\u25a0' # 0x00fe -> BLACK SQUARE
+ '\xa0' # 0x00ff -> NO-BREAK SPACE
+)
+
+### Encoding Map
+
+encoding_map = {
+ 0x0000: 0x0000, # NULL
+ 0x0001: 0x0001, # START OF HEADING
+ 0x0002: 0x0002, # START OF TEXT
+ 0x0003: 0x0003, # END OF TEXT
+ 0x0004: 0x0004, # END OF TRANSMISSION
+ 0x0005: 0x0005, # ENQUIRY
+ 0x0006: 0x0006, # ACKNOWLEDGE
+ 0x0007: 0x0007, # BELL
+ 0x0008: 0x0008, # BACKSPACE
+ 0x0009: 0x0009, # HORIZONTAL TABULATION
+ 0x000a: 0x000a, # LINE FEED
+ 0x000b: 0x000b, # VERTICAL TABULATION
+ 0x000c: 0x000c, # FORM FEED
+ 0x000d: 0x000d, # CARRIAGE RETURN
+ 0x000e: 0x000e, # SHIFT OUT
+ 0x000f: 0x000f, # SHIFT IN
+ 0x0010: 0x0010, # DATA LINK ESCAPE
+ 0x0011: 0x0011, # DEVICE CONTROL ONE
+ 0x0012: 0x0012, # DEVICE CONTROL TWO
+ 0x0013: 0x0013, # DEVICE CONTROL THREE
+ 0x0014: 0x0014, # DEVICE CONTROL FOUR
+ 0x0015: 0x0015, # NEGATIVE ACKNOWLEDGE
+ 0x0016: 0x0016, # SYNCHRONOUS IDLE
+ 0x0017: 0x0017, # END OF TRANSMISSION BLOCK
+ 0x0018: 0x0018, # CANCEL
+ 0x0019: 0x0019, # END OF MEDIUM
+ 0x001a: 0x001a, # SUBSTITUTE
+ 0x001b: 0x001b, # ESCAPE
+ 0x001c: 0x001c, # FILE SEPARATOR
+ 0x001d: 0x001d, # GROUP SEPARATOR
+ 0x001e: 0x001e, # RECORD SEPARATOR
+ 0x001f: 0x001f, # UNIT SEPARATOR
+ 0x0020: 0x0020, # SPACE
+ 0x0021: 0x0021, # EXCLAMATION MARK
+ 0x0022: 0x0022, # QUOTATION MARK
+ 0x0023: 0x0023, # NUMBER SIGN
+ 0x0024: 0x0024, # DOLLAR SIGN
+ 0x0025: 0x0025, # PERCENT SIGN
+ 0x0026: 0x0026, # AMPERSAND
+ 0x0027: 0x0027, # APOSTROPHE
+ 0x0028: 0x0028, # LEFT PARENTHESIS
+ 0x0029: 0x0029, # RIGHT PARENTHESIS
+ 0x002a: 0x002a, # ASTERISK
+ 0x002b: 0x002b, # PLUS SIGN
+ 0x002c: 0x002c, # COMMA
+ 0x002d: 0x002d, # HYPHEN-MINUS
+ 0x002e: 0x002e, # FULL STOP
+ 0x002f: 0x002f, # SOLIDUS
+ 0x0030: 0x0030, # DIGIT ZERO
+ 0x0031: 0x0031, # DIGIT ONE
+ 0x0032: 0x0032, # DIGIT TWO
+ 0x0033: 0x0033, # DIGIT THREE
+ 0x0034: 0x0034, # DIGIT FOUR
+ 0x0035: 0x0035, # DIGIT FIVE
+ 0x0036: 0x0036, # DIGIT SIX
+ 0x0037: 0x0037, # DIGIT SEVEN
+ 0x0038: 0x0038, # DIGIT EIGHT
+ 0x0039: 0x0039, # DIGIT NINE
+ 0x003a: 0x003a, # COLON
+ 0x003b: 0x003b, # SEMICOLON
+ 0x003c: 0x003c, # LESS-THAN SIGN
+ 0x003d: 0x003d, # EQUALS SIGN
+ 0x003e: 0x003e, # GREATER-THAN SIGN
+ 0x003f: 0x003f, # QUESTION MARK
+ 0x0040: 0x0040, # COMMERCIAL AT
+ 0x0041: 0x0041, # LATIN CAPITAL LETTER A
+ 0x0042: 0x0042, # LATIN CAPITAL LETTER B
+ 0x0043: 0x0043, # LATIN CAPITAL LETTER C
+ 0x0044: 0x0044, # LATIN CAPITAL LETTER D
+ 0x0045: 0x0045, # LATIN CAPITAL LETTER E
+ 0x0046: 0x0046, # LATIN CAPITAL LETTER F
+ 0x0047: 0x0047, # LATIN CAPITAL LETTER G
+ 0x0048: 0x0048, # LATIN CAPITAL LETTER H
+ 0x0049: 0x0049, # LATIN CAPITAL LETTER I
+ 0x004a: 0x004a, # LATIN CAPITAL LETTER J
+ 0x004b: 0x004b, # LATIN CAPITAL LETTER K
+ 0x004c: 0x004c, # LATIN CAPITAL LETTER L
+ 0x004d: 0x004d, # LATIN CAPITAL LETTER M
+ 0x004e: 0x004e, # LATIN CAPITAL LETTER N
+ 0x004f: 0x004f, # LATIN CAPITAL LETTER O
+ 0x0050: 0x0050, # LATIN CAPITAL LETTER P
+ 0x0051: 0x0051, # LATIN CAPITAL LETTER Q
+ 0x0052: 0x0052, # LATIN CAPITAL LETTER R
+ 0x0053: 0x0053, # LATIN CAPITAL LETTER S
+ 0x0054: 0x0054, # LATIN CAPITAL LETTER T
+ 0x0055: 0x0055, # LATIN CAPITAL LETTER U
+ 0x0056: 0x0056, # LATIN CAPITAL LETTER V
+ 0x0057: 0x0057, # LATIN CAPITAL LETTER W
+ 0x0058: 0x0058, # LATIN CAPITAL LETTER X
+ 0x0059: 0x0059, # LATIN CAPITAL LETTER Y
+ 0x005a: 0x005a, # LATIN CAPITAL LETTER Z
+ 0x005b: 0x005b, # LEFT SQUARE BRACKET
+ 0x005c: 0x005c, # REVERSE SOLIDUS
+ 0x005d: 0x005d, # RIGHT SQUARE BRACKET
+ 0x005e: 0x005e, # CIRCUMFLEX ACCENT
+ 0x005f: 0x005f, # LOW LINE
+ 0x0060: 0x0060, # GRAVE ACCENT
+ 0x0061: 0x0061, # LATIN SMALL LETTER A
+ 0x0062: 0x0062, # LATIN SMALL LETTER B
+ 0x0063: 0x0063, # LATIN SMALL LETTER C
+ 0x0064: 0x0064, # LATIN SMALL LETTER D
+ 0x0065: 0x0065, # LATIN SMALL LETTER E
+ 0x0066: 0x0066, # LATIN SMALL LETTER F
+ 0x0067: 0x0067, # LATIN SMALL LETTER G
+ 0x0068: 0x0068, # LATIN SMALL LETTER H
+ 0x0069: 0x0069, # LATIN SMALL LETTER I
+ 0x006a: 0x006a, # LATIN SMALL LETTER J
+ 0x006b: 0x006b, # LATIN SMALL LETTER K
+ 0x006c: 0x006c, # LATIN SMALL LETTER L
+ 0x006d: 0x006d, # LATIN SMALL LETTER M
+ 0x006e: 0x006e, # LATIN SMALL LETTER N
+ 0x006f: 0x006f, # LATIN SMALL LETTER O
+ 0x0070: 0x0070, # LATIN SMALL LETTER P
+ 0x0071: 0x0071, # LATIN SMALL LETTER Q
+ 0x0072: 0x0072, # LATIN SMALL LETTER R
+ 0x0073: 0x0073, # LATIN SMALL LETTER S
+ 0x0074: 0x0074, # LATIN SMALL LETTER T
+ 0x0075: 0x0075, # LATIN SMALL LETTER U
+ 0x0076: 0x0076, # LATIN SMALL LETTER V
+ 0x0077: 0x0077, # LATIN SMALL LETTER W
+ 0x0078: 0x0078, # LATIN SMALL LETTER X
+ 0x0079: 0x0079, # LATIN SMALL LETTER Y
+ 0x007a: 0x007a, # LATIN SMALL LETTER Z
+ 0x007b: 0x007b, # LEFT CURLY BRACKET
+ 0x007c: 0x007c, # VERTICAL LINE
+ 0x007d: 0x007d, # RIGHT CURLY BRACKET
+ 0x007e: 0x007e, # TILDE
+ 0x007f: 0x007f, # DELETE
+ 0x00a0: 0x00ff, # NO-BREAK SPACE
+ 0x00a4: 0x00fd, # CURRENCY SIGN
+ 0x00b7: 0x00fa, # MIDDLE DOT
+ 0x0401: 0x00f0, # CYRILLIC CAPITAL LETTER IO
+ 0x0404: 0x00f4, # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+ 0x0406: 0x00f6, # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+ 0x0407: 0x00f8, # CYRILLIC CAPITAL LETTER YI
+ 0x0410: 0x0080, # CYRILLIC CAPITAL LETTER A
+ 0x0411: 0x0081, # CYRILLIC CAPITAL LETTER BE
+ 0x0412: 0x0082, # CYRILLIC CAPITAL LETTER VE
+ 0x0413: 0x0083, # CYRILLIC CAPITAL LETTER GHE
+ 0x0414: 0x0084, # CYRILLIC CAPITAL LETTER DE
+ 0x0415: 0x0085, # CYRILLIC CAPITAL LETTER IE
+ 0x0416: 0x0086, # CYRILLIC CAPITAL LETTER ZHE
+ 0x0417: 0x0087, # CYRILLIC CAPITAL LETTER ZE
+ 0x0418: 0x0088, # CYRILLIC CAPITAL LETTER I
+ 0x0419: 0x0089, # CYRILLIC CAPITAL LETTER SHORT I
+ 0x041a: 0x008a, # CYRILLIC CAPITAL LETTER KA
+ 0x041b: 0x008b, # CYRILLIC CAPITAL LETTER EL
+ 0x041c: 0x008c, # CYRILLIC CAPITAL LETTER EM
+ 0x041d: 0x008d, # CYRILLIC CAPITAL LETTER EN
+ 0x041e: 0x008e, # CYRILLIC CAPITAL LETTER O
+ 0x041f: 0x008f, # CYRILLIC CAPITAL LETTER PE
+ 0x0420: 0x0090, # CYRILLIC CAPITAL LETTER ER
+ 0x0421: 0x0091, # CYRILLIC CAPITAL LETTER ES
+ 0x0422: 0x0092, # CYRILLIC CAPITAL LETTER TE
+ 0x0423: 0x0093, # CYRILLIC CAPITAL LETTER U
+ 0x0424: 0x0094, # CYRILLIC CAPITAL LETTER EF
+ 0x0425: 0x0095, # CYRILLIC CAPITAL LETTER HA
+ 0x0426: 0x0096, # CYRILLIC CAPITAL LETTER TSE
+ 0x0427: 0x0097, # CYRILLIC CAPITAL LETTER CHE
+ 0x0428: 0x0098, # CYRILLIC CAPITAL LETTER SHA
+ 0x0429: 0x0099, # CYRILLIC CAPITAL LETTER SHCHA
+ 0x042a: 0x009a, # CYRILLIC CAPITAL LETTER HARD SIGN
+ 0x042b: 0x009b, # CYRILLIC CAPITAL LETTER YERU
+ 0x042c: 0x009c, # CYRILLIC CAPITAL LETTER SOFT SIGN
+ 0x042d: 0x009d, # CYRILLIC CAPITAL LETTER E
+ 0x042e: 0x009e, # CYRILLIC CAPITAL LETTER YU
+ 0x042f: 0x009f, # CYRILLIC CAPITAL LETTER YA
+ 0x0430: 0x00a0, # CYRILLIC SMALL LETTER A
+ 0x0431: 0x00a1, # CYRILLIC SMALL LETTER BE
+ 0x0432: 0x00a2, # CYRILLIC SMALL LETTER VE
+ 0x0433: 0x00a3, # CYRILLIC SMALL LETTER GHE
+ 0x0434: 0x00a4, # CYRILLIC SMALL LETTER DE
+ 0x0435: 0x00a5, # CYRILLIC SMALL LETTER IE
+ 0x0436: 0x00a6, # CYRILLIC SMALL LETTER ZHE
+ 0x0437: 0x00a7, # CYRILLIC SMALL LETTER ZE
+ 0x0438: 0x00a8, # CYRILLIC SMALL LETTER I
+ 0x0439: 0x00a9, # CYRILLIC SMALL LETTER SHORT I
+ 0x043a: 0x00aa, # CYRILLIC SMALL LETTER KA
+ 0x043b: 0x00ab, # CYRILLIC SMALL LETTER EL
+ 0x043c: 0x00ac, # CYRILLIC SMALL LETTER EM
+ 0x043d: 0x00ad, # CYRILLIC SMALL LETTER EN
+ 0x043e: 0x00ae, # CYRILLIC SMALL LETTER O
+ 0x043f: 0x00af, # CYRILLIC SMALL LETTER PE
+ 0x0440: 0x00e0, # CYRILLIC SMALL LETTER ER
+ 0x0441: 0x00e1, # CYRILLIC SMALL LETTER ES
+ 0x0442: 0x00e2, # CYRILLIC SMALL LETTER TE
+ 0x0443: 0x00e3, # CYRILLIC SMALL LETTER U
+ 0x0444: 0x00e4, # CYRILLIC SMALL LETTER EF
+ 0x0445: 0x00e5, # CYRILLIC SMALL LETTER HA
+ 0x0446: 0x00e6, # CYRILLIC SMALL LETTER TSE
+ 0x0447: 0x00e7, # CYRILLIC SMALL LETTER CHE
+ 0x0448: 0x00e8, # CYRILLIC SMALL LETTER SHA
+ 0x0449: 0x00e9, # CYRILLIC SMALL LETTER SHCHA
+ 0x044a: 0x00ea, # CYRILLIC SMALL LETTER HARD SIGN
+ 0x044b: 0x00eb, # CYRILLIC SMALL LETTER YERU
+ 0x044c: 0x00ec, # CYRILLIC SMALL LETTER SOFT SIGN
+ 0x044d: 0x00ed, # CYRILLIC SMALL LETTER E
+ 0x044e: 0x00ee, # CYRILLIC SMALL LETTER YU
+ 0x044f: 0x00ef, # CYRILLIC SMALL LETTER YA
+ 0x0451: 0x00f1, # CYRILLIC SMALL LETTER IO
+ 0x0454: 0x00f5, # CYRILLIC SMALL LETTER UKRAINIAN IE
+ 0x0456: 0x00f7, # CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I
+ 0x0457: 0x00f9, # CYRILLIC SMALL LETTER YI
+ 0x0490: 0x00f2, # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+ 0x0491: 0x00f3, # CYRILLIC SMALL LETTER GHE WITH UPTURN
+ 0x2116: 0x00fc, # NUMERO SIGN
+ 0x221a: 0x00fb, # SQUARE ROOT
+ 0x2500: 0x00c4, # BOX DRAWINGS LIGHT HORIZONTAL
+ 0x2502: 0x00b3, # BOX DRAWINGS LIGHT VERTICAL
+ 0x250c: 0x00da, # BOX DRAWINGS LIGHT DOWN AND RIGHT
+ 0x2510: 0x00bf, # BOX DRAWINGS LIGHT DOWN AND LEFT
+ 0x2514: 0x00c0, # BOX DRAWINGS LIGHT UP AND RIGHT
+ 0x2518: 0x00d9, # BOX DRAWINGS LIGHT UP AND LEFT
+ 0x251c: 0x00c3, # BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+ 0x2524: 0x00b4, # BOX DRAWINGS LIGHT VERTICAL AND LEFT
+ 0x252c: 0x00c2, # BOX DRAWINGS LIGHT DOWN AND HORIZONTAL
+ 0x2534: 0x00c1, # BOX DRAWINGS LIGHT UP AND HORIZONTAL
+ 0x253c: 0x00c5, # BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL
+ 0x2550: 0x00cd, # BOX DRAWINGS DOUBLE HORIZONTAL
+ 0x2551: 0x00ba, # BOX DRAWINGS DOUBLE VERTICAL
+ 0x2552: 0x00d5, # BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE
+ 0x2553: 0x00d6, # BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE
+ 0x2554: 0x00c9, # BOX DRAWINGS DOUBLE DOWN AND RIGHT
+ 0x2555: 0x00b8, # BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE
+ 0x2556: 0x00b7, # BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE
+ 0x2557: 0x00bb, # BOX DRAWINGS DOUBLE DOWN AND LEFT
+ 0x2558: 0x00d4, # BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE
+ 0x2559: 0x00d3, # BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE
+ 0x255a: 0x00c8, # BOX DRAWINGS DOUBLE UP AND RIGHT
+ 0x255b: 0x00be, # BOX DRAWINGS UP SINGLE AND LEFT DOUBLE
+ 0x255c: 0x00bd, # BOX DRAWINGS UP DOUBLE AND LEFT SINGLE
+ 0x255d: 0x00bc, # BOX DRAWINGS DOUBLE UP AND LEFT
+ 0x255e: 0x00c6, # BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE
+ 0x255f: 0x00c7, # BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE
+ 0x2560: 0x00cc, # BOX DRAWINGS DOUBLE VERTICAL AND RIGHT
+ 0x2561: 0x00b5, # BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE
+ 0x2562: 0x00b6, # BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE
+ 0x2563: 0x00b9, # BOX DRAWINGS DOUBLE VERTICAL AND LEFT
+ 0x2564: 0x00d1, # BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE
+ 0x2565: 0x00d2, # BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE
+ 0x2566: 0x00cb, # BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL
+ 0x2567: 0x00cf, # BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE
+ 0x2568: 0x00d0, # BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE
+ 0x2569: 0x00ca, # BOX DRAWINGS DOUBLE UP AND HORIZONTAL
+ 0x256a: 0x00d8, # BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE
+ 0x256b: 0x00d7, # BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE
+ 0x256c: 0x00ce, # BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL
+ 0x2580: 0x00df, # UPPER HALF BLOCK
+ 0x2584: 0x00dc, # LOWER HALF BLOCK
+ 0x2588: 0x00db, # FULL BLOCK
+ 0x258c: 0x00dd, # LEFT HALF BLOCK
+ 0x2590: 0x00de, # RIGHT HALF BLOCK
+ 0x2591: 0x00b0, # LIGHT SHADE
+ 0x2592: 0x00b1, # MEDIUM SHADE
+ 0x2593: 0x00b2, # DARK SHADE
+ 0x25a0: 0x00fe, # BLACK SQUARE
+}
diff --git a/Lib/encodings/cp273.py b/Lib/encodings/cp273.py
new file mode 100644
index 0000000..69c6d77
--- /dev/null
+++ b/Lib/encodings/cp273.py
@@ -0,0 +1,307 @@
+""" Python Character Mapping Codec cp273 generated from 'python-mappings/CP273.TXT' with gencodec.py.
+
+"""#"
+
+import codecs
+
+### Codec APIs
+
+class Codec(codecs.Codec):
+
+ def encode(self,input,errors='strict'):
+ return codecs.charmap_encode(input,errors,encoding_table)
+
+ def decode(self,input,errors='strict'):
+ return codecs.charmap_decode(input,errors,decoding_table)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+ def encode(self, input, final=False):
+ return codecs.charmap_encode(input,self.errors,encoding_table)[0]
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+ def decode(self, input, final=False):
+ return codecs.charmap_decode(input,self.errors,decoding_table)[0]
+
+class StreamWriter(Codec,codecs.StreamWriter):
+ pass
+
+class StreamReader(Codec,codecs.StreamReader):
+ pass
+
+### encodings module API
+
+def getregentry():
+ return codecs.CodecInfo(
+ name='cp273',
+ encode=Codec().encode,
+ decode=Codec().decode,
+ incrementalencoder=IncrementalEncoder,
+ incrementaldecoder=IncrementalDecoder,
+ streamreader=StreamReader,
+ streamwriter=StreamWriter,
+ )
+
+
+### Decoding Table
+
+decoding_table = (
+ '\x00' # 0x00 -> NULL (NUL)
+ '\x01' # 0x01 -> START OF HEADING (SOH)
+ '\x02' # 0x02 -> START OF TEXT (STX)
+ '\x03' # 0x03 -> END OF TEXT (ETX)
+ '\x9c' # 0x04 -> STRING TERMINATOR (ST)
+ '\t' # 0x05 -> CHARACTER TABULATION (HT)
+ '\x86' # 0x06 -> START OF SELECTED AREA (SSA)
+ '\x7f' # 0x07 -> DELETE (DEL)
+ '\x97' # 0x08 -> END OF GUARDED AREA (EPA)
+ '\x8d' # 0x09 -> REVERSE LINE FEED (RI)
+ '\x8e' # 0x0A -> SINGLE-SHIFT TWO (SS2)
+ '\x0b' # 0x0B -> LINE TABULATION (VT)
+ '\x0c' # 0x0C -> FORM FEED (FF)
+ '\r' # 0x0D -> CARRIAGE RETURN (CR)
+ '\x0e' # 0x0E -> SHIFT OUT (SO)
+ '\x0f' # 0x0F -> SHIFT IN (SI)
+ '\x10' # 0x10 -> DATALINK ESCAPE (DLE)
+ '\x11' # 0x11 -> DEVICE CONTROL ONE (DC1)
+ '\x12' # 0x12 -> DEVICE CONTROL TWO (DC2)
+ '\x13' # 0x13 -> DEVICE CONTROL THREE (DC3)
+ '\x9d' # 0x14 -> OPERATING SYSTEM COMMAND (OSC)
+ '\x85' # 0x15 -> NEXT LINE (NEL)
+ '\x08' # 0x16 -> BACKSPACE (BS)
+ '\x87' # 0x17 -> END OF SELECTED AREA (ESA)
+ '\x18' # 0x18 -> CANCEL (CAN)
+ '\x19' # 0x19 -> END OF MEDIUM (EM)
+ '\x92' # 0x1A -> PRIVATE USE TWO (PU2)
+ '\x8f' # 0x1B -> SINGLE-SHIFT THREE (SS3)
+ '\x1c' # 0x1C -> FILE SEPARATOR (IS4)
+ '\x1d' # 0x1D -> GROUP SEPARATOR (IS3)
+ '\x1e' # 0x1E -> RECORD SEPARATOR (IS2)
+ '\x1f' # 0x1F -> UNIT SEPARATOR (IS1)
+ '\x80' # 0x20 -> PADDING CHARACTER (PAD)
+ '\x81' # 0x21 -> HIGH OCTET PRESET (HOP)
+ '\x82' # 0x22 -> BREAK PERMITTED HERE (BPH)
+ '\x83' # 0x23 -> NO BREAK HERE (NBH)
+ '\x84' # 0x24 -> INDEX (IND)
+ '\n' # 0x25 -> LINE FEED (LF)
+ '\x17' # 0x26 -> END OF TRANSMISSION BLOCK (ETB)
+ '\x1b' # 0x27 -> ESCAPE (ESC)
+ '\x88' # 0x28 -> CHARACTER TABULATION SET (HTS)
+ '\x89' # 0x29 -> CHARACTER TABULATION WITH JUSTIFICATION (HTJ)
+ '\x8a' # 0x2A -> LINE TABULATION SET (VTS)
+ '\x8b' # 0x2B -> PARTIAL LINE FORWARD (PLD)
+ '\x8c' # 0x2C -> PARTIAL LINE BACKWARD (PLU)
+ '\x05' # 0x2D -> ENQUIRY (ENQ)
+ '\x06' # 0x2E -> ACKNOWLEDGE (ACK)
+ '\x07' # 0x2F -> BELL (BEL)
+ '\x90' # 0x30 -> DEVICE CONTROL STRING (DCS)
+ '\x91' # 0x31 -> PRIVATE USE ONE (PU1)
+ '\x16' # 0x32 -> SYNCHRONOUS IDLE (SYN)
+ '\x93' # 0x33 -> SET TRANSMIT STATE (STS)
+ '\x94' # 0x34 -> CANCEL CHARACTER (CCH)
+ '\x95' # 0x35 -> MESSAGE WAITING (MW)
+ '\x96' # 0x36 -> START OF GUARDED AREA (SPA)
+ '\x04' # 0x37 -> END OF TRANSMISSION (EOT)
+ '\x98' # 0x38 -> START OF STRING (SOS)
+ '\x99' # 0x39 -> SINGLE GRAPHIC CHARACTER INTRODUCER (SGCI)
+ '\x9a' # 0x3A -> SINGLE CHARACTER INTRODUCER (SCI)
+ '\x9b' # 0x3B -> CONTROL SEQUENCE INTRODUCER (CSI)
+ '\x14' # 0x3C -> DEVICE CONTROL FOUR (DC4)
+ '\x15' # 0x3D -> NEGATIVE ACKNOWLEDGE (NAK)
+ '\x9e' # 0x3E -> PRIVACY MESSAGE (PM)
+ '\x1a' # 0x3F -> SUBSTITUTE (SUB)
+ ' ' # 0x40 -> SPACE
+ '\xa0' # 0x41 -> NO-BREAK SPACE
+ '\xe2' # 0x42 -> LATIN SMALL LETTER A WITH CIRCUMFLEX
+ '{' # 0x43 -> LEFT CURLY BRACKET
+ '\xe0' # 0x44 -> LATIN SMALL LETTER A WITH GRAVE
+ '\xe1' # 0x45 -> LATIN SMALL LETTER A WITH ACUTE
+ '\xe3' # 0x46 -> LATIN SMALL LETTER A WITH TILDE
+ '\xe5' # 0x47 -> LATIN SMALL LETTER A WITH RING ABOVE
+ '\xe7' # 0x48 -> LATIN SMALL LETTER C WITH CEDILLA
+ '\xf1' # 0x49 -> LATIN SMALL LETTER N WITH TILDE
+ '\xc4' # 0x4A -> LATIN CAPITAL LETTER A WITH DIAERESIS
+ '.' # 0x4B -> FULL STOP
+ '<' # 0x4C -> LESS-THAN SIGN
+ '(' # 0x4D -> LEFT PARENTHESIS
+ '+' # 0x4E -> PLUS SIGN
+ '!' # 0x4F -> EXCLAMATION MARK
+ '&' # 0x50 -> AMPERSAND
+ '\xe9' # 0x51 -> LATIN SMALL LETTER E WITH ACUTE
+ '\xea' # 0x52 -> LATIN SMALL LETTER E WITH CIRCUMFLEX
+ '\xeb' # 0x53 -> LATIN SMALL LETTER E WITH DIAERESIS
+ '\xe8' # 0x54 -> LATIN SMALL LETTER E WITH GRAVE
+ '\xed' # 0x55 -> LATIN SMALL LETTER I WITH ACUTE
+ '\xee' # 0x56 -> LATIN SMALL LETTER I WITH CIRCUMFLEX
+ '\xef' # 0x57 -> LATIN SMALL LETTER I WITH DIAERESIS
+ '\xec' # 0x58 -> LATIN SMALL LETTER I WITH GRAVE
+ '~' # 0x59 -> TILDE
+ '\xdc' # 0x5A -> LATIN CAPITAL LETTER U WITH DIAERESIS
+ '$' # 0x5B -> DOLLAR SIGN
+ '*' # 0x5C -> ASTERISK
+ ')' # 0x5D -> RIGHT PARENTHESIS
+ ';' # 0x5E -> SEMICOLON
+ '^' # 0x5F -> CIRCUMFLEX ACCENT
+ '-' # 0x60 -> HYPHEN-MINUS
+ '/' # 0x61 -> SOLIDUS
+ '\xc2' # 0x62 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+ '[' # 0x63 -> LEFT SQUARE BRACKET
+ '\xc0' # 0x64 -> LATIN CAPITAL LETTER A WITH GRAVE
+ '\xc1' # 0x65 -> LATIN CAPITAL LETTER A WITH ACUTE
+ '\xc3' # 0x66 -> LATIN CAPITAL LETTER A WITH TILDE
+ '\xc5' # 0x67 -> LATIN CAPITAL LETTER A WITH RING ABOVE
+ '\xc7' # 0x68 -> LATIN CAPITAL LETTER C WITH CEDILLA
+ '\xd1' # 0x69 -> LATIN CAPITAL LETTER N WITH TILDE
+ '\xf6' # 0x6A -> LATIN SMALL LETTER O WITH DIAERESIS
+ ',' # 0x6B -> COMMA
+ '%' # 0x6C -> PERCENT SIGN
+ '_' # 0x6D -> LOW LINE
+ '>' # 0x6E -> GREATER-THAN SIGN
+ '?' # 0x6F -> QUESTION MARK
+ '\xf8' # 0x70 -> LATIN SMALL LETTER O WITH STROKE
+ '\xc9' # 0x71 -> LATIN CAPITAL LETTER E WITH ACUTE
+ '\xca' # 0x72 -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+ '\xcb' # 0x73 -> LATIN CAPITAL LETTER E WITH DIAERESIS
+ '\xc8' # 0x74 -> LATIN CAPITAL LETTER E WITH GRAVE
+ '\xcd' # 0x75 -> LATIN CAPITAL LETTER I WITH ACUTE
+ '\xce' # 0x76 -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+ '\xcf' # 0x77 -> LATIN CAPITAL LETTER I WITH DIAERESIS
+ '\xcc' # 0x78 -> LATIN CAPITAL LETTER I WITH GRAVE
+ '`' # 0x79 -> GRAVE ACCENT
+ ':' # 0x7A -> COLON
+ '#' # 0x7B -> NUMBER SIGN
+ '\xa7' # 0x7C -> SECTION SIGN
+ "'" # 0x7D -> APOSTROPHE
+ '=' # 0x7E -> EQUALS SIGN
+ '"' # 0x7F -> QUOTATION MARK
+ '\xd8' # 0x80 -> LATIN CAPITAL LETTER O WITH STROKE
+ 'a' # 0x81 -> LATIN SMALL LETTER A
+ 'b' # 0x82 -> LATIN SMALL LETTER B
+ 'c' # 0x83 -> LATIN SMALL LETTER C
+ 'd' # 0x84 -> LATIN SMALL LETTER D
+ 'e' # 0x85 -> LATIN SMALL LETTER E
+ 'f' # 0x86 -> LATIN SMALL LETTER F
+ 'g' # 0x87 -> LATIN SMALL LETTER G
+ 'h' # 0x88 -> LATIN SMALL LETTER H
+ 'i' # 0x89 -> LATIN SMALL LETTER I
+ '\xab' # 0x8A -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\xbb' # 0x8B -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+ '\xf0' # 0x8C -> LATIN SMALL LETTER ETH (Icelandic)
+ '\xfd' # 0x8D -> LATIN SMALL LETTER Y WITH ACUTE
+ '\xfe' # 0x8E -> LATIN SMALL LETTER THORN (Icelandic)
+ '\xb1' # 0x8F -> PLUS-MINUS SIGN
+ '\xb0' # 0x90 -> DEGREE SIGN
+ 'j' # 0x91 -> LATIN SMALL LETTER J
+ 'k' # 0x92 -> LATIN SMALL LETTER K
+ 'l' # 0x93 -> LATIN SMALL LETTER L
+ 'm' # 0x94 -> LATIN SMALL LETTER M
+ 'n' # 0x95 -> LATIN SMALL LETTER N
+ 'o' # 0x96 -> LATIN SMALL LETTER O
+ 'p' # 0x97 -> LATIN SMALL LETTER P
+ 'q' # 0x98 -> LATIN SMALL LETTER Q
+ 'r' # 0x99 -> LATIN SMALL LETTER R
+ '\xaa' # 0x9A -> FEMININE ORDINAL INDICATOR
+ '\xba' # 0x9B -> MASCULINE ORDINAL INDICATOR
+ '\xe6' # 0x9C -> LATIN SMALL LETTER AE
+ '\xb8' # 0x9D -> CEDILLA
+ '\xc6' # 0x9E -> LATIN CAPITAL LETTER AE
+ '\xa4' # 0x9F -> CURRENCY SIGN
+ '\xb5' # 0xA0 -> MICRO SIGN
+ '\xdf' # 0xA1 -> LATIN SMALL LETTER SHARP S (German)
+ 's' # 0xA2 -> LATIN SMALL LETTER S
+ 't' # 0xA3 -> LATIN SMALL LETTER T
+ 'u' # 0xA4 -> LATIN SMALL LETTER U
+ 'v' # 0xA5 -> LATIN SMALL LETTER V
+ 'w' # 0xA6 -> LATIN SMALL LETTER W
+ 'x' # 0xA7 -> LATIN SMALL LETTER X
+ 'y' # 0xA8 -> LATIN SMALL LETTER Y
+ 'z' # 0xA9 -> LATIN SMALL LETTER Z
+ '\xa1' # 0xAA -> INVERTED EXCLAMATION MARK
+ '\xbf' # 0xAB -> INVERTED QUESTION MARK
+ '\xd0' # 0xAC -> LATIN CAPITAL LETTER ETH (Icelandic)
+ '\xdd' # 0xAD -> LATIN CAPITAL LETTER Y WITH ACUTE
+ '\xde' # 0xAE -> LATIN CAPITAL LETTER THORN (Icelandic)
+ '\xae' # 0xAF -> REGISTERED SIGN
+ '\xa2' # 0xB0 -> CENT SIGN
+ '\xa3' # 0xB1 -> POUND SIGN
+ '\xa5' # 0xB2 -> YEN SIGN
+ '\xb7' # 0xB3 -> MIDDLE DOT
+ '\xa9' # 0xB4 -> COPYRIGHT SIGN
+ '@' # 0xB5 -> COMMERCIAL AT
+ '\xb6' # 0xB6 -> PILCROW SIGN
+ '\xbc' # 0xB7 -> VULGAR FRACTION ONE QUARTER
+ '\xbd' # 0xB8 -> VULGAR FRACTION ONE HALF
+ '\xbe' # 0xB9 -> VULGAR FRACTION THREE QUARTERS
+ '\xac' # 0xBA -> NOT SIGN
+ '|' # 0xBB -> VERTICAL LINE
+ '\u203e' # 0xBC -> OVERLINE
+ '\xa8' # 0xBD -> DIAERESIS
+ '\xb4' # 0xBE -> ACUTE ACCENT
+ '\xd7' # 0xBF -> MULTIPLICATION SIGN
+ '\xe4' # 0xC0 -> LATIN SMALL LETTER A WITH DIAERESIS
+ 'A' # 0xC1 -> LATIN CAPITAL LETTER A
+ 'B' # 0xC2 -> LATIN CAPITAL LETTER B
+ 'C' # 0xC3 -> LATIN CAPITAL LETTER C
+ 'D' # 0xC4 -> LATIN CAPITAL LETTER D
+ 'E' # 0xC5 -> LATIN CAPITAL LETTER E
+ 'F' # 0xC6 -> LATIN CAPITAL LETTER F
+ 'G' # 0xC7 -> LATIN CAPITAL LETTER G
+ 'H' # 0xC8 -> LATIN CAPITAL LETTER H
+ 'I' # 0xC9 -> LATIN CAPITAL LETTER I
+ '\xad' # 0xCA -> SOFT HYPHEN
+ '\xf4' # 0xCB -> LATIN SMALL LETTER O WITH CIRCUMFLEX
+ '\xa6' # 0xCC -> BROKEN BAR
+ '\xf2' # 0xCD -> LATIN SMALL LETTER O WITH GRAVE
+ '\xf3' # 0xCE -> LATIN SMALL LETTER O WITH ACUTE
+ '\xf5' # 0xCF -> LATIN SMALL LETTER O WITH TILDE
+ '\xfc' # 0xD0 -> LATIN SMALL LETTER U WITH DIAERESIS
+ 'J' # 0xD1 -> LATIN CAPITAL LETTER J
+ 'K' # 0xD2 -> LATIN CAPITAL LETTER K
+ 'L' # 0xD3 -> LATIN CAPITAL LETTER L
+ 'M' # 0xD4 -> LATIN CAPITAL LETTER M
+ 'N' # 0xD5 -> LATIN CAPITAL LETTER N
+ 'O' # 0xD6 -> LATIN CAPITAL LETTER O
+ 'P' # 0xD7 -> LATIN CAPITAL LETTER P
+ 'Q' # 0xD8 -> LATIN CAPITAL LETTER Q
+ 'R' # 0xD9 -> LATIN CAPITAL LETTER R
+ '\xb9' # 0xDA -> SUPERSCRIPT ONE
+ '\xfb' # 0xDB -> LATIN SMALL LETTER U WITH CIRCUMFLEX
+ '}' # 0xDC -> RIGHT CURLY BRACKET
+ '\xf9' # 0xDD -> LATIN SMALL LETTER U WITH GRAVE
+ '\xfa' # 0xDE -> LATIN SMALL LETTER U WITH ACUTE
+ '\xff' # 0xDF -> LATIN SMALL LETTER Y WITH DIAERESIS
+ '\xd6' # 0xE0 -> LATIN CAPITAL LETTER O WITH DIAERESIS
+ '\xf7' # 0xE1 -> DIVISION SIGN
+ 'S' # 0xE2 -> LATIN CAPITAL LETTER S
+ 'T' # 0xE3 -> LATIN CAPITAL LETTER T
+ 'U' # 0xE4 -> LATIN CAPITAL LETTER U
+ 'V' # 0xE5 -> LATIN CAPITAL LETTER V
+ 'W' # 0xE6 -> LATIN CAPITAL LETTER W
+ 'X' # 0xE7 -> LATIN CAPITAL LETTER X
+ 'Y' # 0xE8 -> LATIN CAPITAL LETTER Y
+ 'Z' # 0xE9 -> LATIN CAPITAL LETTER Z
+ '\xb2' # 0xEA -> SUPERSCRIPT TWO
+ '\xd4' # 0xEB -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+ '\\' # 0xEC -> REVERSE SOLIDUS
+ '\xd2' # 0xED -> LATIN CAPITAL LETTER O WITH GRAVE
+ '\xd3' # 0xEE -> LATIN CAPITAL LETTER O WITH ACUTE
+ '\xd5' # 0xEF -> LATIN CAPITAL LETTER O WITH TILDE
+ '0' # 0xF0 -> DIGIT ZERO
+ '1' # 0xF1 -> DIGIT ONE
+ '2' # 0xF2 -> DIGIT TWO
+ '3' # 0xF3 -> DIGIT THREE
+ '4' # 0xF4 -> DIGIT FOUR
+ '5' # 0xF5 -> DIGIT FIVE
+ '6' # 0xF6 -> DIGIT SIX
+ '7' # 0xF7 -> DIGIT SEVEN
+ '8' # 0xF8 -> DIGIT EIGHT
+ '9' # 0xF9 -> DIGIT NINE
+ '\xb3' # 0xFA -> SUPERSCRIPT THREE
+ '\xdb' # 0xFB -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+ ']' # 0xFC -> RIGHT SQUARE BRACKET
+ '\xd9' # 0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
+ '\xda' # 0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
+ '\x9f' # 0xFF -> APPLICATION PROGRAM COMMAND (APC)
+)
+
+### Encoding table
+encoding_table=codecs.charmap_build(decoding_table)
diff --git a/Lib/encodings/cp500.py b/Lib/encodings/cp500.py
index a975be7..5f61535 100644
--- a/Lib/encodings/cp500.py
+++ b/Lib/encodings/cp500.py
@@ -301,7 +301,6 @@ decoding_table = (
'\xd9' # 0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
'\xda' # 0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
'\x9f' # 0xFF -> CONTROL
- '\ufffe' ## Widen to UCS2 for optimization
)
### Encoding table
diff --git a/Lib/encodings/iso8859_1.py b/Lib/encodings/iso8859_1.py
index d9cc516..8cfc01f 100644
--- a/Lib/encodings/iso8859_1.py
+++ b/Lib/encodings/iso8859_1.py
@@ -301,7 +301,6 @@ decoding_table = (
'\xfd' # 0xFD -> LATIN SMALL LETTER Y WITH ACUTE
'\xfe' # 0xFE -> LATIN SMALL LETTER THORN (Icelandic)
'\xff' # 0xFF -> LATIN SMALL LETTER Y WITH DIAERESIS
- '\ufffe' ## Widen to UCS2 for optimization
)
### Encoding table
diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py
new file mode 100644
index 0000000..7cf6a4b
--- /dev/null
+++ b/Lib/ensurepip/__init__.py
@@ -0,0 +1,210 @@
+import os
+import os.path
+import pkgutil
+import sys
+import tempfile
+
+
+__all__ = ["version", "bootstrap"]
+
+
+_SETUPTOOLS_VERSION = "2.1"
+
+_PIP_VERSION = "1.5.4"
+
+# pip currently requires ssl support, so we try to provide a nicer
+# error message when that is missing (http://bugs.python.org/issue19744)
+_MISSING_SSL_MESSAGE = ("pip {} requires SSL/TLS".format(_PIP_VERSION))
+try:
+ import ssl
+except ImportError:
+ ssl = None
+ def _require_ssl_for_pip():
+ raise RuntimeError(_MISSING_SSL_MESSAGE)
+else:
+ def _require_ssl_for_pip():
+ pass
+
+_PROJECTS = [
+ ("setuptools", _SETUPTOOLS_VERSION),
+ ("pip", _PIP_VERSION),
+]
+
+
+def _run_pip(args, additional_paths=None):
+ # Add our bundled software to the sys.path so we can import it
+ if additional_paths is not None:
+ sys.path = additional_paths + sys.path
+
+ # Install the bundled software
+ import pip
+ pip.main(args)
+
+
+def version():
+ """
+ Returns a string specifying the bundled version of pip.
+ """
+ return _PIP_VERSION
+
+def _disable_pip_configuration_settings():
+ # We deliberately ignore all pip environment variables
+ # when invoking pip
+ # See http://bugs.python.org/issue19734 for details
+ keys_to_remove = [k for k in os.environ if k.startswith("PIP_")]
+ for k in keys_to_remove:
+ del os.environ[k]
+ # We also ignore the settings in the default pip configuration file
+ # See http://bugs.python.org/issue20053 for details
+ os.environ['PIP_CONFIG_FILE'] = os.devnull
+
+
+def bootstrap(*, root=None, upgrade=False, user=False,
+ altinstall=False, default_pip=False,
+ verbosity=0):
+ """
+ Bootstrap pip into the current Python installation (or the given root
+ directory).
+
+ Note that calling this function will alter both sys.path and os.environ.
+ """
+ if altinstall and default_pip:
+ raise ValueError("Cannot use altinstall and default_pip together")
+
+ _require_ssl_for_pip()
+ _disable_pip_configuration_settings()
+
+ # By default, installing pip and setuptools installs all of the
+ # following scripts (X.Y == running Python version):
+ #
+ # pip, pipX, pipX.Y, easy_install, easy_install-X.Y
+ #
+ # pip 1.5+ allows ensurepip to request that some of those be left out
+ if altinstall:
+ # omit pip, pipX and easy_install
+ os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
+ elif not default_pip:
+ # omit pip and easy_install
+ os.environ["ENSUREPIP_OPTIONS"] = "install"
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # Put our bundled wheels into a temporary directory and construct the
+ # additional paths that need added to sys.path
+ additional_paths = []
+ for project, version in _PROJECTS:
+ wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
+ whl = pkgutil.get_data(
+ "ensurepip",
+ "_bundled/{}".format(wheel_name),
+ )
+ with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
+ fp.write(whl)
+
+ additional_paths.append(os.path.join(tmpdir, wheel_name))
+
+ # Construct the arguments to be passed to the pip command
+ args = ["install", "--no-index", "--find-links", tmpdir]
+ if root:
+ args += ["--root", root]
+ if upgrade:
+ args += ["--upgrade"]
+ if user:
+ args += ["--user"]
+ if verbosity:
+ args += ["-" + "v" * verbosity]
+
+ _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+
+def _uninstall_helper(*, verbosity=0):
+ """Helper to support a clean default uninstall process on Windows
+
+ Note that calling this function may alter os.environ.
+ """
+ # Nothing to do if pip was never installed, or has been removed
+ try:
+ import pip
+ except ImportError:
+ return
+
+ # If the pip version doesn't match the bundled one, leave it alone
+ if pip.__version__ != _PIP_VERSION:
+ msg = ("ensurepip will only uninstall a matching version "
+ "({!r} installed, {!r} bundled)")
+ print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+ return
+
+ _require_ssl_for_pip()
+ _disable_pip_configuration_settings()
+
+ # Construct the arguments to be passed to the pip command
+ args = ["uninstall", "-y"]
+ if verbosity:
+ args += ["-" + "v" * verbosity]
+
+ _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+
+
+def _main(argv=None):
+ if ssl is None:
+ print("Ignoring ensurepip failure: {}".format(_MISSING_SSL_MESSAGE),
+ file=sys.stderr)
+ return
+
+ import argparse
+ parser = argparse.ArgumentParser(prog="python -m ensurepip")
+ parser.add_argument(
+ "--version",
+ action="version",
+ version="pip {}".format(version()),
+ help="Show the version of pip that is bundled with this Python.",
+ )
+ parser.add_argument(
+ "-v", "--verbose",
+ action="count",
+ default=0,
+ dest="verbosity",
+ help=("Give more output. Option is additive, and can be used up to 3 "
+ "times."),
+ )
+ parser.add_argument(
+ "-U", "--upgrade",
+ action="store_true",
+ default=False,
+ help="Upgrade pip and dependencies, even if already installed.",
+ )
+ parser.add_argument(
+ "--user",
+ action="store_true",
+ default=False,
+ help="Install using the user scheme.",
+ )
+ parser.add_argument(
+ "--root",
+ default=None,
+ help="Install everything relative to this alternate root directory.",
+ )
+ parser.add_argument(
+ "--altinstall",
+ action="store_true",
+ default=False,
+ help=("Make an alternate install, installing only the X.Y versioned"
+ "scripts (Default: pipX, pipX.Y, easy_install-X.Y)"),
+ )
+ parser.add_argument(
+ "--default-pip",
+ action="store_true",
+ default=False,
+ help=("Make a default pip install, installing the unqualified pip "
+ "and easy_install in addition to the versioned scripts"),
+ )
+
+ args = parser.parse_args(argv)
+
+ bootstrap(
+ root=args.root,
+ upgrade=args.upgrade,
+ user=args.user,
+ verbosity=args.verbosity,
+ altinstall=args.altinstall,
+ default_pip=args.default_pip,
+ )
diff --git a/Lib/ensurepip/__main__.py b/Lib/ensurepip/__main__.py
new file mode 100644
index 0000000..77527d7
--- /dev/null
+++ b/Lib/ensurepip/__main__.py
@@ -0,0 +1,4 @@
+import ensurepip
+
+if __name__ == "__main__":
+ ensurepip._main()
diff --git a/Lib/ensurepip/_bundled/pip-1.5.4-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-1.5.4-py2.py3-none-any.whl
new file mode 100644
index 0000000..e07d476
--- /dev/null
+++ b/Lib/ensurepip/_bundled/pip-1.5.4-py2.py3-none-any.whl
Binary files differ
diff --git a/Lib/ensurepip/_bundled/setuptools-2.1-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-2.1-py2.py3-none-any.whl
new file mode 100644
index 0000000..ed77b59
--- /dev/null
+++ b/Lib/ensurepip/_bundled/setuptools-2.1-py2.py3-none-any.whl
Binary files differ
diff --git a/Lib/ensurepip/_uninstall.py b/Lib/ensurepip/_uninstall.py
new file mode 100644
index 0000000..750365e
--- /dev/null
+++ b/Lib/ensurepip/_uninstall.py
@@ -0,0 +1,30 @@
+"""Basic pip uninstallation support, helper for the Windows uninstaller"""
+
+import argparse
+import ensurepip
+
+
+def _main(argv=None):
+ parser = argparse.ArgumentParser(prog="python -m ensurepip._uninstall")
+ parser.add_argument(
+ "--version",
+ action="version",
+ version="pip {}".format(ensurepip.version()),
+ help="Show the version of pip this will attempt to uninstall.",
+ )
+ parser.add_argument(
+ "-v", "--verbose",
+ action="count",
+ default=0,
+ dest="verbosity",
+ help=("Give more output. Option is additive, and can be used up to 3 "
+ "times."),
+ )
+
+ args = parser.parse_args(argv)
+
+ ensurepip._uninstall_helper(verbosity=args.verbosity)
+
+
+if __name__ == "__main__":
+ _main()
diff --git a/Lib/enum.py b/Lib/enum.py
new file mode 100644
index 0000000..844a956
--- /dev/null
+++ b/Lib/enum.py
@@ -0,0 +1,525 @@
+import sys
+from collections import OrderedDict
+from types import MappingProxyType, DynamicClassAttribute
+
+__all__ = ['Enum', 'IntEnum', 'unique']
+
+
+def _is_descriptor(obj):
+ """Returns True if obj is a descriptor, False otherwise."""
+ return (
+ hasattr(obj, '__get__') or
+ hasattr(obj, '__set__') or
+ hasattr(obj, '__delete__'))
+
+
+def _is_dunder(name):
+ """Returns True if a __dunder__ name, False otherwise."""
+ return (name[:2] == name[-2:] == '__' and
+ name[2:3] != '_' and
+ name[-3:-2] != '_' and
+ len(name) > 4)
+
+
+def _is_sunder(name):
+ """Returns True if a _sunder_ name, False otherwise."""
+ return (name[0] == name[-1] == '_' and
+ name[1:2] != '_' and
+ name[-2:-1] != '_' and
+ len(name) > 2)
+
+
+def _make_class_unpicklable(cls):
+ """Make the given class un-picklable."""
+ def _break_on_call_reduce(self, proto):
+ raise TypeError('%r cannot be pickled' % self)
+ cls.__reduce_ex__ = _break_on_call_reduce
+ cls.__module__ = '<unknown>'
+
+
+class _EnumDict(dict):
+ """Track enum member order and ensure member names are not reused.
+
+ EnumMeta will use the names found in self._member_names as the
+ enumeration member names.
+
+ """
+ def __init__(self):
+ super().__init__()
+ self._member_names = []
+
+ def __setitem__(self, key, value):
+ """Changes anything not dundered or not a descriptor.
+
+ If an enum member name is used twice, an error is raised; duplicate
+ values are not checked for.
+
+ Single underscore (sunder) names are reserved.
+
+ """
+ if _is_sunder(key):
+ raise ValueError('_names_ are reserved for future Enum use')
+ elif _is_dunder(key):
+ pass
+ elif key in self._member_names:
+ # descriptor overwriting an enum?
+ raise TypeError('Attempted to reuse key: %r' % key)
+ elif not _is_descriptor(value):
+ if key in self:
+ # enum overwriting a descriptor?
+ raise TypeError('Key already defined as: %r' % self[key])
+ self._member_names.append(key)
+ super().__setitem__(key, value)
+
+
+
+# Dummy value for Enum as EnumMeta explicitly checks for it, but of course
+# until EnumMeta finishes running the first time the Enum class doesn't exist.
+# This is also why there are checks in EnumMeta like `if Enum is not None`
+Enum = None
+
+
+class EnumMeta(type):
+ """Metaclass for Enum"""
+ @classmethod
+ def __prepare__(metacls, cls, bases):
+ return _EnumDict()
+
+ def __new__(metacls, cls, bases, classdict):
+ # an Enum class is final once enumeration items have been defined; it
+ # cannot be mixed with other types (int, float, etc.) if it has an
+ # inherited __new__ unless a new __new__ is defined (or the resulting
+ # class will fail).
+ member_type, first_enum = metacls._get_mixins_(bases)
+ __new__, save_new, use_args = metacls._find_new_(classdict, member_type,
+ first_enum)
+
+ # save enum items into separate mapping so they don't get baked into
+ # the new class
+ members = {k: classdict[k] for k in classdict._member_names}
+ for name in classdict._member_names:
+ del classdict[name]
+
+ # check for illegal enum names (any others?)
+ invalid_names = set(members) & {'mro', }
+ if invalid_names:
+ raise ValueError('Invalid enum member name: {0}'.format(
+ ','.join(invalid_names)))
+
+ # create our new Enum type
+ enum_class = super().__new__(metacls, cls, bases, classdict)
+ enum_class._member_names_ = [] # names in definition order
+ enum_class._member_map_ = OrderedDict() # name->value map
+ enum_class._member_type_ = member_type
+
+ # Reverse value->name map for hashable values.
+ enum_class._value2member_map_ = {}
+
+ # If a custom type is mixed into the Enum, and it does not know how
+ # to pickle itself, pickle.dumps will succeed but pickle.loads will
+ # fail. Rather than have the error show up later and possibly far
+ # from the source, sabotage the pickle protocol for this class so
+ # that pickle.dumps also fails.
+ #
+ # However, if the new class implements its own __reduce_ex__, do not
+ # sabotage -- it's on them to make sure it works correctly. We use
+ # __reduce_ex__ instead of any of the others as it is preferred by
+ # pickle over __reduce__, and it handles all pickle protocols.
+ if '__reduce_ex__' not in classdict:
+ if member_type is not object:
+ methods = ('__getnewargs_ex__', '__getnewargs__',
+ '__reduce_ex__', '__reduce__')
+ if not any(m in member_type.__dict__ for m in methods):
+ _make_class_unpicklable(enum_class)
+
+ # instantiate them, checking for duplicates as we go
+ # we instantiate first instead of checking for duplicates first in case
+ # a custom __new__ is doing something funky with the values -- such as
+ # auto-numbering ;)
+ for member_name in classdict._member_names:
+ value = members[member_name]
+ if not isinstance(value, tuple):
+ args = (value, )
+ else:
+ args = value
+ if member_type is tuple: # special case for tuple enums
+ args = (args, ) # wrap it one more time
+ if not use_args:
+ enum_member = __new__(enum_class)
+ if not hasattr(enum_member, '_value_'):
+ enum_member._value_ = value
+ else:
+ enum_member = __new__(enum_class, *args)
+ if not hasattr(enum_member, '_value_'):
+ enum_member._value_ = member_type(*args)
+ value = enum_member._value_
+ enum_member._name_ = member_name
+ enum_member.__objclass__ = enum_class
+ enum_member.__init__(*args)
+ # If another member with the same value was already defined, the
+ # new member becomes an alias to the existing one.
+ for name, canonical_member in enum_class._member_map_.items():
+ if canonical_member.value == enum_member._value_:
+ enum_member = canonical_member
+ break
+ else:
+ # Aliases don't appear in member names (only in __members__).
+ enum_class._member_names_.append(member_name)
+ enum_class._member_map_[member_name] = enum_member
+ try:
+ # This may fail if value is not hashable. We can't add the value
+ # to the map, and by-value lookups for this value will be
+ # linear.
+ enum_class._value2member_map_[value] = enum_member
+ except TypeError:
+ pass
+
+ # double check that repr and friends are not the mixin's or various
+ # things break (such as pickle)
+ for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):
+ class_method = getattr(enum_class, name)
+ obj_method = getattr(member_type, name, None)
+ enum_method = getattr(first_enum, name, None)
+ if obj_method is not None and obj_method is class_method:
+ setattr(enum_class, name, enum_method)
+
+ # replace any other __new__ with our own (as long as Enum is not None,
+ # anyway) -- again, this is to support pickle
+ if Enum is not None:
+ # if the user defined their own __new__, save it before it gets
+ # clobbered in case they subclass later
+ if save_new:
+ enum_class.__new_member__ = __new__
+ enum_class.__new__ = Enum.__new__
+ return enum_class
+
+ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None):
+ """Either returns an existing member, or creates a new enum class.
+
+ This method is used both when an enum class is given a value to match
+ to an enumeration member (i.e. Color(3)) and for the functional API
+ (i.e. Color = Enum('Color', names='red green blue')).
+
+ When used for the functional API:
+
+ `value` will be the name of the new class.
+
+ `names` should be either a string of white-space/comma delimited names
+ (values will start at 1), or an iterator/mapping of name, value pairs.
+
+ `module` should be set to the module this class is being created in;
+ if it is not set, an attempt to find that module will be made, but if
+ it fails the class will not be picklable.
+
+ `qualname` should be set to the actual location this class can be found
+ at in its module; by default it is set to the global scope. If this is
+ not correct, unpickling will fail in some circumstances.
+
+ `type`, if set, will be mixed in as the first base class.
+
+ """
+ if names is None: # simple value lookup
+ return cls.__new__(cls, value)
+ # otherwise, functional API: we're creating a new Enum type
+ return cls._create_(value, names, module=module, qualname=qualname, type=type)
+
+ def __contains__(cls, member):
+ return isinstance(member, cls) and member.name in cls._member_map_
+
+ def __delattr__(cls, attr):
+ # nicer error message when someone tries to delete an attribute
+ # (see issue19025).
+ if attr in cls._member_map_:
+ raise AttributeError(
+ "%s: cannot delete Enum member." % cls.__name__)
+ super().__delattr__(attr)
+
+ def __dir__(self):
+ return (['__class__', '__doc__', '__members__', '__module__'] +
+ self._member_names_)
+
+ def __getattr__(cls, name):
+ """Return the enum member matching `name`
+
+ We use __getattr__ instead of descriptors or inserting into the enum
+ class' __dict__ in order to support `name` and `value` being both
+ properties for enum members (which live in the class' __dict__) and
+ enum members themselves.
+
+ """
+ if _is_dunder(name):
+ raise AttributeError(name)
+ try:
+ return cls._member_map_[name]
+ except KeyError:
+ raise AttributeError(name) from None
+
+ def __getitem__(cls, name):
+ return cls._member_map_[name]
+
+ def __iter__(cls):
+ return (cls._member_map_[name] for name in cls._member_names_)
+
+ def __len__(cls):
+ return len(cls._member_names_)
+
+ @property
+ def __members__(cls):
+ """Returns a mapping of member name->value.
+
+ This mapping lists all enum members, including aliases. Note that this
+ is a read-only view of the internal mapping.
+
+ """
+ return MappingProxyType(cls._member_map_)
+
+ def __repr__(cls):
+ return "<enum %r>" % cls.__name__
+
+ def __reversed__(cls):
+ return (cls._member_map_[name] for name in reversed(cls._member_names_))
+
+ def __setattr__(cls, name, value):
+ """Block attempts to reassign Enum members.
+
+ A simple assignment to the class namespace only changes one of the
+ several possible ways to get an Enum member from the Enum class,
+ resulting in an inconsistent Enumeration.
+
+ """
+ member_map = cls.__dict__.get('_member_map_', {})
+ if name in member_map:
+ raise AttributeError('Cannot reassign members.')
+ super().__setattr__(name, value)
+
+ def _create_(cls, class_name, names=None, *, module=None, qualname=None, type=None):
+ """Convenience method to create a new Enum class.
+
+ `names` can be:
+
+ * A string containing member names, separated either with spaces or
+ commas. Values are auto-numbered from 1.
+ * An iterable of member names. Values are auto-numbered from 1.
+ * An iterable of (member name, value) pairs.
+ * A mapping of member name -> value.
+
+ """
+ metacls = cls.__class__
+ bases = (cls, ) if type is None else (type, cls)
+ classdict = metacls.__prepare__(class_name, bases)
+
+ # special processing needed for names?
+ if isinstance(names, str):
+ names = names.replace(',', ' ').split()
+ if isinstance(names, (tuple, list)) and isinstance(names[0], str):
+ names = [(e, i) for (i, e) in enumerate(names, 1)]
+
+ # Here, names is either an iterable of (name, value) or a mapping.
+ for item in names:
+ if isinstance(item, str):
+ member_name, member_value = item, names[item]
+ else:
+ member_name, member_value = item
+ classdict[member_name] = member_value
+ enum_class = metacls.__new__(metacls, class_name, bases, classdict)
+
+ # TODO: replace the frame hack if a blessed way to know the calling
+ # module is ever developed
+ if module is None:
+ try:
+ module = sys._getframe(2).f_globals['__name__']
+ except (AttributeError, ValueError) as exc:
+ pass
+ if module is None:
+ _make_class_unpicklable(enum_class)
+ else:
+ enum_class.__module__ = module
+ if qualname is not None:
+ enum_class.__qualname__ = qualname
+
+ return enum_class
+
+ @staticmethod
+ def _get_mixins_(bases):
+ """Returns the type for creating enum members, and the first inherited
+ enum class.
+
+ bases: the tuple of bases that was given to __new__
+
+ """
+ if not bases:
+ return object, Enum
+
+ # double check that we are not subclassing a class with existing
+ # enumeration members; while we're at it, see if any other data
+ # type has been mixed in so we can use the correct __new__
+ member_type = first_enum = None
+ for base in bases:
+ if (base is not Enum and
+ issubclass(base, Enum) and
+ base._member_names_):
+ raise TypeError("Cannot extend enumerations")
+ # base is now the last base in bases
+ if not issubclass(base, Enum):
+ raise TypeError("new enumerations must be created as "
+ "`ClassName([mixin_type,] enum_type)`")
+
+ # get correct mix-in type (either mix-in type of Enum subclass, or
+ # first base if last base is Enum)
+ if not issubclass(bases[0], Enum):
+ member_type = bases[0] # first data type
+ first_enum = bases[-1] # enum type
+ else:
+ for base in bases[0].__mro__:
+ # most common: (IntEnum, int, Enum, object)
+ # possible: (<Enum 'AutoIntEnum'>, <Enum 'IntEnum'>,
+ # <class 'int'>, <Enum 'Enum'>,
+ # <class 'object'>)
+ if issubclass(base, Enum):
+ if first_enum is None:
+ first_enum = base
+ else:
+ if member_type is None:
+ member_type = base
+
+ return member_type, first_enum
+
+ @staticmethod
+ def _find_new_(classdict, member_type, first_enum):
+ """Returns the __new__ to be used for creating the enum members.
+
+ classdict: the class dictionary given to __new__
+ member_type: the data type whose __new__ will be used by default
+ first_enum: enumeration to check for an overriding __new__
+
+ """
+ # now find the correct __new__, checking to see of one was defined
+ # by the user; also check earlier enum classes in case a __new__ was
+ # saved as __new_member__
+ __new__ = classdict.get('__new__', None)
+
+ # should __new__ be saved as __new_member__ later?
+ save_new = __new__ is not None
+
+ if __new__ is None:
+ # check all possibles for __new_member__ before falling back to
+ # __new__
+ for method in ('__new_member__', '__new__'):
+ for possible in (member_type, first_enum):
+ target = getattr(possible, method, None)
+ if target not in {
+ None,
+ None.__new__,
+ object.__new__,
+ Enum.__new__,
+ }:
+ __new__ = target
+ break
+ if __new__ is not None:
+ break
+ else:
+ __new__ = object.__new__
+
+ # if a non-object.__new__ is used then whatever value/tuple was
+ # assigned to the enum member name will be passed to __new__ and to the
+ # new enum member's __init__
+ if __new__ is object.__new__:
+ use_args = False
+ else:
+ use_args = True
+
+ return __new__, save_new, use_args
+
+
+class Enum(metaclass=EnumMeta):
+ """Generic enumeration.
+
+ Derive from this class to define new enumerations.
+
+ """
+ def __new__(cls, value):
+ # all enum instances are actually created during class construction
+ # without calling this method; this method is called by the metaclass'
+ # __call__ (i.e. Color(3) ), and by pickle
+ if type(value) is cls:
+ # For lookups like Color(Color.red)
+ return value
+ # by-value search for a matching enum member
+ # see if it's in the reverse mapping (for hashable values)
+ try:
+ if value in cls._value2member_map_:
+ return cls._value2member_map_[value]
+ except TypeError:
+ # not there, now do long search -- O(n) behavior
+ for member in cls._member_map_.values():
+ if member.value == value:
+ return member
+ raise ValueError("%s is not a valid %s" % (value, cls.__name__))
+
+ def __repr__(self):
+ return "<%s.%s: %r>" % (
+ self.__class__.__name__, self._name_, self._value_)
+
+ def __str__(self):
+ return "%s.%s" % (self.__class__.__name__, self._name_)
+
+ def __dir__(self):
+ added_behavior = [m for m in self.__class__.__dict__ if m[0] != '_']
+ return (['__class__', '__doc__', '__module__', 'name', 'value'] +
+ added_behavior)
+
+ def __format__(self, format_spec):
+ # mixed-in Enums should use the mixed-in type's __format__, otherwise
+ # we can get strange results with the Enum name showing up instead of
+ # the value
+
+ # pure Enum branch
+ if self._member_type_ is object:
+ cls = str
+ val = str(self)
+ # mix-in branch
+ else:
+ cls = self._member_type_
+ val = self.value
+ return cls.__format__(val, format_spec)
+
+ def __hash__(self):
+ return hash(self._name_)
+
+ def __reduce_ex__(self, proto):
+ return self.__class__, (self._value_, )
+
+ # DynamicClassAttribute is used to provide access to the `name` and
+ # `value` properties of enum members while keeping some measure of
+ # protection from modification, while still allowing for an enumeration
+ # to have members named `name` and `value`. This works because enumeration
+ # members are not set directly on the enum class -- __getattr__ is
+ # used to look them up.
+
+ @DynamicClassAttribute
+ def name(self):
+ """The name of the Enum member."""
+ return self._name_
+
+ @DynamicClassAttribute
+ def value(self):
+ """The value of the Enum member."""
+ return self._value_
+
+
+class IntEnum(int, Enum):
+ """Enum where members are also (and must be) ints"""
+
+
+def unique(enumeration):
+ """Class decorator for enumerations ensuring unique member values."""
+ duplicates = []
+ for name, member in enumeration.__members__.items():
+ if name != member.name:
+ duplicates.append((name, member.name))
+ if duplicates:
+ alias_details = ', '.join(
+ ["%s -> %s" % (alias, name) for (alias, name) in duplicates])
+ raise ValueError('duplicate values found in %r: %s' %
+ (enumeration, alias_details))
+ return enumeration
diff --git a/Lib/filecmp.py b/Lib/filecmp.py
index f5cea1d..dd5a2af 100644
--- a/Lib/filecmp.py
+++ b/Lib/filecmp.py
@@ -6,6 +6,7 @@ Classes:
Functions:
cmp(f1, f2, shallow=True) -> int
cmpfiles(a, b, common) -> ([], [], [])
+ clear_cache()
"""
@@ -13,11 +14,18 @@ import os
import stat
from itertools import filterfalse
-__all__ = ["cmp", "dircmp", "cmpfiles"]
+__all__ = ['clear_cache', 'cmp', 'dircmp', 'cmpfiles', 'DEFAULT_IGNORES']
_cache = {}
BUFSIZE = 8*1024
+DEFAULT_IGNORES = [
+ 'RCS', 'CVS', 'tags', '.git', '.hg', '.bzr', '_darcs', '__pycache__']
+
+def clear_cache():
+ """Clear the filecmp cache."""
+ _cache.clear()
+
def cmp(f1, f2, shallow=True):
"""Compare two files.
@@ -35,7 +43,8 @@ def cmp(f1, f2, shallow=True):
True if the files are the same, False otherwise.
This function uses a cache for past comparisons and the results,
- with a cache invalidation mechanism relying on stale signatures.
+ with cache entries invalidated if their stat information
+ changes. The cache may be cleared by calling clear_cache().
"""
@@ -52,7 +61,7 @@ def cmp(f1, f2, shallow=True):
if outcome is None:
outcome = _do_cmp(f1, f2)
if len(_cache) > 100: # limit the maximum size of the cache
- _cache.clear()
+ clear_cache()
_cache[f1, f2, s1, s2] = outcome
return outcome
@@ -80,7 +89,7 @@ class dircmp:
dircmp(a, b, ignore=None, hide=None)
A and B are directories.
IGNORE is a list of names to ignore,
- defaults to ['RCS', 'CVS', 'tags'].
+ defaults to DEFAULT_IGNORES.
HIDE is a list of names to hide,
defaults to [os.curdir, os.pardir].
@@ -116,7 +125,7 @@ class dircmp:
else:
self.hide = hide
if ignore is None:
- self.ignore = ['RCS', 'CVS', 'tags'] # Names ignored in comparison
+ self.ignore = DEFAULT_IGNORES
else:
self.ignore = ignore
@@ -147,12 +156,12 @@ class dircmp:
ok = 1
try:
a_stat = os.stat(a_path)
- except os.error as why:
+ except OSError as why:
# print('Can\'t stat', a_path, ':', why.args[1])
ok = 0
try:
b_stat = os.stat(b_path)
- except os.error as why:
+ except OSError as why:
# print('Can\'t stat', b_path, ':', why.args[1])
ok = 0
@@ -268,7 +277,7 @@ def cmpfiles(a, b, common, shallow=True):
def _cmp(a, b, sh, abs=abs, cmp=cmp):
try:
return not abs(cmp(a, b, sh))
- except os.error:
+ except OSError:
return 2
diff --git a/Lib/fileinput.py b/Lib/fileinput.py
index 879a0fd..de29518 100644
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -30,7 +30,7 @@ pertaining to the last line read; nextfile() has no effect.
All files are opened in text mode by default, you can override this by
setting the mode parameter to input() or FileInput.__init__().
-If an I/O error occurs during opening or reading a file, the IOError
+If an I/O error occurs during opening or reading a file, the OSError
exception is raised.
If sys.stdin is used more than once, the second and further use will
@@ -222,6 +222,10 @@ class FileInput:
if mode not in ('r', 'rU', 'U', 'rb'):
raise ValueError("FileInput opening mode must be one of "
"'r', 'rU', 'U' and 'rb'")
+ if 'U' in mode:
+ import warnings
+ warnings.warn("'U' mode is deprecated",
+ DeprecationWarning, 2)
self._mode = mode
if openhook:
if inplace:
@@ -322,9 +326,11 @@ class FileInput:
if self._inplace:
self._backupfilename = (
self._filename + (self._backup or ".bak"))
- try: os.unlink(self._backupfilename)
- except os.error: pass
- # The next few lines may raise IOError
+ try:
+ os.unlink(self._backupfilename)
+ except OSError:
+ pass
+ # The next few lines may raise OSError
os.rename(self._filename, self._backupfilename)
self._file = open(self._backupfilename, self._mode)
try:
@@ -346,7 +352,7 @@ class FileInput:
self._savestdout = sys.stdout
sys.stdout = self._output
else:
- # This may raise IOError
+ # This may raise OSError
if self._openhook:
self._file = self._openhook(self._filename, self._mode)
else:
diff --git a/Lib/formatter.py b/Lib/formatter.py
index 60e60f1..d8cca52 100644
--- a/Lib/formatter.py
+++ b/Lib/formatter.py
@@ -19,6 +19,9 @@ manage and inserting data into the output.
"""
import sys
+import warnings
+warnings.warn('the formatter module is deprecated and will be removed in '
+ 'Python 3.6', PendingDeprecationWarning)
AS_IS = None
diff --git a/Lib/fractions.py b/Lib/fractions.py
index 8be52d2..79e83ff 100644
--- a/Lib/fractions.py
+++ b/Lib/fractions.py
@@ -182,8 +182,10 @@ class Fraction(numbers.Rational):
elif not isinstance(f, float):
raise TypeError("%s.from_float() only takes floats, not %r (%s)" %
(cls.__name__, f, type(f).__name__))
- if math.isnan(f) or math.isinf(f):
- raise TypeError("Cannot convert %r to %s." % (f, cls.__name__))
+ if math.isnan(f):
+ raise ValueError("Cannot convert %r to %s." % (f, cls.__name__))
+ if math.isinf(f):
+ raise OverflowError("Cannot convert %r to %s." % (f, cls.__name__))
return cls(*f.as_integer_ratio())
@classmethod
@@ -196,9 +198,11 @@ class Fraction(numbers.Rational):
raise TypeError(
"%s.from_decimal() only takes Decimals, not %r (%s)" %
(cls.__name__, dec, type(dec).__name__))
- if not dec.is_finite():
- # Catches infinities and nans.
- raise TypeError("Cannot convert %s to %s." % (dec, cls.__name__))
+ if dec.is_infinite():
+ raise OverflowError(
+ "Cannot convert %s to %s." % (dec, cls.__name__))
+ if dec.is_nan():
+ raise ValueError("Cannot convert %s to %s." % (dec, cls.__name__))
sign, digits, exp = dec.as_tuple()
digits = int(''.join(map(str, digits)))
if sign:
diff --git a/Lib/ftplib.py b/Lib/ftplib.py
index 5e75e6d..c83be2b 100644
--- a/Lib/ftplib.py
+++ b/Lib/ftplib.py
@@ -39,9 +39,10 @@ python ftplib.py -d localhost -l -p -l
import os
import sys
import socket
+import warnings
from socket import _GLOBAL_DEFAULT_TIMEOUT
-__all__ = ["FTP","Netrc"]
+__all__ = ["FTP", "Netrc"]
# Magic number from <socket.h>
MSG_OOB = 0x1 # Process data out of band
@@ -63,7 +64,7 @@ class error_proto(Error): pass # response does not begin with [1-5]
# All exceptions (hopefully) that may be raised here and that aren't
# (always) programming errors on our side
-all_errors = (Error, IOError, EOFError)
+all_errors = (Error, OSError, EOFError)
# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
@@ -126,7 +127,7 @@ class FTP:
if self.sock is not None:
try:
self.quit()
- except (socket.error, EOFError):
+ except (OSError, EOFError):
pass
finally:
if self.sock is not None:
@@ -136,6 +137,7 @@ class FTP:
'''Connect to host. Arguments are:
- host: hostname to connect to (string, default previous host)
- port: port to connect to (integer, default previous port)
+ - timeout: the timeout to set against the ftp socket(s)
- source_address: a 2-tuple (host, port) for the socket to bind
to as its source address before connecting.
'''
@@ -186,7 +188,8 @@ class FTP:
# Internal: send one line to the server, appending CRLF
def putline(self, line):
line = line + CRLF
- if self.debugging > 1: print('*put*', self.sanitize(line))
+ if self.debugging > 1:
+ print('*put*', self.sanitize(line))
self.sock.sendall(line.encode(self.encoding))
# Internal: send one command to the server (through putline())
@@ -202,9 +205,12 @@ class FTP:
raise Error("got more than %d bytes" % self.maxline)
if self.debugging > 1:
print('*get*', self.sanitize(line))
- if not line: raise EOFError
- if line[-2:] == CRLF: line = line[:-2]
- elif line[-1:] in CRLF: line = line[:-1]
+ if not line:
+ raise EOFError
+ if line[-2:] == CRLF:
+ line = line[:-2]
+ elif line[-1:] in CRLF:
+ line = line[:-1]
return line
# Internal: get a response from the server, which may possibly
@@ -227,7 +233,8 @@ class FTP:
# Raise various errors if the response indicates an error
def getresp(self):
resp = self.getmultiline()
- if self.debugging: print('*resp*', self.sanitize(resp))
+ if self.debugging:
+ print('*resp*', self.sanitize(resp))
self.lastresp = resp[:3]
c = resp[:1]
if c in {'1', '2', '3'}:
@@ -251,7 +258,8 @@ class FTP:
IP and Synch; that doesn't seem to work with the servers I've
tried. Instead, just send the ABOR command as OOB data.'''
line = b'ABOR' + B_CRLF
- if self.debugging > 1: print('*put urgent*', self.sanitize(line))
+ if self.debugging > 1:
+ print('*put urgent*', self.sanitize(line))
self.sock.sendall(line, MSG_OOB)
resp = self.getmultiline()
if resp[:3] not in {'426', '225', '226'}:
@@ -300,7 +308,7 @@ class FTP:
try:
sock = socket.socket(af, socktype, proto)
sock.bind(sa)
- except socket.error as _:
+ except OSError as _:
err = _
if sock:
sock.close()
@@ -311,8 +319,7 @@ class FTP:
if err is not None:
raise err
else:
- raise socket.error("getaddrinfo returns an empty list")
- raise socket.error(msg)
+ raise OSError("getaddrinfo returns an empty list")
sock.listen(1)
port = sock.getsockname()[1] # Get proper port
host = self.sock.getsockname()[0] # Get proper host
@@ -392,9 +399,12 @@ class FTP:
def login(self, user = '', passwd = '', acct = ''):
'''Login, default anonymous.'''
- if not user: user = 'anonymous'
- if not passwd: passwd = ''
- if not acct: acct = ''
+ if not user:
+ user = 'anonymous'
+ if not passwd:
+ passwd = ''
+ if not acct:
+ acct = ''
if user == 'anonymous' and passwd in {'', '-'}:
# If there is no anonymous ftp password specified
# then we'll just use anonymous@
@@ -405,8 +415,10 @@ class FTP:
# host or country.
passwd = passwd + 'anonymous@'
resp = self.sendcmd('USER ' + user)
- if resp[0] == '3': resp = self.sendcmd('PASS ' + passwd)
- if resp[0] == '3': resp = self.sendcmd('ACCT ' + acct)
+ if resp[0] == '3':
+ resp = self.sendcmd('PASS ' + passwd)
+ if resp[0] == '3':
+ resp = self.sendcmd('ACCT ' + acct)
if resp[0] != '2':
raise error_reply(resp)
return resp
@@ -432,6 +444,9 @@ class FTP:
if not data:
break
callback(data)
+ # shutdown ssl layer
+ if _SSLSocket is not None and isinstance(conn, _SSLSocket):
+ conn.unwrap()
return self.voidresp()
def retrlines(self, cmd, callback = None):
@@ -446,7 +461,8 @@ class FTP:
Returns:
The response code.
"""
- if callback is None: callback = print_line
+ if callback is None:
+ callback = print_line
resp = self.sendcmd('TYPE A')
with self.transfercmd(cmd) as conn, \
conn.makefile('r', encoding=self.encoding) as fp:
@@ -454,7 +470,8 @@ class FTP:
line = fp.readline(self.maxline + 1)
if len(line) > self.maxline:
raise Error("got more than %d bytes" % self.maxline)
- if self.debugging > 2: print('*retr*', repr(line))
+ if self.debugging > 2:
+ print('*retr*', repr(line))
if not line:
break
if line[-2:] == CRLF:
@@ -462,6 +479,9 @@ class FTP:
elif line[-1:] == '\n':
line = line[:-1]
callback(line)
+ # shutdown ssl layer
+ if _SSLSocket is not None and isinstance(conn, _SSLSocket):
+ conn.unwrap()
return self.voidresp()
def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None):
@@ -483,9 +503,14 @@ class FTP:
with self.transfercmd(cmd, rest) as conn:
while 1:
buf = fp.read(blocksize)
- if not buf: break
+ if not buf:
+ break
conn.sendall(buf)
- if callback: callback(buf)
+ if callback:
+ callback(buf)
+ # shutdown ssl layer
+ if _SSLSocket is not None and isinstance(conn, _SSLSocket):
+ conn.unwrap()
return self.voidresp()
def storlines(self, cmd, fp, callback=None):
@@ -506,12 +531,17 @@ class FTP:
buf = fp.readline(self.maxline + 1)
if len(buf) > self.maxline:
raise Error("got more than %d bytes" % self.maxline)
- if not buf: break
+ if not buf:
+ break
if buf[-2:] != B_CRLF:
if buf[-1] in B_CRLF: buf = buf[:-1]
buf = buf + B_CRLF
conn.sendall(buf)
- if callback: callback(buf)
+ if callback:
+ callback(buf)
+ # shutdown ssl layer
+ if _SSLSocket is not None and isinstance(conn, _SSLSocket):
+ conn.unwrap()
return self.voidresp()
def acct(self, password):
@@ -646,8 +676,10 @@ class FTP:
try:
import ssl
except ImportError:
- pass
+ _SSLSocket = None
else:
+ _SSLSocket = ssl.SSLSocket
+
class FTP_TLS(FTP):
'''A FTP subclass which adds TLS support to FTP as described
in RFC-4217.
@@ -694,6 +726,10 @@ else:
"exclusive")
self.keyfile = keyfile
self.certfile = certfile
+ if context is None:
+ context = ssl._create_stdlib_context(self.ssl_version,
+ certfile=certfile,
+ keyfile=keyfile)
self.context = context
self._prot_p = False
FTP.__init__(self, host, user, passwd, acct, timeout, source_address)
@@ -711,12 +747,9 @@ else:
resp = self.voidcmd('AUTH TLS')
else:
resp = self.voidcmd('AUTH SSL')
- if self.context is not None:
- self.sock = self.context.wrap_socket(self.sock)
- else:
- self.sock = ssl.wrap_socket(self.sock, self.keyfile,
- self.certfile,
- ssl_version=self.ssl_version)
+ server_hostname = self.host if ssl.HAS_SNI else None
+ self.sock = self.context.wrap_socket(self.sock,
+ server_hostname=server_hostname)
self.file = self.sock.makefile(mode='r', encoding=self.encoding)
return resp
@@ -755,80 +788,11 @@ else:
def ntransfercmd(self, cmd, rest=None):
conn, size = FTP.ntransfercmd(self, cmd, rest)
if self._prot_p:
- if self.context is not None:
- conn = self.context.wrap_socket(conn)
- else:
- conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
- ssl_version=self.ssl_version)
+ server_hostname = self.host if ssl.HAS_SNI else None
+ conn = self.context.wrap_socket(conn,
+ server_hostname=server_hostname)
return conn, size
- def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
- self.voidcmd('TYPE I')
- with self.transfercmd(cmd, rest) as conn:
- while 1:
- data = conn.recv(blocksize)
- if not data:
- break
- callback(data)
- # shutdown ssl layer
- if isinstance(conn, ssl.SSLSocket):
- conn.unwrap()
- return self.voidresp()
-
- def retrlines(self, cmd, callback = None):
- if callback is None: callback = print_line
- resp = self.sendcmd('TYPE A')
- conn = self.transfercmd(cmd)
- fp = conn.makefile('r', encoding=self.encoding)
- with fp, conn:
- while 1:
- line = fp.readline(self.maxline + 1)
- if len(line) > self.maxline:
- raise Error("got more than %d bytes" % self.maxline)
- if self.debugging > 2: print('*retr*', repr(line))
- if not line:
- break
- if line[-2:] == CRLF:
- line = line[:-2]
- elif line[-1:] == '\n':
- line = line[:-1]
- callback(line)
- # shutdown ssl layer
- if isinstance(conn, ssl.SSLSocket):
- conn.unwrap()
- return self.voidresp()
-
- def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None):
- self.voidcmd('TYPE I')
- with self.transfercmd(cmd, rest) as conn:
- while 1:
- buf = fp.read(blocksize)
- if not buf: break
- conn.sendall(buf)
- if callback: callback(buf)
- # shutdown ssl layer
- if isinstance(conn, ssl.SSLSocket):
- conn.unwrap()
- return self.voidresp()
-
- def storlines(self, cmd, fp, callback=None):
- self.voidcmd('TYPE A')
- with self.transfercmd(cmd) as conn:
- while 1:
- buf = fp.readline(self.maxline + 1)
- if len(buf) > self.maxline:
- raise Error("got more than %d bytes" % self.maxline)
- if not buf: break
- if buf[-2:] != B_CRLF:
- if buf[-1] in B_CRLF: buf = buf[:-1]
- buf = buf + B_CRLF
- conn.sendall(buf)
- if callback: callback(buf)
- # shutdown ssl layer
- if isinstance(conn, ssl.SSLSocket):
- conn.unwrap()
- return self.voidresp()
-
def abort(self):
# overridden as we can't pass MSG_OOB flag to sendall()
line = b'ABOR' + B_CRLF
@@ -839,7 +803,7 @@ else:
return resp
__all__.append('FTP_TLS')
- all_errors = (Error, IOError, EOFError, ssl.SSLError)
+ all_errors = (Error, OSError, EOFError, ssl.SSLError)
_150_re = None
@@ -936,7 +900,8 @@ def print_line(line):
def ftpcp(source, sourcename, target, targetname = '', type = 'I'):
'''Copy file from one FTP-instance to another.'''
- if not targetname: targetname = sourcename
+ if not targetname:
+ targetname = sourcename
type = 'TYPE ' + type
source.voidcmd(type)
target.voidcmd(type)
@@ -946,9 +911,11 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'):
# transfer request.
# So: STOR before RETR, because here the target is a "user".
treply = target.sendcmd('STOR ' + targetname)
- if treply[:3] not in {'125', '150'}: raise error_proto # RFC 959
+ if treply[:3] not in {'125', '150'}:
+ raise error_proto # RFC 959
sreply = source.sendcmd('RETR ' + sourcename)
- if sreply[:3] not in {'125', '150'}: raise error_proto # RFC 959
+ if sreply[:3] not in {'125', '150'}:
+ raise error_proto # RFC 959
source.voidresp()
target.voidresp()
@@ -966,19 +933,22 @@ class Netrc:
__defacct = None
def __init__(self, filename=None):
+ warnings.warn("This class is deprecated, use the netrc module instead",
+ DeprecationWarning, 2)
if filename is None:
if "HOME" in os.environ:
filename = os.path.join(os.environ["HOME"],
".netrc")
else:
- raise IOError("specify file to load or set $HOME")
+ raise OSError("specify file to load or set $HOME")
self.__hosts = {}
self.__macros = {}
fp = open(filename, "r")
in_macro = 0
while 1:
line = fp.readline()
- if not line: break
+ if not line:
+ break
if in_macro and line.strip():
macro_lines.append(line)
continue
@@ -1087,7 +1057,7 @@ def test():
userid = passwd = acct = ''
try:
netrc = Netrc(rcfile)
- except IOError:
+ except OSError:
if rcfile is not None:
sys.stderr.write("Could not open account file"
" -- using anonymous login.")
diff --git a/Lib/functools.py b/Lib/functools.py
index 053e44e..b8463ad 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -3,16 +3,24 @@
# Python module wrapper for _functools C module
# to allow utilities written in Python to be added
# to the functools module.
-# Written by Nick Coghlan <ncoghlan at gmail.com>
-# and Raymond Hettinger <python at rcn.com>
-# Copyright (C) 2006-2010 Python Software Foundation.
+# Written by Nick Coghlan <ncoghlan at gmail.com>,
+# Raymond Hettinger <python at rcn.com>,
+# and Łukasz Langa <lukasz at langa.pl>.
+# Copyright (C) 2006-2013 Python Software Foundation.
# See C source code for _functools credits/copyright
__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
- 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial']
+ 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial',
+ 'partialmethod', 'singledispatch']
-from _functools import partial, reduce
+try:
+ from _functools import reduce
+except ImportError:
+ pass
+from abc import get_cache_token
from collections import namedtuple
+from types import MappingProxyType
+from weakref import WeakKeyDictionary
try:
from _thread import RLock
except:
@@ -47,7 +55,6 @@ def update_wrapper(wrapper,
are updated with the corresponding attribute from the wrapped
function (defaults to functools.WRAPPER_UPDATES)
"""
- wrapper.__wrapped__ = wrapped
for attr in assigned:
try:
value = getattr(wrapped, attr)
@@ -57,6 +64,9 @@ def update_wrapper(wrapper,
setattr(wrapper, attr, value)
for attr in updated:
getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
+ # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
+ # from the wrapped function when updating __dict__
+ wrapper.__wrapped__ = wrapped
# Return the wrapper so this can be used as a decorator via partial()
return wrapper
@@ -79,21 +89,91 @@ def wraps(wrapped,
### total_ordering class decorator
################################################################################
+# The correct way to indicate that a comparison operation doesn't
+# recognise the other type is to return NotImplemented and let the
+# interpreter handle raising TypeError if both operands return
+# NotImplemented from their respective comparison methods
+#
+# This makes the implementation of total_ordering more complicated, since
+# we need to be careful not to trigger infinite recursion when two
+# different types that both use this decorator encounter each other.
+#
+# For example, if a type implements __lt__, it's natural to define
+# __gt__ as something like:
+#
+# lambda self, other: not self < other and not self == other
+#
+# However, using the operator syntax like that ends up invoking the full
+# type checking machinery again and means we can end up bouncing back and
+# forth between the two operands until we run out of stack space.
+#
+# The solution is to define helper functions that invoke the appropriate
+# magic methods directly, ensuring we only try each operand once, and
+# return NotImplemented immediately if it is returned from the
+# underlying user provided method. Using this scheme, the __gt__ derived
+# from a user provided __lt__ becomes:
+#
+# lambda self, other: _not_op_and_not_eq(self.__lt__, self, other))
+
+def _not_op(op, other):
+ # "not a < b" handles "a >= b"
+ # "not a <= b" handles "a > b"
+ # "not a >= b" handles "a < b"
+ # "not a > b" handles "a <= b"
+ op_result = op(other)
+ if op_result is NotImplemented:
+ return NotImplemented
+ return not op_result
+
+def _op_or_eq(op, self, other):
+ # "a < b or a == b" handles "a <= b"
+ # "a > b or a == b" handles "a >= b"
+ op_result = op(other)
+ if op_result is NotImplemented:
+ return NotImplemented
+ return op_result or self == other
+
+def _not_op_and_not_eq(op, self, other):
+ # "not (a < b or a == b)" handles "a > b"
+ # "not a < b and a != b" is equivalent
+ # "not (a > b or a == b)" handles "a < b"
+ # "not a > b and a != b" is equivalent
+ op_result = op(other)
+ if op_result is NotImplemented:
+ return NotImplemented
+ return not op_result and self != other
+
+def _not_op_or_eq(op, self, other):
+ # "not a <= b or a == b" handles "a >= b"
+ # "not a >= b or a == b" handles "a <= b"
+ op_result = op(other)
+ if op_result is NotImplemented:
+ return NotImplemented
+ return not op_result or self == other
+
+def _op_and_not_eq(op, self, other):
+ # "a <= b and not a == b" handles "a < b"
+ # "a >= b and not a == b" handles "a > b"
+ op_result = op(other)
+ if op_result is NotImplemented:
+ return NotImplemented
+ return op_result and self != other
+
def total_ordering(cls):
"""Class decorator that fills in missing ordering methods"""
convert = {
- '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
- ('__le__', lambda self, other: self < other or self == other),
- ('__ge__', lambda self, other: not self < other)],
- '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
- ('__lt__', lambda self, other: self <= other and not self == other),
- ('__gt__', lambda self, other: not self <= other)],
- '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
- ('__ge__', lambda self, other: self > other or self == other),
- ('__le__', lambda self, other: not self > other)],
- '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
- ('__gt__', lambda self, other: self >= other and not self == other),
- ('__lt__', lambda self, other: not self >= other)]
+ '__lt__': [('__gt__', lambda self, other: _not_op_and_not_eq(self.__lt__, self, other)),
+ ('__le__', lambda self, other: _op_or_eq(self.__lt__, self, other)),
+ ('__ge__', lambda self, other: _not_op(self.__lt__, other))],
+ '__le__': [('__ge__', lambda self, other: _not_op_or_eq(self.__le__, self, other)),
+ ('__lt__', lambda self, other: _op_and_not_eq(self.__le__, self, other)),
+ ('__gt__', lambda self, other: _not_op(self.__le__, other))],
+ '__gt__': [('__lt__', lambda self, other: _not_op_and_not_eq(self.__gt__, self, other)),
+ ('__ge__', lambda self, other: _op_or_eq(self.__gt__, self, other)),
+ ('__le__', lambda self, other: _not_op(self.__gt__, other))],
+ '__ge__': [('__le__', lambda self, other: _not_op_or_eq(self.__ge__, self, other)),
+ ('__gt__', lambda self, other: _op_and_not_eq(self.__ge__, self, other)),
+ ('__lt__', lambda self, other: _not_op(self.__ge__, other))]
}
# Find user-defined comparisons (not those inherited from object).
roots = [op for op in convert if getattr(cls, op, None) is not getattr(object, op, None)]
@@ -140,6 +220,104 @@ except ImportError:
################################################################################
+### partial() argument application
+################################################################################
+
+# Purely functional, no descriptor behaviour
+def partial(func, *args, **keywords):
+ """New function with partial application of the given arguments
+ and keywords.
+ """
+ def newfunc(*fargs, **fkeywords):
+ newkeywords = keywords.copy()
+ newkeywords.update(fkeywords)
+ return func(*(args + fargs), **newkeywords)
+ newfunc.func = func
+ newfunc.args = args
+ newfunc.keywords = keywords
+ return newfunc
+
+try:
+ from _functools import partial
+except ImportError:
+ pass
+
+# Descriptor version
+class partialmethod(object):
+ """Method descriptor with partial application of the given arguments
+ and keywords.
+
+ Supports wrapping existing descriptors and handles non-descriptor
+ callables as instance methods.
+ """
+
+ def __init__(self, func, *args, **keywords):
+ if not callable(func) and not hasattr(func, "__get__"):
+ raise TypeError("{!r} is not callable or a descriptor"
+ .format(func))
+
+ # func could be a descriptor like classmethod which isn't callable,
+ # so we can't inherit from partial (it verifies func is callable)
+ if isinstance(func, partialmethod):
+ # flattening is mandatory in order to place cls/self before all
+ # other arguments
+ # it's also more efficient since only one function will be called
+ self.func = func.func
+ self.args = func.args + args
+ self.keywords = func.keywords.copy()
+ self.keywords.update(keywords)
+ else:
+ self.func = func
+ self.args = args
+ self.keywords = keywords
+
+ def __repr__(self):
+ args = ", ".join(map(repr, self.args))
+ keywords = ", ".join("{}={!r}".format(k, v)
+ for k, v in self.keywords.items())
+ format_string = "{module}.{cls}({func}, {args}, {keywords})"
+ return format_string.format(module=self.__class__.__module__,
+ cls=self.__class__.__name__,
+ func=self.func,
+ args=args,
+ keywords=keywords)
+
+ def _make_unbound_method(self):
+ def _method(*args, **keywords):
+ call_keywords = self.keywords.copy()
+ call_keywords.update(keywords)
+ cls_or_self, *rest = args
+ call_args = (cls_or_self,) + self.args + tuple(rest)
+ return self.func(*call_args, **call_keywords)
+ _method.__isabstractmethod__ = self.__isabstractmethod__
+ _method._partialmethod = self
+ return _method
+
+ def __get__(self, obj, cls):
+ get = getattr(self.func, "__get__", None)
+ result = None
+ if get is not None:
+ new_func = get(obj, cls)
+ if new_func is not self.func:
+ # Assume __get__ returning something new indicates the
+ # creation of an appropriate callable
+ result = partial(new_func, *self.args, **self.keywords)
+ try:
+ result.__self__ = new_func.__self__
+ except AttributeError:
+ pass
+ if result is None:
+ # If the underlying descriptor didn't do anything, treat this
+ # like an instance method
+ result = self._make_unbound_method().__get__(obj, cls)
+ return result
+
+ @property
+ def __isabstractmethod__(self):
+ return getattr(self.func, "__isabstractmethod__", False)
+
+
+################################################################################
### LRU Cache function decorator
################################################################################
@@ -220,7 +398,6 @@ def lru_cache(maxsize=128, typed=False):
PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields
def decorating_function(user_function):
-
cache = {}
hits = misses = 0
full = False
@@ -329,3 +506,210 @@ def lru_cache(maxsize=128, typed=False):
return update_wrapper(wrapper, user_function)
return decorating_function
+
+
+################################################################################
+### singledispatch() - single-dispatch generic function decorator
+################################################################################
+
+def _c3_merge(sequences):
+ """Merges MROs in *sequences* to a single MRO using the C3 algorithm.
+
+ Adapted from http://www.python.org/download/releases/2.3/mro/.
+
+ """
+ result = []
+ while True:
+ sequences = [s for s in sequences if s] # purge empty sequences
+ if not sequences:
+ return result
+ for s1 in sequences: # find merge candidates among seq heads
+ candidate = s1[0]
+ for s2 in sequences:
+ if candidate in s2[1:]:
+ candidate = None
+ break # reject the current head, it appears later
+ else:
+ break
+ if not candidate:
+ raise RuntimeError("Inconsistent hierarchy")
+ result.append(candidate)
+ # remove the chosen candidate
+ for seq in sequences:
+ if seq[0] == candidate:
+ del seq[0]
+
+def _c3_mro(cls, abcs=None):
+ """Computes the method resolution order using extended C3 linearization.
+
+ If no *abcs* are given, the algorithm works exactly like the built-in C3
+ linearization used for method resolution.
+
+ If given, *abcs* is a list of abstract base classes that should be inserted
+ into the resulting MRO. Unrelated ABCs are ignored and don't end up in the
+ result. The algorithm inserts ABCs where their functionality is introduced,
+ i.e. issubclass(cls, abc) returns True for the class itself but returns
+ False for all its direct base classes. Implicit ABCs for a given class
+ (either registered or inferred from the presence of a special method like
+ __len__) are inserted directly after the last ABC explicitly listed in the
+ MRO of said class. If two implicit ABCs end up next to each other in the
+ resulting MRO, their ordering depends on the order of types in *abcs*.
+
+ """
+ for i, base in enumerate(reversed(cls.__bases__)):
+ if hasattr(base, '__abstractmethods__'):
+ boundary = len(cls.__bases__) - i
+ break # Bases up to the last explicit ABC are considered first.
+ else:
+ boundary = 0
+ abcs = list(abcs) if abcs else []
+ explicit_bases = list(cls.__bases__[:boundary])
+ abstract_bases = []
+ other_bases = list(cls.__bases__[boundary:])
+ for base in abcs:
+ if issubclass(cls, base) and not any(
+ issubclass(b, base) for b in cls.__bases__
+ ):
+ # If *cls* is the class that introduces behaviour described by
+ # an ABC *base*, insert said ABC to its MRO.
+ abstract_bases.append(base)
+ for base in abstract_bases:
+ abcs.remove(base)
+ explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases]
+ abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases]
+ other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases]
+ return _c3_merge(
+ [[cls]] +
+ explicit_c3_mros + abstract_c3_mros + other_c3_mros +
+ [explicit_bases] + [abstract_bases] + [other_bases]
+ )
+
+def _compose_mro(cls, types):
+ """Calculates the method resolution order for a given class *cls*.
+
+ Includes relevant abstract base classes (with their respective bases) from
+ the *types* iterable. Uses a modified C3 linearization algorithm.
+
+ """
+ bases = set(cls.__mro__)
+ # Remove entries which are already present in the __mro__ or unrelated.
+ def is_related(typ):
+ return (typ not in bases and hasattr(typ, '__mro__')
+ and issubclass(cls, typ))
+ types = [n for n in types if is_related(n)]
+ # Remove entries which are strict bases of other entries (they will end up
+ # in the MRO anyway.
+ def is_strict_base(typ):
+ for other in types:
+ if typ != other and typ in other.__mro__:
+ return True
+ return False
+ types = [n for n in types if not is_strict_base(n)]
+ # Subclasses of the ABCs in *types* which are also implemented by
+ # *cls* can be used to stabilize ABC ordering.
+ type_set = set(types)
+ mro = []
+ for typ in types:
+ found = []
+ for sub in typ.__subclasses__():
+ if sub not in bases and issubclass(cls, sub):
+ found.append([s for s in sub.__mro__ if s in type_set])
+ if not found:
+ mro.append(typ)
+ continue
+ # Favor subclasses with the biggest number of useful bases
+ found.sort(key=len, reverse=True)
+ for sub in found:
+ for subcls in sub:
+ if subcls not in mro:
+ mro.append(subcls)
+ return _c3_mro(cls, abcs=mro)
+
+def _find_impl(cls, registry):
+ """Returns the best matching implementation from *registry* for type *cls*.
+
+ Where there is no registered implementation for a specific type, its method
+ resolution order is used to find a more generic implementation.
+
+ Note: if *registry* does not contain an implementation for the base
+ *object* type, this function may return None.
+
+ """
+ mro = _compose_mro(cls, registry.keys())
+ match = None
+ for t in mro:
+ if match is not None:
+ # If *match* is an implicit ABC but there is another unrelated,
+ # equally matching implicit ABC, refuse the temptation to guess.
+ if (t in registry and t not in cls.__mro__
+ and match not in cls.__mro__
+ and not issubclass(match, t)):
+ raise RuntimeError("Ambiguous dispatch: {} or {}".format(
+ match, t))
+ break
+ if t in registry:
+ match = t
+ return registry.get(match)
+
+def singledispatch(func):
+ """Single-dispatch generic function decorator.
+
+ Transforms a function into a generic function, which can have different
+ behaviours depending upon the type of its first argument. The decorated
+ function acts as the default implementation, and additional
+ implementations can be registered using the register() attribute of the
+ generic function.
+
+ """
+ registry = {}
+ dispatch_cache = WeakKeyDictionary()
+ cache_token = None
+
+ def dispatch(cls):
+ """generic_func.dispatch(cls) -> <function implementation>
+
+ Runs the dispatch algorithm to return the best available implementation
+ for the given *cls* registered on *generic_func*.
+
+ """
+ nonlocal cache_token
+ if cache_token is not None:
+ current_token = get_cache_token()
+ if cache_token != current_token:
+ dispatch_cache.clear()
+ cache_token = current_token
+ try:
+ impl = dispatch_cache[cls]
+ except KeyError:
+ try:
+ impl = registry[cls]
+ except KeyError:
+ impl = _find_impl(cls, registry)
+ dispatch_cache[cls] = impl
+ return impl
+
+ def register(cls, func=None):
+ """generic_func.register(cls, func) -> func
+
+ Registers a new implementation for the given *cls* on a *generic_func*.
+
+ """
+ nonlocal cache_token
+ if func is None:
+ return lambda f: register(cls, f)
+ registry[cls] = func
+ if cache_token is None and hasattr(cls, '__abstractmethods__'):
+ cache_token = get_cache_token()
+ dispatch_cache.clear()
+ return func
+
+ def wrapper(*args, **kw):
+ return dispatch(args[0].__class__)(*args, **kw)
+
+ registry[object] = func
+ wrapper.register = register
+ wrapper.dispatch = dispatch
+ wrapper.registry = MappingProxyType(registry)
+ wrapper._clear_cache = dispatch_cache.clear
+ update_wrapper(wrapper, func)
+ return wrapper
diff --git a/Lib/genericpath.py b/Lib/genericpath.py
index 340c004..ca4a510 100644
--- a/Lib/genericpath.py
+++ b/Lib/genericpath.py
@@ -7,7 +7,8 @@ import os
import stat
__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
- 'getsize', 'isdir', 'isfile']
+ 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile',
+ 'samestat']
# Does a path exist?
@@ -16,7 +17,7 @@ def exists(path):
"""Test whether a path exists. Returns False for broken symbolic links"""
try:
os.stat(path)
- except os.error:
+ except OSError:
return False
return True
@@ -27,7 +28,7 @@ def isfile(path):
"""Test whether a path is a regular file"""
try:
st = os.stat(path)
- except os.error:
+ except OSError:
return False
return stat.S_ISREG(st.st_mode)
@@ -39,7 +40,7 @@ def isdir(s):
"""Return true if the pathname refers to an existing directory."""
try:
st = os.stat(s)
- except os.error:
+ except OSError:
return False
return stat.S_ISDIR(st.st_mode)
@@ -75,6 +76,31 @@ def commonprefix(m):
return s1[:i]
return s1
+# Are two stat buffers (obtained from stat, fstat or lstat)
+# describing the same file?
+def samestat(s1, s2):
+ """Test whether two stat buffers reference the same file"""
+ return (s1.st_ino == s2.st_ino and
+ s1.st_dev == s2.st_dev)
+
+
+# Are two filenames really pointing to the same file?
+def samefile(f1, f2):
+ """Test whether two pathnames reference the same actual file"""
+ s1 = os.stat(f1)
+ s2 = os.stat(f2)
+ return samestat(s1, s2)
+
+
+# Are two open files really referencing the same file?
+# (Not necessarily the same file descriptor!)
+def sameopenfile(fp1, fp2):
+ """Test whether two open file objects reference the same file"""
+ s1 = os.fstat(fp1)
+ s2 = os.fstat(fp2)
+ return samestat(s1, s2)
+
+
# Split a path in root and extension.
# The extension is everything starting at the last dot in the last
# pathname component; the root is everything before that.
diff --git a/Lib/gettext.py b/Lib/gettext.py
index e43f044..05d9c1e 100644
--- a/Lib/gettext.py
+++ b/Lib/gettext.py
@@ -244,7 +244,7 @@ class GNUTranslations(NullTranslations):
version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
ii = '>II'
else:
- raise IOError(0, 'Bad magic number', filename)
+ raise OSError(0, 'Bad magic number', filename)
# Now put all messages from the .mo file buffer into the catalog
# dictionary.
for i in range(0, msgcount):
@@ -256,7 +256,7 @@ class GNUTranslations(NullTranslations):
msg = buf[moff:mend]
tmsg = buf[toff:tend]
else:
- raise IOError(0, 'File is corrupt', filename)
+ raise OSError(0, 'File is corrupt', filename)
# See if we're looking at GNU .mo conventions for metadata
if mlen == 0:
# Catalog description
@@ -398,7 +398,7 @@ def translation(domain, localedir=None, languages=None,
if not mofiles:
if fallback:
return NullTranslations()
- raise IOError(ENOENT, 'No translation file found for domain', domain)
+ raise OSError(ENOENT, 'No translation file found for domain', domain)
# Avoid opening, reading, and parsing the .mo file after it's been done
# once.
result = None
@@ -460,7 +460,7 @@ def dgettext(domain, message):
try:
t = translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
- except IOError:
+ except OSError:
return message
return t.gettext(message)
@@ -468,7 +468,7 @@ def ldgettext(domain, message):
try:
t = translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
- except IOError:
+ except OSError:
return message
return t.lgettext(message)
@@ -476,7 +476,7 @@ def dngettext(domain, msgid1, msgid2, n):
try:
t = translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
- except IOError:
+ except OSError:
if n == 1:
return msgid1
else:
@@ -487,7 +487,7 @@ def ldngettext(domain, msgid1, msgid2, n):
try:
t = translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
- except IOError:
+ except OSError:
if n == 1:
return msgid1
else:
diff --git a/Lib/glob.py b/Lib/glob.py
index 1f60265..e388b5f 100644
--- a/Lib/glob.py
+++ b/Lib/glob.py
@@ -32,8 +32,7 @@ def iglob(pathname):
return
dirname, basename = os.path.split(pathname)
if not dirname:
- for name in glob1(None, basename):
- yield name
+ yield from glob1(None, basename)
return
# `os.path.split()` returns the argument itself as a dirname if it is a
# drive or UNC path. Prevent an infinite recursion if a drive or UNC path
@@ -62,7 +61,7 @@ def glob1(dirname, pattern):
dirname = os.curdir
try:
names = os.listdir(dirname)
- except os.error:
+ except OSError:
return []
if not _ishidden(pattern):
names = [x for x in names if not _ishidden(x)]
@@ -80,8 +79,8 @@ def glob0(dirname, basename):
return []
-magic_check = re.compile('[*?[]')
-magic_check_bytes = re.compile(b'[*?[]')
+magic_check = re.compile('([*?[])')
+magic_check_bytes = re.compile(b'([*?[])')
def has_magic(s):
if isinstance(s, bytes):
@@ -92,3 +91,15 @@ def has_magic(s):
def _ishidden(path):
return path[0] in ('.', b'.'[0])
+
+def escape(pathname):
+ """Escape all special characters.
+ """
+ # Escaping is done by wrapping any of "*?[" between square brackets.
+ # Metacharacters do not work in the drive part and shouldn't be escaped.
+ drive, pathname = os.path.splitdrive(pathname)
+ if isinstance(pathname, bytes):
+ pathname = magic_check_bytes.sub(br'[\1]', pathname)
+ else:
+ pathname = magic_check.sub(r'[\1]', pathname)
+ return drive + pathname
diff --git a/Lib/gzip.py b/Lib/gzip.py
index 4ff9820..f934d4f 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -23,9 +23,9 @@ def open(filename, mode="rb", compresslevel=9,
The filename argument can be an actual filename (a str or bytes object), or
an existing file object to read from or write to.
- The mode argument can be "r", "rb", "w", "wb", "a" or "ab" for binary mode,
- or "rt", "wt" or "at" for text mode. The default mode is "rb", and the
- default compresslevel is 9.
+ The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+ binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+ "rb", and the default compresslevel is 9.
For binary mode, this function is equivalent to the GzipFile constructor:
GzipFile(filename, mode, compresslevel). In this case, the encoding, errors
@@ -65,9 +65,6 @@ def write32u(output, value):
# or unsigned.
output.write(struct.pack("<L", value))
-def read32(input):
- return struct.unpack("<I", input.read(4))[0]
-
class _PaddedFile:
"""Minimal read-only file object that prepends a string to the contents
of an actual file. Shouldn't be used outside of gzip.py, as it lacks
@@ -154,11 +151,11 @@ class GzipFile(io.BufferedIOBase):
fileobj, if discernible; otherwise, it defaults to the empty string,
and in this case the original filename is not included in the header.
- The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
- depending on whether the file will be read or written. The default
+ The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or
+ 'xb' depending on whether the file will be read or written. The default
is the mode of fileobj if discernible; otherwise, the default is 'rb'.
A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and
- 'wb', and 'a' and 'ab'.
+ 'wb', 'a' and 'ab', and 'x' and 'xb'.
The compresslevel argument is an integer from 0 to 9 controlling the
level of compression; 1 is fastest and produces the least compression,
@@ -204,7 +201,7 @@ class GzipFile(io.BufferedIOBase):
self.min_readsize = 100
fileobj = _PaddedFile(fileobj)
- elif mode.startswith(('w', 'a')):
+ elif mode.startswith(('w', 'a', 'x')):
self.mode = WRITE
self._init_write(filename)
self.compress = zlib.compressobj(compresslevel,
@@ -281,28 +278,32 @@ class GzipFile(io.BufferedIOBase):
self.crc = zlib.crc32(b"") & 0xffffffff
self.size = 0
+ def _read_exact(self, n):
+ data = self.fileobj.read(n)
+ while len(data) < n:
+ b = self.fileobj.read(n - len(data))
+ if not b:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+ data += b
+ return data
+
def _read_gzip_header(self):
magic = self.fileobj.read(2)
if magic == b'':
- raise EOFError("Reached EOF")
+ return False
if magic != b'\037\213':
- raise IOError('Not a gzipped file')
+ raise OSError('Not a gzipped file')
- method = ord( self.fileobj.read(1) )
+ method, flag, self.mtime = struct.unpack("<BBIxx", self._read_exact(8))
if method != 8:
- raise IOError('Unknown compression method')
- flag = ord( self.fileobj.read(1) )
- self.mtime = read32(self.fileobj)
- # extraflag = self.fileobj.read(1)
- # os = self.fileobj.read(1)
- self.fileobj.read(2)
+ raise OSError('Unknown compression method')
if flag & FEXTRA:
# Read & discard the extra field, if present
- xlen = ord(self.fileobj.read(1))
- xlen = xlen + 256*ord(self.fileobj.read(1))
- self.fileobj.read(xlen)
+ extra_len, = struct.unpack("<H", self._read_exact(2))
+ self._read_exact(extra_len)
if flag & FNAME:
# Read and discard a null-terminated string containing the filename
while True:
@@ -316,18 +317,19 @@ class GzipFile(io.BufferedIOBase):
if not s or s==b'\000':
break
if flag & FHCRC:
- self.fileobj.read(2) # Read & discard the 16-bit header CRC
+ self._read_exact(2) # Read & discard the 16-bit header CRC
unused = self.fileobj.unused()
if unused:
uncompress = self.decompress.decompress(unused)
self._add_read_data(uncompress)
+ return True
def write(self,data):
self._check_closed()
if self.mode != WRITE:
import errno
- raise IOError(errno.EBADF, "write() on read-only GzipFile object")
+ raise OSError(errno.EBADF, "write() on read-only GzipFile object")
if self.fileobj is None:
raise ValueError("write() on closed GzipFile object")
@@ -348,27 +350,23 @@ class GzipFile(io.BufferedIOBase):
self._check_closed()
if self.mode != READ:
import errno
- raise IOError(errno.EBADF, "read() on write-only GzipFile object")
+ raise OSError(errno.EBADF, "read() on write-only GzipFile object")
if self.extrasize <= 0 and self.fileobj is None:
return b''
readsize = 1024
if size < 0: # get the whole thing
- try:
- while True:
- self._read(readsize)
- readsize = min(self.max_read_chunk, readsize * 2)
- except EOFError:
- size = self.extrasize
+ while self._read(readsize):
+ readsize = min(self.max_read_chunk, readsize * 2)
+ size = self.extrasize
else: # just get some more of it
- try:
- while size > self.extrasize:
- self._read(readsize)
- readsize = min(self.max_read_chunk, readsize * 2)
- except EOFError:
- if size > self.extrasize:
- size = self.extrasize
+ while size > self.extrasize:
+ if not self._read(readsize):
+ if size > self.extrasize:
+ size = self.extrasize
+ break
+ readsize = min(self.max_read_chunk, readsize * 2)
offset = self.offset - self.extrastart
chunk = self.extrabuf[offset: offset + size]
@@ -381,17 +379,14 @@ class GzipFile(io.BufferedIOBase):
self._check_closed()
if self.mode != READ:
import errno
- raise IOError(errno.EBADF, "read1() on write-only GzipFile object")
+ raise OSError(errno.EBADF, "read1() on write-only GzipFile object")
if self.extrasize <= 0 and self.fileobj is None:
return b''
- try:
- # For certain input data, a single call to _read() may not return
- # any data. In this case, retry until we get some data or reach EOF.
- while self.extrasize <= 0:
- self._read()
- except EOFError:
+ # For certain input data, a single call to _read() may not return
+ # any data. In this case, retry until we get some data or reach EOF.
+ while self.extrasize <= 0 and self._read():
pass
if size < 0 or size > self.extrasize:
size = self.extrasize
@@ -405,7 +400,7 @@ class GzipFile(io.BufferedIOBase):
def peek(self, n):
if self.mode != READ:
import errno
- raise IOError(errno.EBADF, "peek() on write-only GzipFile object")
+ raise OSError(errno.EBADF, "peek() on write-only GzipFile object")
# Do not return ridiculously small buffers, for one common idiom
# is to call peek(1) and expect more bytes in return.
@@ -414,12 +409,9 @@ class GzipFile(io.BufferedIOBase):
if self.extrasize == 0:
if self.fileobj is None:
return b''
- try:
- # Ensure that we don't return b"" if we haven't reached EOF.
- while self.extrasize == 0:
- # 1024 is the same buffering heuristic used in read()
- self._read(max(n, 1024))
- except EOFError:
+ # Ensure that we don't return b"" if we haven't reached EOF.
+ # 1024 is the same buffering heuristic used in read()
+ while self.extrasize == 0 and self._read(max(n, 1024)):
pass
offset = self.offset - self.extrastart
remaining = self.extrasize
@@ -432,13 +424,14 @@ class GzipFile(io.BufferedIOBase):
def _read(self, size=1024):
if self.fileobj is None:
- raise EOFError("Reached EOF")
+ return False
if self._new_member:
# If the _new_member flag is set, we have to
# jump to the next member, if there is one.
self._init_read()
- self._read_gzip_header()
+ if not self._read_gzip_header():
+ return False
self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
self._new_member = False
@@ -455,7 +448,7 @@ class GzipFile(io.BufferedIOBase):
self.fileobj.prepend(self.decompress.unused_data, True)
self._read_eof()
self._add_read_data( uncompress )
- raise EOFError('Reached EOF')
+ return False
uncompress = self.decompress.decompress(buf)
self._add_read_data( uncompress )
@@ -471,6 +464,7 @@ class GzipFile(io.BufferedIOBase):
# a new member on the next call
self._read_eof()
self._new_member = True
+ return True
def _add_read_data(self, data):
self.crc = zlib.crc32(data, self.crc) & 0xffffffff
@@ -485,13 +479,12 @@ class GzipFile(io.BufferedIOBase):
# We check the that the computed CRC and size of the
# uncompressed data matches the stored values. Note that the size
# stored is the true file size mod 2**32.
- crc32 = read32(self.fileobj)
- isize = read32(self.fileobj) # may exceed 2GB
+ crc32, isize = struct.unpack("<II", self._read_exact(8))
if crc32 != self.crc:
- raise IOError("CRC check failed %s != %s" % (hex(crc32),
+ raise OSError("CRC check failed %s != %s" % (hex(crc32),
hex(self.crc)))
elif isize != (self.size & 0xffffffff):
- raise IOError("Incorrect length of data produced")
+ raise OSError("Incorrect length of data produced")
# Gzip files can be padded with zeroes and still have archives.
# Consume all zero bytes and set the file position to the first
@@ -540,7 +533,7 @@ class GzipFile(io.BufferedIOBase):
'''Return the uncompressed stream file position indicator to the
beginning of the file'''
if self.mode != READ:
- raise IOError("Can't rewind in write mode")
+ raise OSError("Can't rewind in write mode")
self.fileobj.seek(0)
self._new_member = True
self.extrabuf = b""
@@ -565,7 +558,7 @@ class GzipFile(io.BufferedIOBase):
raise ValueError('Seek from end not supported')
if self.mode == WRITE:
if offset < self.offset:
- raise IOError('Negative seek in write mode')
+ raise OSError('Negative seek in write mode')
count = offset - self.offset
chunk = bytes(1024)
for i in range(count // 1024):
diff --git a/Lib/hashlib.py b/Lib/hashlib.py
index 21454c7..e652fc6 100644
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org)
+#. Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org)
# Licensed to PSF under a Contributor Agreement.
#
@@ -60,34 +60,38 @@ algorithms_guaranteed = set(__always_supported)
algorithms_available = set(__always_supported)
__all__ = __always_supported + ('new', 'algorithms_guaranteed',
- 'algorithms_available')
+ 'algorithms_available', 'pbkdf2_hmac')
+__builtin_constructor_cache = {}
+
def __get_builtin_constructor(name):
+ cache = __builtin_constructor_cache
+ constructor = cache.get(name)
+ if constructor is not None:
+ return constructor
try:
if name in ('SHA1', 'sha1'):
import _sha1
- return _sha1.sha1
+ cache['SHA1'] = cache['sha1'] = _sha1.sha1
elif name in ('MD5', 'md5'):
import _md5
- return _md5.md5
+ cache['MD5'] = cache['md5'] = _md5.md5
elif name in ('SHA256', 'sha256', 'SHA224', 'sha224'):
import _sha256
- bs = name[3:]
- if bs == '256':
- return _sha256.sha256
- elif bs == '224':
- return _sha256.sha224
+ cache['SHA224'] = cache['sha224'] = _sha256.sha224
+ cache['SHA256'] = cache['sha256'] = _sha256.sha256
elif name in ('SHA512', 'sha512', 'SHA384', 'sha384'):
import _sha512
- bs = name[3:]
- if bs == '512':
- return _sha512.sha512
- elif bs == '384':
- return _sha512.sha384
+ cache['SHA384'] = cache['sha384'] = _sha512.sha384
+ cache['SHA512'] = cache['sha512'] = _sha512.sha512
except ImportError:
pass # no extension module, this hash is unsupported.
+ constructor = cache.get(name)
+ if constructor is not None:
+ return constructor
+
raise ValueError('unsupported hash type ' + name)
@@ -134,6 +138,71 @@ except ImportError:
new = __py_new
__get_hash = __get_builtin_constructor
+try:
+ # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA
+ from _hashlib import pbkdf2_hmac
+except ImportError:
+ _trans_5C = bytes((x ^ 0x5C) for x in range(256))
+ _trans_36 = bytes((x ^ 0x36) for x in range(256))
+
+ def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None):
+ """Password based key derivation function 2 (PKCS #5 v2.0)
+
+ This Python implementations based on the hmac module about as fast
+ as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster
+ for long passwords.
+ """
+ if not isinstance(hash_name, str):
+ raise TypeError(hash_name)
+
+ if not isinstance(password, (bytes, bytearray)):
+ password = bytes(memoryview(password))
+ if not isinstance(salt, (bytes, bytearray)):
+ salt = bytes(memoryview(salt))
+
+ # Fast inline HMAC implementation
+ inner = new(hash_name)
+ outer = new(hash_name)
+ blocksize = getattr(inner, 'block_size', 64)
+ if len(password) > blocksize:
+ password = new(hash_name, password).digest()
+ password = password + b'\x00' * (blocksize - len(password))
+ inner.update(password.translate(_trans_36))
+ outer.update(password.translate(_trans_5C))
+
+ def prf(msg, inner=inner, outer=outer):
+ # PBKDF2_HMAC uses the password as key. We can re-use the same
+ # digest objects and and just update copies to skip initialization.
+ icpy = inner.copy()
+ ocpy = outer.copy()
+ icpy.update(msg)
+ ocpy.update(icpy.digest())
+ return ocpy.digest()
+
+ if iterations < 1:
+ raise ValueError(iterations)
+ if dklen is None:
+ dklen = outer.digest_size
+ if dklen < 1:
+ raise ValueError(dklen)
+
+ dkey = b''
+ loop = 1
+ from_bytes = int.from_bytes
+ while len(dkey) < dklen:
+ prev = prf(salt + loop.to_bytes(4, 'big'))
+ # endianess doesn't matter here as long to / from use the same
+ rkey = int.from_bytes(prev, 'big')
+ for i in range(iterations - 1):
+ prev = prf(prev)
+ # rkey = rkey ^ prev
+ rkey ^= from_bytes(prev, 'big')
+ loop += 1
+ dkey += rkey.to_bytes(inner.digest_size, 'big')
+
+ return dkey[:dklen]
+
+
for __func_name in __always_supported:
# try them all, some may not work due to the OpenSSL
# version not supporting that algorithm.
diff --git a/Lib/hmac.py b/Lib/hmac.py
index 4297a71..77785a2 100644
--- a/Lib/hmac.py
+++ b/Lib/hmac.py
@@ -4,7 +4,8 @@ Implements the HMAC algorithm as described by RFC 2104.
"""
import warnings as _warnings
-from operator import _compare_digest as compare_digest
+from _operator import _compare_digest as compare_digest
+import hashlib as _hashlib
trans_5C = bytes((x ^ 0x5C) for x in range(256))
trans_36 = bytes((x ^ 0x36) for x in range(256))
@@ -28,21 +29,27 @@ class HMAC:
key: key for the keyed hash object.
msg: Initial input for the hash, if provided.
digestmod: A module supporting PEP 247. *OR*
- A hashlib constructor returning a new hash object.
+ A hashlib constructor returning a new hash object. *OR*
+ A hash name suitable for hashlib.new().
Defaults to hashlib.md5.
+ Implicit default to hashlib.md5 is deprecated and will be
+ removed in Python 3.6.
- Note: key and msg must be bytes objects.
+ Note: key and msg must be a bytes or bytearray objects.
"""
- if not isinstance(key, bytes):
- raise TypeError("key: expected bytes, but got %r" % type(key).__name__)
+ if not isinstance(key, (bytes, bytearray)):
+ raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
if digestmod is None:
- import hashlib
- digestmod = hashlib.md5
+ _warnings.warn("HMAC() without an explicit digestmod argument "
+ "is deprecated.", PendingDeprecationWarning, 2)
+ digestmod = _hashlib.md5
if callable(digestmod):
self.digest_cons = digestmod
+ elif isinstance(digestmod, str):
+ self.digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
else:
self.digest_cons = lambda d=b'': digestmod.new(d)
@@ -63,6 +70,10 @@ class HMAC:
RuntimeWarning, 2)
blocksize = self.blocksize
+ # self.blocksize is the default blocksize. self.block_size is
+ # effective block size as well as the public API attribute.
+ self.block_size = blocksize
+
if len(key) > blocksize:
key = self.digest_cons(key).digest()
@@ -72,11 +83,13 @@ class HMAC:
if msg is not None:
self.update(msg)
+ @property
+ def name(self):
+ return "hmac-" + self.inner.name
+
def update(self, msg):
"""Update this hashing object with the string msg.
"""
- if not isinstance(msg, bytes):
- raise TypeError("expected bytes, but got %r" % type(msg).__name__)
self.inner.update(msg)
def copy(self):
diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py
index 02652ef..da0a0a3 100644
--- a/Lib/html/__init__.py
+++ b/Lib/html/__init__.py
@@ -2,12 +2,12 @@
General functions for HTML manipulation.
"""
+import re as _re
+from html.entities import html5 as _html5
-_escape_map = {ord('&'): '&amp;', ord('<'): '&lt;', ord('>'): '&gt;'}
-_escape_map_full = {ord('&'): '&amp;', ord('<'): '&lt;', ord('>'): '&gt;',
- ord('"'): '&quot;', ord('\''): '&#x27;'}
-# NB: this is a candidate for a bytes/string polymorphic interface
+__all__ = ['escape', 'unescape']
+
def escape(s, quote=True):
"""
@@ -16,6 +16,117 @@ def escape(s, quote=True):
characters, both double quote (") and single quote (') characters are also
translated.
"""
+ s = s.replace("&", "&amp;") # Must be done first!
+ s = s.replace("<", "&lt;")
+ s = s.replace(">", "&gt;")
if quote:
- return s.translate(_escape_map_full)
- return s.translate(_escape_map)
+ s = s.replace('"', "&quot;")
+ s = s.replace('\'', "&#x27;")
+ return s
+
+
+# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references
+
+_invalid_charrefs = {
+ 0x00: '\ufffd', # REPLACEMENT CHARACTER
+ 0x0d: '\r', # CARRIAGE RETURN
+ 0x80: '\u20ac', # EURO SIGN
+ 0x81: '\x81', # <control>
+ 0x82: '\u201a', # SINGLE LOW-9 QUOTATION MARK
+ 0x83: '\u0192', # LATIN SMALL LETTER F WITH HOOK
+ 0x84: '\u201e', # DOUBLE LOW-9 QUOTATION MARK
+ 0x85: '\u2026', # HORIZONTAL ELLIPSIS
+ 0x86: '\u2020', # DAGGER
+ 0x87: '\u2021', # DOUBLE DAGGER
+ 0x88: '\u02c6', # MODIFIER LETTER CIRCUMFLEX ACCENT
+ 0x89: '\u2030', # PER MILLE SIGN
+ 0x8a: '\u0160', # LATIN CAPITAL LETTER S WITH CARON
+ 0x8b: '\u2039', # SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+ 0x8c: '\u0152', # LATIN CAPITAL LIGATURE OE
+ 0x8d: '\x8d', # <control>
+ 0x8e: '\u017d', # LATIN CAPITAL LETTER Z WITH CARON
+ 0x8f: '\x8f', # <control>
+ 0x90: '\x90', # <control>
+ 0x91: '\u2018', # LEFT SINGLE QUOTATION MARK
+ 0x92: '\u2019', # RIGHT SINGLE QUOTATION MARK
+ 0x93: '\u201c', # LEFT DOUBLE QUOTATION MARK
+ 0x94: '\u201d', # RIGHT DOUBLE QUOTATION MARK
+ 0x95: '\u2022', # BULLET
+ 0x96: '\u2013', # EN DASH
+ 0x97: '\u2014', # EM DASH
+ 0x98: '\u02dc', # SMALL TILDE
+ 0x99: '\u2122', # TRADE MARK SIGN
+ 0x9a: '\u0161', # LATIN SMALL LETTER S WITH CARON
+ 0x9b: '\u203a', # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+ 0x9c: '\u0153', # LATIN SMALL LIGATURE OE
+ 0x9d: '\x9d', # <control>
+ 0x9e: '\u017e', # LATIN SMALL LETTER Z WITH CARON
+ 0x9f: '\u0178', # LATIN CAPITAL LETTER Y WITH DIAERESIS
+}
+
+_invalid_codepoints = {
+ # 0x0001 to 0x0008
+ 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8,
+ # 0x000E to 0x001F
+ 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19,
+ 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+ # 0x007F to 0x009F
+ 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a,
+ 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+ # 0xFDD0 to 0xFDEF
+ 0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, 0xfdd8,
+ 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, 0xfde0, 0xfde1,
+ 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, 0xfde8, 0xfde9, 0xfdea,
+ 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef,
+ # others
+ 0xb, 0xfffe, 0xffff, 0x1fffe, 0x1ffff, 0x2fffe, 0x2ffff, 0x3fffe, 0x3ffff,
+ 0x4fffe, 0x4ffff, 0x5fffe, 0x5ffff, 0x6fffe, 0x6ffff, 0x7fffe, 0x7ffff,
+ 0x8fffe, 0x8ffff, 0x9fffe, 0x9ffff, 0xafffe, 0xaffff, 0xbfffe, 0xbffff,
+ 0xcfffe, 0xcffff, 0xdfffe, 0xdffff, 0xefffe, 0xeffff, 0xffffe, 0xfffff,
+ 0x10fffe, 0x10ffff
+}
+
+
+def _replace_charref(s):
+ s = s.group(1)
+ if s[0] == '#':
+ # numeric charref
+ if s[1] in 'xX':
+ num = int(s[2:].rstrip(';'), 16)
+ else:
+ num = int(s[1:].rstrip(';'))
+ if num in _invalid_charrefs:
+ return _invalid_charrefs[num]
+ if 0xD800 <= num <= 0xDFFF or num > 0x10FFFF:
+ return '\uFFFD'
+ if num in _invalid_codepoints:
+ return ''
+ return chr(num)
+ else:
+ # named charref
+ if s in _html5:
+ return _html5[s]
+ # find the longest matching name (as defined by the standard)
+ for x in range(len(s)-1, 1, -1):
+ if s[:x] in _html5:
+ return _html5[s[:x]] + s[x:]
+ else:
+ return '&' + s
+
+
+_charref = _re.compile(r'&(#[0-9]+;?'
+ r'|#[xX][0-9a-fA-F]+;?'
+ r'|[^\t\n\f <&#;]{1,32};?)')
+
+def unescape(s):
+ """
+ Convert all named and numeric character references (e.g. &gt;, &#62;,
+ &x3e;) in the string s to the corresponding unicode characters.
+ This function uses the rules defined by the HTML 5 standard
+ for both valid and invalid character references, and the list of
+ HTML 5 named character references defined in html.entities.html5.
+ """
+ if '&' not in s:
+ return s
+ return _charref.sub(_replace_charref, s)
diff --git a/Lib/html/parser.py b/Lib/html/parser.py
index 63fe774..a650d5e 100644
--- a/Lib/html/parser.py
+++ b/Lib/html/parser.py
@@ -8,9 +8,14 @@
# and CDATA (character data -- only end tags are special).
-import _markupbase
import re
import warnings
+import _markupbase
+
+from html import unescape
+
+
+__all__ = ['HTMLParser']
# Regular expressions used for parsing
@@ -92,6 +97,8 @@ class HTMLParseError(Exception):
return result
+_default_sentinel = object()
+
class HTMLParser(_markupbase.ParserBase):
"""Find tags and other markup and call handler functions.
@@ -105,26 +112,39 @@ class HTMLParser(_markupbase.ParserBase):
self.handle_startendtag(); end tags by self.handle_endtag(). The
data between tags is passed from the parser to the derived class
by calling self.handle_data() with the data as argument (the data
- may be split up in arbitrary chunks). Entity references are
- passed by calling self.handle_entityref() with the entity
- reference as the argument. Numeric character references are
- passed to self.handle_charref() with the string containing the
- reference as the argument.
+ may be split up in arbitrary chunks). If convert_charrefs is
+ True the character references are converted automatically to the
+ corresponding Unicode character (and self.handle_data() is no
+ longer split in chunks), otherwise they are passed by calling
+ self.handle_entityref() or self.handle_charref() with the string
+ containing respectively the named or numeric reference as the
+ argument.
"""
CDATA_CONTENT_ELEMENTS = ("script", "style")
- def __init__(self, strict=False):
+ def __init__(self, strict=_default_sentinel, *,
+ convert_charrefs=_default_sentinel):
"""Initialize and reset this instance.
+ If convert_charrefs is True (default: False), all character references
+ are automatically converted to the corresponding Unicode characters.
If strict is set to False (the default) the parser will parse invalid
markup, otherwise it will raise an error. Note that the strict mode
- is deprecated.
+ and argument are deprecated.
"""
- if strict:
- warnings.warn("The strict mode is deprecated.",
+ if strict is not _default_sentinel:
+ warnings.warn("The strict argument and mode are deprecated.",
DeprecationWarning, stacklevel=2)
+ else:
+ strict = False # default
self.strict = strict
+ if convert_charrefs is _default_sentinel:
+ convert_charrefs = False # default
+ warnings.warn("The value of convert_charrefs will become True in "
+ "3.5. You are encouraged to set the value explicitly.",
+ DeprecationWarning, stacklevel=2)
+ self.convert_charrefs = convert_charrefs
self.reset()
def reset(self):
@@ -149,6 +169,8 @@ class HTMLParser(_markupbase.ParserBase):
self.goahead(1)
def error(self, message):
+ warnings.warn("The 'error' method is deprecated.",
+ DeprecationWarning, stacklevel=2)
raise HTMLParseError(message, self.getpos())
__starttag_text = None
@@ -173,14 +195,25 @@ class HTMLParser(_markupbase.ParserBase):
i = 0
n = len(rawdata)
while i < n:
- match = self.interesting.search(rawdata, i) # < or &
- if match:
- j = match.start()
+ if self.convert_charrefs and not self.cdata_elem:
+ j = rawdata.find('<', i)
+ if j < 0:
+ if not end:
+ break # wait till we get all the text
+ j = n
else:
- if self.cdata_elem:
- break
- j = n
- if i < j: self.handle_data(rawdata[i:j])
+ match = self.interesting.search(rawdata, i) # < or &
+ if match:
+ j = match.start()
+ else:
+ if self.cdata_elem:
+ break
+ j = n
+ if i < j:
+ if self.convert_charrefs and not self.cdata_elem:
+ self.handle_data(unescape(rawdata[i:j]))
+ else:
+ self.handle_data(rawdata[i:j])
i = self.updatepos(i, j)
if i == n: break
startswith = rawdata.startswith
@@ -215,7 +248,10 @@ class HTMLParser(_markupbase.ParserBase):
k = i + 1
else:
k += 1
- self.handle_data(rawdata[i:k])
+ if self.convert_charrefs and not self.cdata_elem:
+ self.handle_data(unescape(rawdata[i:k]))
+ else:
+ self.handle_data(rawdata[i:k])
i = self.updatepos(i, k)
elif startswith("&#", i):
match = charref.match(rawdata, i)
@@ -266,7 +302,10 @@ class HTMLParser(_markupbase.ParserBase):
assert 0, "interesting.search() lied"
# end while
if end and i < n and not self.cdata_elem:
- self.handle_data(rawdata[i:n])
+ if self.convert_charrefs and not self.cdata_elem:
+ self.handle_data(unescape(rawdata[i:n]))
+ else:
+ self.handle_data(rawdata[i:n])
i = self.updatepos(i, n)
self.rawdata = rawdata[i:]
@@ -349,7 +388,7 @@ class HTMLParser(_markupbase.ParserBase):
attrvalue[:1] == '"' == attrvalue[-1:]:
attrvalue = attrvalue[1:-1]
if attrvalue:
- attrvalue = self.unescape(attrvalue)
+ attrvalue = unescape(attrvalue)
attrs.append((attrname.lower(), attrvalue))
k = m.end()
@@ -505,31 +544,7 @@ class HTMLParser(_markupbase.ParserBase):
# Internal -- helper to remove special character quoting
def unescape(self, s):
- if '&' not in s:
- return s
- def replaceEntities(s):
- s = s.groups()[0]
- try:
- if s[0] == "#":
- s = s[1:]
- if s[0] in ['x','X']:
- c = int(s[1:].rstrip(';'), 16)
- else:
- c = int(s.rstrip(';'))
- return chr(c)
- except ValueError:
- return '&#' + s
- else:
- from html.entities import html5
- if s in html5:
- return html5[s]
- elif s.endswith(';'):
- return '&' + s
- for x in range(2, len(s)):
- if s[:x] in html5:
- return html5[s[:x]] + s[x:]
- else:
- return '&' + s
-
- return re.sub(r"&(#?[xX]?(?:[0-9a-fA-F]+;|\w{1,32};?))",
- replaceEntities, s, flags=re.ASCII)
+ warnings.warn('The unescape method is deprecated and will be removed '
+ 'in 3.5, use html.unescape() instead.',
+ DeprecationWarning, stacklevel=2)
+ return unescape(s)
diff --git a/Lib/http/client.py b/Lib/http/client.py
index e05c84d..12c1a5f 100644
--- a/Lib/http/client.py
+++ b/Lib/http/client.py
@@ -73,7 +73,6 @@ import os
import socket
import collections
from urllib.parse import urlsplit
-import warnings
__all__ = ["HTTPResponse", "HTTPConnection",
"HTTPException", "NotConnected", "UnknownProtocol",
@@ -271,8 +270,6 @@ def parse_headers(fp, _class=HTTPMessage):
return email.parser.Parser(_class=_class).parsestr(hstring)
-_strict_sentinel = object()
-
class HTTPResponse(io.RawIOBase):
# See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
@@ -282,7 +279,7 @@ class HTTPResponse(io.RawIOBase):
# text following RFC 2047. The basic status line parsing only
# accepts iso-8859-1.
- def __init__(self, sock, debuglevel=0, strict=_strict_sentinel, method=None, url=None):
+ def __init__(self, sock, debuglevel=0, method=None, url=None):
# If the response includes a content-length header, we need to
# make sure that the client doesn't read more than the
# specified number of bytes. If it does, it will block until
@@ -292,10 +289,6 @@ class HTTPResponse(io.RawIOBase):
# clients unless they know what they are doing.
self.fp = sock.makefile("rb")
self.debuglevel = debuglevel
- if strict is not _strict_sentinel:
- warnings.warn("the 'strict' argument isn't supported anymore; "
- "http.client now always assumes HTTP/1.x compliant servers.",
- DeprecationWarning, 2)
self._method = method
# The HTTPResponse object is returned via urllib. The clients
@@ -732,13 +725,17 @@ class HTTPConnection:
default_port = HTTP_PORT
auto_open = 1
debuglevel = 0
-
- def __init__(self, host, port=None, strict=_strict_sentinel,
- timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
- if strict is not _strict_sentinel:
- warnings.warn("the 'strict' argument isn't supported anymore; "
- "http.client now always assumes HTTP/1.x compliant servers.",
- DeprecationWarning, 2)
+ # TCP Maximum Segment Size (MSS) is determined by the TCP stack on
+ # a per-connection basis. There is no simple and efficient
+ # platform independent mechanism for determining the MSS, so
+ # instead a reasonable estimate is chosen. The getsockopt()
+ # interface using the TCP_MAXSEG parameter may be a suitable
+ # approach on some operating systems. A value of 16KiB is chosen
+ # as a reasonable estimate of the maximum MSS.
+ mss = 16384
+
+ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None):
self.timeout = timeout
self.source_address = source_address
self.sock = None
@@ -804,8 +801,8 @@ class HTTPConnection:
if code != 200:
self.close()
- raise socket.error("Tunnel connection failed: %d %s" % (code,
- message.strip()))
+ raise OSError("Tunnel connection failed: %d %s" % (code,
+ message.strip()))
while True:
line = response.fp.readline(_MAXLINE + 1)
if len(line) > _MAXLINE:
@@ -899,8 +896,11 @@ class HTTPConnection:
del self._buffer[:]
# If msg and message_body are sent in a single send() call,
# it will avoid performance problems caused by the interaction
- # between delayed ack and the Nagle algorithm.
- if isinstance(message_body, bytes):
+ # between delayed ack and the Nagle algorithm. However,
+ # there is no performance gain if the message is larger
+ # than MSS (and there is a memory penalty for the message
+ # copy).
+ if isinstance(message_body, bytes) and len(message_body) < self.mss:
msg += message_body
message_body = None
self.send(msg)
@@ -1170,16 +1170,15 @@ else:
# XXX Should key_file and cert_file be deprecated in favour of context?
def __init__(self, host, port=None, key_file=None, cert_file=None,
- strict=_strict_sentinel, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
- source_address=None, *, context=None, check_hostname=None):
- super(HTTPSConnection, self).__init__(host, port, strict, timeout,
+ timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+ source_address=None, *, context=None,
+ check_hostname=None):
+ super(HTTPSConnection, self).__init__(host, port, timeout,
source_address)
self.key_file = key_file
self.cert_file = cert_file
if context is None:
- # Some reasonable defaults
- context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
- context.options |= ssl.OP_NO_SSLv2
+ context = ssl._create_stdlib_context()
will_verify = context.verify_mode != ssl.CERT_NONE
if check_hostname is None:
check_hostname = will_verify
@@ -1204,13 +1203,13 @@ else:
server_hostname = self.host if ssl.HAS_SNI else None
self.sock = self._context.wrap_socket(sock,
server_hostname=server_hostname)
- try:
- if self._check_hostname:
+ if not self._context.check_hostname and self._check_hostname:
+ try:
ssl.match_hostname(self.sock.getpeercert(), self.host)
- except Exception:
- self.sock.shutdown(socket.SHUT_RDWR)
- self.sock.close()
- raise
+ except Exception:
+ self.sock.shutdown(socket.SHUT_RDWR)
+ self.sock.close()
+ raise
__all__.append("HTTPSConnection")
diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py
index 9fcd4c6..4dc468b 100644
--- a/Lib/http/cookiejar.py
+++ b/Lib/http/cookiejar.py
@@ -1193,8 +1193,7 @@ def deepvalues(mapping):
pass
else:
mapping = True
- for subobj in deepvalues(obj):
- yield subobj
+ yield from deepvalues(obj)
if not mapping:
yield obj
@@ -1731,8 +1730,8 @@ class CookieJar:
return "<%s[%s]>" % (self.__class__, ", ".join(r))
-# derives from IOError for backwards-compatibility with Python 2.4.0
-class LoadError(IOError): pass
+# derives from OSError for backwards-compatibility with Python 2.4.0
+class LoadError(OSError): pass
class FileCookieJar(CookieJar):
"""CookieJar that can be loaded from and saved to a file."""
@@ -1762,17 +1761,14 @@ class FileCookieJar(CookieJar):
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
- f = open(filename)
- try:
+ with open(filename) as f:
self._really_load(f, filename, ignore_discard, ignore_expires)
- finally:
- f.close()
def revert(self, filename=None,
ignore_discard=False, ignore_expires=False):
"""Clear all cookies and reload cookies from a saved file.
- Raises LoadError (or IOError) if reversion is not successful; the
+ Raises LoadError (or OSError) if reversion is not successful; the
object's state will not be altered if this happens.
"""
@@ -1787,7 +1783,7 @@ class FileCookieJar(CookieJar):
self._cookies = {}
try:
self.load(filename, ignore_discard, ignore_expires)
- except (LoadError, IOError):
+ except OSError:
self._cookies = old_state
raise
@@ -1857,15 +1853,12 @@ class LWPCookieJar(FileCookieJar):
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
- f = open(filename, "w")
- try:
+ with open(filename, "w") as f:
# There really isn't an LWP Cookies 2.0 format, but this indicates
# that there is extra information in here (domain_dot and
# port_spec) while still being compatible with libwww-perl, I hope.
f.write("#LWP-Cookies-2.0\n")
f.write(self.as_lwp_str(ignore_discard, ignore_expires))
- finally:
- f.close()
def _really_load(self, f, filename, ignore_discard, ignore_expires):
magic = f.readline()
@@ -1938,8 +1931,7 @@ class LWPCookieJar(FileCookieJar):
if not ignore_expires and c.is_expired(now):
continue
self.set_cookie(c)
-
- except IOError:
+ except OSError:
raise
except Exception:
_warn_unhandled_exception()
@@ -2045,7 +2037,7 @@ class MozillaCookieJar(FileCookieJar):
continue
self.set_cookie(c)
- except IOError:
+ except OSError:
raise
except Exception:
_warn_unhandled_exception()
@@ -2057,8 +2049,7 @@ class MozillaCookieJar(FileCookieJar):
if self.filename is not None: filename = self.filename
else: raise ValueError(MISSING_FILENAME_TEXT)
- f = open(filename, "w")
- try:
+ with open(filename, "w") as f:
f.write(self.header)
now = time.time()
for cookie in self:
@@ -2087,5 +2078,3 @@ class MozillaCookieJar(FileCookieJar):
"\t".join([cookie.domain, initial_dot, cookie.path,
secure, expires, name, value])+
"\n")
- finally:
- f.close()
diff --git a/Lib/http/server.py b/Lib/http/server.py
index dab1eb6..6ce6bda 100644
--- a/Lib/http/server.py
+++ b/Lib/http/server.py
@@ -85,8 +85,6 @@ __version__ = "0.6"
__all__ = ["HTTPServer", "BaseHTTPRequestHandler"]
import html
-import email.message
-import email.parser
import http.client
import io
import mimetypes
@@ -401,12 +399,17 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
while not self.close_connection:
self.handle_one_request()
- def send_error(self, code, message=None):
+ def send_error(self, code, message=None, explain=None):
"""Send and log an error reply.
- Arguments are the error code, and a detailed message.
- The detailed message defaults to the short entry matching the
- response code.
+ Arguments are
+ * code: an HTTP error code
+ 3 digits
+ * message: a simple optional 1 line reason phrase.
+ *( HTAB / SP / VCHAR / %x80-FF )
+ defaults to short entry matching the response code
+ * explain: a detailed message defaults to the long entry
+ matching the response code.
This sends an error response (so it must be called before any
output has been generated), logs the error, and finally sends
@@ -420,17 +423,20 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler):
shortmsg, longmsg = '???', '???'
if message is None:
message = shortmsg
- explain = longmsg
+ if explain is None:
+ explain = longmsg
self.log_error("code %d, message %s", code, message)
# using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
content = (self.error_message_format %
- {'code': code, 'message': _quote_html(message), 'explain': explain})
+ {'code': code, 'message': _quote_html(message), 'explain': _quote_html(explain)})
+ body = content.encode('UTF-8', 'replace')
self.send_response(code, message)
self.send_header("Content-Type", self.error_content_type)
self.send_header('Connection', 'close')
+ self.send_header('Content-Length', int(len(body)))
self.end_headers()
if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
- self.wfile.write(content.encode('UTF-8', 'replace'))
+ self.wfile.write(body)
def send_response(self, code, message=None):
"""Add the response header to the headers buffer and log the
@@ -711,7 +717,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
ctype = self.guess_type(path)
try:
f = open(path, 'rb')
- except IOError:
+ except OSError:
self.send_error(404, "File not found")
return None
try:
@@ -736,7 +742,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
"""
try:
list = os.listdir(path)
- except os.error:
+ except OSError:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
@@ -1130,7 +1136,7 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
try:
try:
os.setuid(nobody)
- except os.error:
+ except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
@@ -1183,15 +1189,15 @@ class CGIHTTPRequestHandler(SimpleHTTPRequestHandler):
self.log_message("CGI script exited OK")
-def test(HandlerClass = BaseHTTPRequestHandler,
- ServerClass = HTTPServer, protocol="HTTP/1.0", port=8000):
+def test(HandlerClass=BaseHTTPRequestHandler,
+ ServerClass=HTTPServer, protocol="HTTP/1.0", port=8000, bind=""):
"""Test the HTTP request handler class.
This runs an HTTP server on port 8000 (or the first command line
argument).
"""
- server_address = ('', port)
+ server_address = (bind, port)
HandlerClass.protocol_version = protocol
httpd = ServerClass(server_address, HandlerClass)
@@ -1209,12 +1215,16 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--cgi', action='store_true',
help='Run as CGI Server')
+ parser.add_argument('--bind', '-b', default='', metavar='ADDRESS',
+ help='Specify alternate bind address '
+ '[default: all interfaces]')
parser.add_argument('port', action='store',
default=8000, type=int,
nargs='?',
help='Specify alternate port [default: 8000]')
args = parser.parse_args()
if args.cgi:
- test(HandlerClass=CGIHTTPRequestHandler, port=args.port)
+ handler_class = CGIHTTPRequestHandler
else:
- test(HandlerClass=SimpleHTTPRequestHandler, port=args.port)
+ handler_class = SimpleHTTPRequestHandler
+ test(HandlerClass=handler_class, port=args.port, bind=args.bind)
diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py
index 65c0317..df2b251 100644
--- a/Lib/idlelib/Bindings.py
+++ b/Lib/idlelib/Bindings.py
@@ -8,9 +8,14 @@ the PythonShell window, and a Format menu which is only present in the Editor
windows.
"""
-import sys
from idlelib.configHandler import idleConf
-from idlelib import macosxSupport
+
+# Warning: menudefs is altered in macosxSupport.overrideRootMenu()
+# after it is determined that an OS X Aqua Tk is in use,
+# which cannot be done until after Tk() is first called.
+# Do not alter the 'file', 'options', or 'help' cascades here
+# without altering overrideRootMenu() as well.
+# TODO: Make this more robust
menudefs = [
# underscore prefixes character to underscore
@@ -81,27 +86,4 @@ menudefs = [
]),
]
-if macosxSupport.runningAsOSXApp():
- # Running as a proper MacOS application bundle. This block restructures
- # the menus a little to make them conform better to the HIG.
-
- quitItem = menudefs[0][1][-1]
- closeItem = menudefs[0][1][-2]
-
- # Remove the last 3 items of the file menu: a separator, close window and
- # quit. Close window will be reinserted just above the save item, where
- # it should be according to the HIG. Quit is in the application menu.
- del menudefs[0][1][-3:]
- menudefs[0][1].insert(6, closeItem)
-
- # Remove the 'About' entry from the help menu, it is in the application
- # menu
- del menudefs[-1][1][0:2]
-
- # Remove the 'Configure' entry from the options menu, it is in the
- # application menu as 'Preferences'
- del menudefs[-2][1][0:2]
-
default_keydefs = idleConf.GetCurrentKeySet()
-
-del sys
diff --git a/Lib/idlelib/Debugger.py b/Lib/idlelib/Debugger.py
index d4872ed..ca98b10 100644
--- a/Lib/idlelib/Debugger.py
+++ b/Lib/idlelib/Debugger.py
@@ -322,7 +322,7 @@ class Debugger:
class StackViewer(ScrolledList):
def __init__(self, master, flist, gui):
- if macosxSupport.runningAsOSXApp():
+ if macosxSupport.isAquaTk():
# At least on with the stock AquaTk version on OSX 10.4 you'll
# get an shaking GUI that eventually kills IDLE if the width
# argument is specified.
diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py
index 4bf1111..cdb6775 100644
--- a/Lib/idlelib/EditorWindow.py
+++ b/Lib/idlelib/EditorWindow.py
@@ -1,5 +1,6 @@
import importlib
import importlib.abc
+import importlib.util
import os
from platform import python_version
import re
@@ -108,8 +109,8 @@ class EditorWindow(object):
'Python%s.chm' % _sphinx_version())
if os.path.isfile(chmfile):
dochome = chmfile
- elif macosxSupport.runningAsOSXApp():
- # documentation is stored inside the python framework
+ elif sys.platform == 'darwin':
+ # documentation may be stored inside a python framework
dochome = os.path.join(sys.base_prefix,
'Resources/English.lproj/Documentation/index.html')
dochome = os.path.normpath(dochome)
@@ -165,7 +166,7 @@ class EditorWindow(object):
self.top.protocol("WM_DELETE_WINDOW", self.close)
self.top.bind("<<close-window>>", self.close_event)
- if macosxSupport.runningAsOSXApp():
+ if macosxSupport.isAquaTk():
# Command-W on editorwindows doesn't work without this.
text.bind('<<close-window>>', self.close_event)
# Some OS X systems have only one mouse button,
@@ -408,7 +409,7 @@ class EditorWindow(object):
def set_status_bar(self):
self.status_bar = self.MultiStatusBar(self.top)
- if macosxSupport.runningAsOSXApp():
+ if sys.platform == "darwin":
# Insert some padding to avoid obscuring some of the statusbar
# by the resize widget.
self.status_bar.set_label('_padding1', ' ', side=RIGHT)
@@ -435,7 +436,7 @@ class EditorWindow(object):
("help", "_Help"),
]
- if macosxSupport.runningAsOSXApp():
+ if sys.platform == "darwin":
menu_specs[-2] = ("windows", "_Window")
@@ -446,7 +447,7 @@ class EditorWindow(object):
underline, label = prepstr(label)
menudict[name] = menu = Menu(mbar, name=name)
mbar.add_cascade(label=label, menu=menu, underline=underline)
- if macosxSupport.isCarbonAquaTk(self.root):
+ if macosxSupport.isCarbonTk():
# Insert the application menu
menudict['application'] = menu = Menu(mbar, name='apple')
mbar.add_cascade(label='IDLE', menu=menu)
@@ -549,7 +550,7 @@ class EditorWindow(object):
if sys.platform[:3] == 'win':
try:
os.startfile(self.help_url)
- except WindowsError as why:
+ except OSError as why:
tkMessageBox.showerror(title='Document Start Failure',
message=str(why), parent=self.text)
else:
@@ -660,20 +661,20 @@ class EditorWindow(object):
return
# XXX Ought to insert current file's directory in front of path
try:
- loader = importlib.find_loader(name)
+ spec = importlib.util.find_spec(name)
except (ValueError, ImportError) as msg:
tkMessageBox.showerror("Import error", str(msg), parent=self.text)
return
- if loader is None:
+ if spec is None:
tkMessageBox.showerror("Import error", "module not found",
parent=self.text)
return
- if not isinstance(loader, importlib.abc.SourceLoader):
+ if not isinstance(spec.loader, importlib.abc.SourceLoader):
tkMessageBox.showerror("Import error", "not a source-based module",
parent=self.text)
return
try:
- file_path = loader.get_filename(name)
+ file_path = spec.loader.get_filename(name)
except AttributeError:
tkMessageBox.showerror("Import error",
"loader does not support get_filename",
@@ -872,7 +873,7 @@ class EditorWindow(object):
if sys.platform[:3] == 'win':
try:
os.startfile(helpfile)
- except WindowsError as why:
+ except OSError as why:
tkMessageBox.showerror(title='Document Start Failure',
message=str(why), parent=self.text)
else:
@@ -1672,7 +1673,7 @@ def get_accelerator(keydefs, eventname):
keylist = keydefs.get(eventname)
# issue10940: temporary workaround to prevent hang with OS X Cocoa Tk 8.5
# if not keylist:
- if (not keylist) or (macosxSupport.runningAsOSXApp() and eventname in {
+ if (not keylist) or (macosxSupport.isCocoaTk() and eventname in {
"<<open-module>>",
"<<goto-line>>",
"<<change-indentwidth>>"}):
diff --git a/Lib/idlelib/FileList.py b/Lib/idlelib/FileList.py
index 37a337e..a9989a8 100644
--- a/Lib/idlelib/FileList.py
+++ b/Lib/idlelib/FileList.py
@@ -103,7 +103,7 @@ class FileList:
if not os.path.isabs(filename):
try:
pwd = os.getcwd()
- except os.error:
+ except OSError:
pass
else:
filename = os.path.join(pwd, filename)
diff --git a/Lib/idlelib/GrepDialog.py b/Lib/idlelib/GrepDialog.py
index c359074..f73d70a 100644
--- a/Lib/idlelib/GrepDialog.py
+++ b/Lib/idlelib/GrepDialog.py
@@ -125,4 +125,3 @@ if __name__ == "__main__":
# Hence Idle must be restarted after editing this file for a live test.
import unittest
unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False)
-
diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py
index f008b46..3cd7a4c 100644
--- a/Lib/idlelib/IOBinding.py
+++ b/Lib/idlelib/IOBinding.py
@@ -506,7 +506,7 @@ class IOBinding:
else:
try:
pwd = os.getcwd()
- except os.error:
+ except OSError:
pwd = ""
return pwd, ""
diff --git a/Lib/idlelib/MultiCall.py b/Lib/idlelib/MultiCall.py
index 64729ea..cc6bffd 100644
--- a/Lib/idlelib/MultiCall.py
+++ b/Lib/idlelib/MultiCall.py
@@ -32,7 +32,6 @@ Each function will be called at most once for each event.
import sys
import re
import tkinter
-from idlelib import macosxSupport
# the event type constants, which define the meaning of mc_type
MC_KEYPRESS=0; MC_KEYRELEASE=1; MC_BUTTONPRESS=2; MC_BUTTONRELEASE=3;
@@ -45,7 +44,7 @@ MC_SHIFT = 1<<0; MC_CONTROL = 1<<2; MC_ALT = 1<<3; MC_META = 1<<5
MC_OPTION = 1<<6; MC_COMMAND = 1<<7
# define the list of modifiers, to be used in complex event types.
-if macosxSupport.runningAsOSXApp():
+if sys.platform == "darwin":
_modifiers = (("Shift",), ("Control",), ("Option",), ("Command",))
_modifier_masks = (MC_SHIFT, MC_CONTROL, MC_OPTION, MC_COMMAND)
else:
@@ -57,6 +56,13 @@ _modifier_names = dict([(name, number)
for number in range(len(_modifiers))
for name in _modifiers[number]])
+# In 3.4, if no shell window is ever open, the underlying Tk widget is
+# destroyed before .__del__ methods here are called. The following
+# is used to selectively ignore shutdown exceptions to avoid
+# 'Exception ignored' messages. See http://bugs.python.org/issue20167
+APPLICATION_GONE = '''\
+can't invoke "bind" command: application has been destroyed'''
+
# A binder is a class which binds functions to one type of event. It has two
# methods: bind and unbind, which get a function and a parsed sequence, as
# returned by _parse_sequence(). There are two types of binders:
@@ -98,7 +104,14 @@ class _SimpleBinder:
def __del__(self):
if self.handlerid:
- self.widget.unbind(self.widgetinst, self.sequence, self.handlerid)
+ try:
+ self.widget.unbind(self.widgetinst, self.sequence,
+ self.handlerid)
+ except tkinter.TclError as e:
+ if e.args[0] == APPLICATION_GONE:
+ pass
+ else:
+ raise
# An int in range(1 << len(_modifiers)) represents a combination of modifiers
# (if the least significent bit is on, _modifiers[0] is on, and so on).
@@ -227,7 +240,13 @@ class _ComplexBinder:
def __del__(self):
for seq, id in self.handlerids:
- self.widget.unbind(self.widgetinst, seq, id)
+ try:
+ self.widget.unbind(self.widgetinst, seq, id)
+ except tkinter.TclError as e:
+ if e.args[0] == APPLICATION_GONE:
+ break
+ else:
+ raise
# define the list of event types to be handled by MultiEvent. the order is
# compatible with the definition of event type constants.
@@ -390,8 +409,13 @@ def MultiCallCreator(widget):
func, triplets = self.__eventinfo[virtual]
if func:
for triplet in triplets:
- self.__binders[triplet[1]].unbind(triplet, func)
-
+ try:
+ self.__binders[triplet[1]].unbind(triplet, func)
+ except tkinter.TclError as e:
+ if e.args[0] == APPLICATION_GONE:
+ break
+ else:
+ raise
_multicall_dict[widget] = MultiCall
return MultiCall
diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 6388d0d..953a38d 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -1,74 +1,10 @@
-What's New in IDLE 3.3.4?
-=========================
-
-- Issue #17390: Add Python version to Idle editor window title bar.
- Original patches by Edmond Burnett and Kent Johnson.
-
-- Issue #18960: IDLE now ignores the source encoding declaration on the second
- line if the first line contains anything except a comment.
-
-- Issue #20058: sys.stdin.readline() in IDLE now always returns only one line.
-
-- Issue #19481: print() of string subclass instance in IDLE no longer hangs.
-
-- Issue #18270: Prevent possible IDLE AttributeError on OS X when no initial
- shell window is present.
-
-
-What's New in IDLE 3.3.3?
-=========================
-
-- Issue #18873: IDLE now detects Python source code encoding only in comment
- lines.
-
-- Issue #18988: The "Tab" key now works when a word is already autocompleted.
-
-- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster.
-
-- Issue #18429: Format / Format Paragraph, now works when comment blocks
- are selected. As with text blocks, this works best when the selection
- only includes complete lines.
-
-- Issue #18226: Add docstrings and unittests for FormatParagraph.py.
- Original patches by Todd Rovito and Phil Webster.
-
-- Issue #18279: Format - Strip trailing whitespace no longer marks a file as
- changed when it has not been changed. This fix followed the addition of a
- test file originally written by Phil Webster (the issue's main goal).
-
-- Issue #7136: In the Idle File menu, "New Window" is renamed "New File".
- Patch by Tal Einat, Roget Serwy, and Todd Rovito.
-
-- Remove dead imports of imp.
-
-- Issue #18196: Avoid displaying spurious SystemExit tracebacks.
-
-- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition.
-
-- Issue #17511: Keep IDLE find dialog open after clicking "Find Next".
- Original patch by Sarah K.
-
-- Issue #18055: Move IDLE off of imp and on to importlib.
-
-- Issue #15392: Create a unittest framework for IDLE.
- Initial patch by Rajagopalasarma Jayakrishnan.
- See Lib/idlelib/idle_test/README.txt for how to run Idle tests.
-
-- Issue #14146: Highlight source line while debugging on Windows.
-
-- Issue #17532: Always include Options menu for IDLE on OS X.
- Patch by Guilherme Simões.
-
-
-What's New in IDLE 3.3.2?
+What's New in IDLE 3.4.0?
=========================
- Issue #17390: Display Python version on Idle title bar.
Initial patch by Edmond Burnett.
-
-What's New in IDLE 3.3.1?
-=========================
+- Issue #5066: Update IDLE docs. Patch by Todd Rovito.
- Issue #17625: Close the replace dialog after it is used.
diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py
index ba40719..5e5c6be 100644
--- a/Lib/idlelib/PathBrowser.py
+++ b/Lib/idlelib/PathBrowser.py
@@ -44,7 +44,7 @@ class DirBrowserTreeItem(TreeItem):
def GetSubList(self):
try:
names = os.listdir(self.dir or os.curdir)
- except os.error:
+ except OSError:
return []
packages = []
for name in names:
diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py
index 2e5ebb2..c23b62a 100755
--- a/Lib/idlelib/PyShell.py
+++ b/Lib/idlelib/PyShell.py
@@ -419,7 +419,7 @@ class ModifiedInterpreter(InteractiveInterpreter):
try:
self.rpcclt = MyRPCClient(addr)
break
- except socket.error as err:
+ except OSError as err:
pass
else:
self.display_port_binding_error()
@@ -844,7 +844,7 @@ class PyShell(OutputWindow):
("help", "_Help"),
]
- if macosxSupport.runningAsOSXApp():
+ if sys.platform == "darwin":
menu_specs[-2] = ("windows", "_Window")
@@ -1034,7 +1034,10 @@ class PyShell(OutputWindow):
self.close()
return False
else:
- nosub = "==== No Subprocess ===="
+ nosub = ("==== No Subprocess ====\n\n" +
+ "WARNING: Running IDLE without a Subprocess is deprecated\n" +
+ "and will be removed in a later version. See Help/IDLE Help\n" +
+ "for details.\n\n")
sys.displayhook = rpc.displayhook
self.write("Python %s on %s\n%s\n%s" %
@@ -1398,7 +1401,8 @@ USAGE: idle [-deins] [-t title] [file]*
idle [-dns] [-t title] - [arg]*
-h print this help message and exit
- -n run IDLE without a subprocess (see Help/IDLE Help for details)
+ -n run IDLE without a subprocess (DEPRECATED,
+ see Help/IDLE Help for details)
The following options will override the IDLE 'settings' configuration:
@@ -1476,6 +1480,8 @@ def main():
if o == '-i':
enable_shell = True
if o == '-n':
+ print(" Warning: running IDLE without a subprocess is deprecated.",
+ file=sys.stderr)
use_subprocess = False
if o == '-r':
script = a
@@ -1554,7 +1560,7 @@ def main():
shell = flist.open_shell()
if not shell:
return # couldn't open shell
- if macosxSupport.runningAsOSXApp() and flist.dict:
+ if macosxSupport.isAquaTk() and flist.dict:
# On OSX: when the user has double-clicked on a file that causes
# IDLE to be launched the shell window will open just in front of
# the file she wants to see. Lower the interpreter window when
diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py
index 6bfe128..b783637 100644
--- a/Lib/idlelib/ScriptBinding.py
+++ b/Lib/idlelib/ScriptBinding.py
@@ -53,7 +53,7 @@ class ScriptBinding:
self.flist = self.editwin.flist
self.root = self.editwin.root
- if macosxSupport.runningAsOSXApp():
+ if macosxSupport.isCocoaTk():
self.editwin.text_frame.bind('<<run-module-event-2>>', self._run_module_event)
def check_module_event(self, event):
@@ -114,7 +114,7 @@ class ScriptBinding:
shell.set_warning_stream(saved_stream)
def run_module_event(self, event):
- if macosxSupport.runningAsOSXApp():
+ if macosxSupport.isCocoaTk():
# Tk-Cocoa in MacOSX is broken until at least
# Tk 8.5.9, and without this rather
# crude workaround IDLE would hang when a user
diff --git a/Lib/idlelib/TreeWidget.py b/Lib/idlelib/TreeWidget.py
index 25bae48..1f4854d 100644
--- a/Lib/idlelib/TreeWidget.py
+++ b/Lib/idlelib/TreeWidget.py
@@ -381,7 +381,7 @@ class FileTreeItem(TreeItem):
try:
os.rename(self.path, newpath)
self.path = newpath
- except os.error:
+ except OSError:
pass
def GetIconName(self):
@@ -394,7 +394,7 @@ class FileTreeItem(TreeItem):
def GetSubList(self):
try:
names = os.listdir(self.path)
- except os.error:
+ except OSError:
return []
names.sort(key = os.path.normcase)
sublist = []
diff --git a/Lib/idlelib/ZoomHeight.py b/Lib/idlelib/ZoomHeight.py
index e8d1710..a5d679e 100644
--- a/Lib/idlelib/ZoomHeight.py
+++ b/Lib/idlelib/ZoomHeight.py
@@ -32,7 +32,7 @@ def zoom_height(top):
newy = 0
newheight = newheight - 72
- elif macosxSupport.runningAsOSXApp():
+ elif macosxSupport.isAquaTk():
# The '88' below is a magic number that avoids placing the bottom
# of the window below the panel on my machine. I don't know how
# to calculate the correct value for this with tkinter.
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
index efe5c43..fefe42b 100644
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -74,7 +74,7 @@ class ConfigDialog(Toplevel):
frameActionButtons = Frame(self,pady=2)
#action buttons
- if macosxSupport.runningAsOSXApp():
+ if macosxSupport.isAquaTk():
# Surpress the padx and pady arguments when
# running as IDLE.app, otherwise the text
# on these buttons will not be readable.
diff --git a/Lib/idlelib/configHandler.py b/Lib/idlelib/configHandler.py
index a974d54..8608f7c 100644
--- a/Lib/idlelib/configHandler.py
+++ b/Lib/idlelib/configHandler.py
@@ -20,7 +20,6 @@ configuration problem notification and resolution.
import os
import sys
-from idlelib import macosxSupport
from configparser import ConfigParser, NoOptionError, NoSectionError
class InvalidConfigType(Exception): pass
@@ -271,8 +270,10 @@ class IdleConf:
except OSError:
pass
return default
+
def SetOption(self, configType, section, option, value):
"""In user's config file, set section's option to value.
+
"""
self.userCfg[configType].SetOption(section, option, value)
@@ -525,10 +526,13 @@ class IdleConf:
def GetCurrentKeySet(self):
result = self.GetKeySet(self.CurrentKeys())
- if macosxSupport.runningAsOSXApp():
- # We're using AquaTk, replace all keybingings that use the
- # Alt key by ones that use the Option key because the former
- # don't work reliably.
+ if sys.platform == "darwin":
+ # OS X Tk variants do not support the "Alt" keyboard modifier.
+ # So replace all keybingings that use "Alt" with ones that
+ # use the "Option" keyboard modifier.
+ # TO DO: the "Option" modifier does not work properly for
+ # Cocoa Tk and XQuartz Tk so we should not use it
+ # in default OS X KeySets.
for k, v in result.items():
v2 = [ x.replace('<Alt-', '<Option-') for x in v ]
if v != v2:
@@ -638,8 +642,10 @@ class IdleConf:
except OSError:
pass
return keyBindings
+
def GetExtraHelpSourceList(self,configSet):
"""Fetch list of extra help sources from a given configSet.
+
Valid configSets are 'user' or 'default'. Return a list of tuples of
the form (menu_item , path_to_help_file , option), or return the empty
list. 'option' is the sequence number of the help resource. 'option'
diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt
index ff786c5..6378a2e 100644
--- a/Lib/idlelib/help.txt
+++ b/Lib/idlelib/help.txt
@@ -1,142 +1,185 @@
[See the end of this file for ** TIPS ** on using IDLE !!]
-Click on the dotted line at the top of a menu to "tear it off": a
-separate window containing the menu is created.
-
-File Menu:
-
- New File -- Create a new file editing window
- Open... -- Open an existing file
- Recent Files... -- Open a list of recent files
- Open Module... -- Open an existing module (searches sys.path)
- Class Browser -- Show classes and methods in current file
- Path Browser -- Show sys.path directories, modules, classes
+IDLE is the Python IDE built with the tkinter GUI toolkit.
+
+IDLE has the following features:
+-coded in 100% pure Python, using the tkinter GUI toolkit
+-cross-platform: works on Windows, Unix, and OS X
+-multi-window text editor with multiple undo, Python colorizing, smart indent,
+call tips, and many other features
+-Python shell window (a.k.a interactive interpreter)
+-debugger (not complete, but you can set breakpoints, view and step)
+
+Menus:
+
+IDLE has two window types the Shell window and the Editor window. It is
+possible to have multiple editor windows simultaneously. IDLE's
+menus dynamically change based on which window is currently selected. Each menu
+documented below indicates which window type it is associated with. Click on
+the dotted line at the top of a menu to "tear it off": a separate window
+containing the menu is created (for Unix and Windows only).
+
+File Menu (Shell and Editor):
+
+ New File -- Create a new file editing window
+ Open... -- Open an existing file
+ Open Module... -- Open an existing module (searches sys.path)
+ Recent Files... -- Open a list of recent files
+ Class Browser -- Show classes and methods in current file
+ Path Browser -- Show sys.path directories, modules, classes,
and methods
- ---
- Save -- Save current window to the associated file (unsaved
- windows have a * before and after the window title)
-
- Save As... -- Save current window to new file, which becomes
- the associated file
- Save Copy As... -- Save current window to different file
- without changing the associated file
- ---
- Print Window -- Print the current window
- ---
- Close -- Close current window (asks to save if unsaved)
- Exit -- Close all windows, quit (asks to save if unsaved)
-
-Edit Menu:
-
- Undo -- Undo last change to current window
- (A maximum of 1000 changes may be undone)
- Redo -- Redo last undone change to current window
- ---
- Cut -- Copy a selection into system-wide clipboard,
+ ---
+ Save -- Save current window to the associated file (unsaved
+ windows have a * before and after the window title)
+
+ Save As... -- Save current window to new file, which becomes
+ the associated file
+ Save Copy As... -- Save current window to different file
+ without changing the associated file
+ ---
+ Print Window -- Print the current window
+ ---
+ Close -- Close current window (asks to save if unsaved)
+ Exit -- Close all windows, quit (asks to save if unsaved)
+
+Edit Menu (Shell and Editor):
+
+ Undo -- Undo last change to current window
+ (a maximum of 1000 changes may be undone)
+ Redo -- Redo last undone change to current window
+ ---
+ Cut -- Copy a selection into system-wide clipboard,
then delete the selection
- Copy -- Copy selection into system-wide clipboard
- Paste -- Insert system-wide clipboard into window
- Select All -- Select the entire contents of the edit buffer
- ---
- Find... -- Open a search dialog box with many options
- Find Again -- Repeat last search
- Find Selection -- Search for the string in the selection
- Find in Files... -- Open a search dialog box for searching files
- Replace... -- Open a search-and-replace dialog box
- Go to Line -- Ask for a line number and show that line
- Show Calltip -- Open a small window with function param hints
- Show Completions -- Open a scroll window allowing selection keywords
- and attributes. (see '*TIPS*', below)
- Show Parens -- Highlight the surrounding parenthesis
- Expand Word -- Expand the word you have typed to match another
- word in the same buffer; repeat to get a
+ Copy -- Copy selection into system-wide clipboard
+ Paste -- Insert system-wide clipboard into window
+ Select All -- Select the entire contents of the edit buffer
+ ---
+ Find... -- Open a search dialog box with many options
+ Find Again -- Repeat last search
+ Find Selection -- Search for the string in the selection
+ Find in Files... -- Open a search dialog box for searching files
+ Replace... -- Open a search-and-replace dialog box
+ Go to Line -- Ask for a line number and show that line
+ Expand Word -- Expand the word you have typed to match another
+ word in the same buffer; repeat to get a
different expansion
-
-Format Menu (only in Edit window):
-
- Indent Region -- Shift selected lines right 4 spaces
- Dedent Region -- Shift selected lines left 4 spaces
- Comment Out Region -- Insert ## in front of selected lines
- Uncomment Region -- Remove leading # or ## from selected lines
- Tabify Region -- Turns *leading* stretches of spaces into tabs
- (Note: We recommend using 4 space blocks to indent Python code.)
- Untabify Region -- Turn *all* tabs into the right number of spaces
- New Indent Width... -- Open dialog to change indent width
- Format Paragraph -- Reformat the current blank-line-separated
- paragraph
-
-Run Menu (only in Edit window):
-
- Python Shell -- Open or wake up the Python shell window
- ---
- Check Module -- Run a syntax check on the module
- Run Module -- Execute the current file in the __main__ namespace
-
-Shell Menu (only in Shell window):
-
- View Last Restart -- Scroll the shell window to the last restart
- Restart Shell -- Restart the interpreter with a fresh environment
-
-Debug Menu (only in Shell window):
-
- Go to File/Line -- look around the insert point for a filename
- and line number, open the file, and show the line
- Debugger (toggle) -- Run commands in the shell under the debugger
- Stack Viewer -- Show the stack traceback of the last exception
- Auto-open Stack Viewer (toggle) -- Open stack viewer on traceback
-
-Options Menu:
-
- Configure IDLE -- Open a configuration dialog. Fonts, indentation,
+ Show Calltip -- After an unclosed parenthesis for a function, open
+ a small window with function parameter hints
+ Show Parens -- Highlight the surrounding parenthesis
+ Show Completions -- Open a scroll window allowing selection keywords
+ and attributes. (see '*TIPS*', below)
+
+Format Menu (Editor window only):
+
+ Indent Region -- Shift selected lines right by the indent width
+ (default 4 spaces)
+ Dedent Region -- Shift selected lines left by the indent width
+ (default 4 spaces)
+ Comment Out Region -- Insert ## in front of selected lines
+ Uncomment Region -- Remove leading # or ## from selected lines
+ Tabify Region -- Turns *leading* stretches of spaces into tabs.
+ (Note: We recommend using 4 space blocks to indent Python code.)
+ Untabify Region -- Turn *all* tabs into the corrent number of spaces
+ Toggle tabs -- Open a dialog to switch between indenting with
+ spaces and tabs.
+ New Indent Width... -- Open a dialog to change indent width. The
+ accepted default by the Python community is 4
+ spaces.
+ Format Paragraph -- Reformat the current blank-line-separated
+ paragraph. All lines in the paragraph will be
+ formatted to less than 80 columns.
+ ---
+ Strip trailing whitespace -- Removed any space characters after the end
+ of the last non-space character
+
+Run Menu (Editor window only):
+
+ Python Shell -- Open or wake up the Python shell window
+ ---
+ Check Module -- Check the syntax of the module currently open in the
+ Editor window. If the module has not been saved IDLE
+ will prompt the user to save the code.
+ Run Module -- Restart the shell to clean the environment, then
+ execute the currently open module. If the module has
+ not been saved IDLE will prompt the user to save the
+ code.
+
+Shell Menu (Shell window only):
+
+ View Last Restart -- Scroll the shell window to the last Shell restart
+ Restart Shell -- Restart the shell to clean the environment
+
+Debug Menu (Shell window only):
+
+ Go to File/Line -- Look around the insert point for a filename
+ and line number, open the file, and show the line.
+ Useful to view the source lines referenced in an
+ exception traceback. Available in the context
+ menu of the Shell window.
+ Debugger (toggle) -- This feature is not complete and considered
+ experimental. Run commands in the shell under the
+ debugger.
+ Stack Viewer -- Show the stack traceback of the last exception
+ Auto-open Stack Viewer (toggle) -- Toggle automatically opening the
+ stack viewer on unhandled
+ exception
+
+Options Menu (Shell and Editor):
+
+ Configure IDLE -- Open a configuration dialog. Fonts, indentation,
keybindings, and color themes may be altered.
- Startup Preferences may be set, and Additional Help
- Sources can be specified.
-
- On OS X this menu is not present, use
- menu 'IDLE -> Preferences...' instead.
- ---
- Code Context -- Open a pane at the top of the edit window which
- shows the block context of the section of code
- which is scrolling off the top or the window.
- (Not present in Shell window.)
-
-Windows Menu:
-
- Zoom Height -- toggles the window between configured size
- and maximum height.
- ---
- The rest of this menu lists the names of all open windows;
- select one to bring it to the foreground (deiconifying it if
- necessary).
+ Startup Preferences may be set, and additional Help
+ sources can be specified.
+
+ ---
+ Code Context (toggle) -- Open a pane at the top of the edit window
+ which shows the block context of the section
+ of code which is scrolling off the top or the
+ window. This is not present in the Shell
+ window only the Editor window.
+
+Windows Menu (Shell and Editor):
+
+ Zoom Height -- Toggles the window between normal size (40x80 initial
+ setting) and maximum height. The initial size is in the Configure
+ IDLE dialog under the general tab.
+ ---
+ The rest of this menu lists the names of all open windows;
+ select one to bring it to the foreground (deiconifying it if
+ necessary).
Help Menu:
- About IDLE -- Version, copyright, license, credits
- IDLE Readme -- Background discussion and change details
- ---
- IDLE Help -- Display this file
- Python Docs -- Access local Python documentation, if
- installed. Otherwise, access www.python.org.
- ---
- (Additional Help Sources may be added here)
-
-Edit context menu (Right-click / Control-click on OS X in Edit window):
-
- Cut -- Copy a selection into system-wide clipboard,
+ About IDLE -- Version, copyright, license, credits
+ ---
+ IDLE Help -- Display this file which is a help file for IDLE
+ detailing the menu options, basic editing and navigation,
+ and other tips.
+ Python Docs -- Access local Python documentation, if
+ installed. Or will start a web browser and open
+ docs.python.org showing the latest Python documentation.
+ ---
+ Additional help sources may be added here with the Configure IDLE
+ dialog under the General tab.
+
+Editor context menu (Right-click / Control-click on OS X in Edit window):
+
+ Cut -- Copy a selection into system-wide clipboard,
then delete the selection
- Copy -- Copy selection into system-wide clipboard
- Paste -- Insert system-wide clipboard into window
- Set Breakpoint -- Sets a breakpoint (when debugger open)
- Clear Breakpoint -- Clears the breakpoint on that line
+ Copy -- Copy selection into system-wide clipboard
+ Paste -- Insert system-wide clipboard into window
+ Set Breakpoint -- Sets a breakpoint. Breakpoints are only enabled
+ when the debugger is open.
+ Clear Breakpoint -- Clears the breakpoint on that line
Shell context menu (Right-click / Control-click on OS X in Shell window):
- Cut -- Copy a selection into system-wide clipboard,
+ Cut -- Copy a selection into system-wide clipboard,
then delete the selection
- Copy -- Copy selection into system-wide clipboard
- Paste -- Insert system-wide clipboard into window
- ---
- Go to file/line -- Same as in Debug menu
+ Copy -- Copy selection into system-wide clipboard
+ Paste -- Insert system-wide clipboard into window
+ ---
+ Go to file/line -- Same as in Debug menu
** TIPS **
@@ -144,159 +187,182 @@ Shell context menu (Right-click / Control-click on OS X in Shell window):
Additional Help Sources:
- Windows users can Google on zopeshelf.chm to access Zope help files in
- the Windows help format. The Additional Help Sources feature of the
- configuration GUI supports .chm, along with any other filetypes
- supported by your browser. Supply a Menu Item title, and enter the
- location in the Help File Path slot of the New Help Source dialog. Use
- http:// and/or www. to identify external URLs, or download the file and
- browse for its path on your machine using the Browse button.
+ Windows users can Google on zopeshelf.chm to access Zope help files in
+ the Windows help format. The Additional Help Sources feature of the
+ configuration GUI supports .chm, along with any other filetypes
+ supported by your browser. Supply a Menu Item title, and enter the
+ location in the Help File Path slot of the New Help Source dialog. Use
+ http:// and/or www. to identify external URLs, or download the file and
+ browse for its path on your machine using the Browse button.
- All users can access the extensive sources of help, including
- tutorials, available at www.python.org/doc. Selected URLs can be added
- or removed from the Help menu at any time using Configure IDLE.
+ All users can access the extensive sources of help, including
+ tutorials, available at docs.python.org. Selected URLs can be added
+ or removed from the Help menu at any time using Configure IDLE.
Basic editing and navigation:
- Backspace deletes char to the left; DEL deletes char to the right.
- Control-backspace deletes word left, Control-DEL deletes word right.
- Arrow keys and Page Up/Down move around.
- Control-left/right Arrow moves by words in a strange but useful way.
- Home/End go to begin/end of line.
- Control-Home/End go to begin/end of file.
- Some useful Emacs bindings are inherited from Tcl/Tk:
- Control-a beginning of line
- Control-e end of line
- Control-k kill line (but doesn't put it in clipboard)
- Control-l center window around the insertion point
- Standard Windows bindings may work on that platform.
- Keybindings are selected in the Settings Dialog, look there.
+ Backspace deletes char to the left; DEL deletes char to the right.
+ Control-backspace deletes word left, Control-DEL deletes word right.
+ Arrow keys and Page Up/Down move around.
+ Control-left/right Arrow moves by words in a strange but useful way.
+ Home/End go to begin/end of line.
+ Control-Home/End go to begin/end of file.
+ Some useful Emacs bindings are inherited from Tcl/Tk:
+ Control-a beginning of line
+ Control-e end of line
+ Control-k kill line (but doesn't put it in clipboard)
+ Control-l center window around the insertion point
+ Standard keybindings (like Control-c to copy and Control-v to
+ paste) may work. Keybindings are selected in the Configure IDLE
+ dialog.
Automatic indentation:
- After a block-opening statement, the next line is indented by 4 spaces
- (in the Python Shell window by one tab). After certain keywords
- (break, return etc.) the next line is dedented. In leading
- indentation, Backspace deletes up to 4 spaces if they are there. Tab
- inserts spaces (in the Python Shell window one tab), number depends on
- Indent Width. (N.B. Currently tabs are restricted to four spaces due
- to Tcl/Tk issues.)
+ After a block-opening statement, the next line is indented by 4 spaces
+ (in the Python Shell window by one tab). After certain keywords
+ (break, return etc.) the next line is dedented. In leading
+ indentation, Backspace deletes up to 4 spaces if they are there. Tab
+ inserts spaces (in the Python Shell window one tab), number depends on
+ Indent Width. Currently tabs are restricted to four spaces due
+ to Tcl/Tk limitations.
See also the indent/dedent region commands in the edit menu.
Completions:
- Completions are supplied for functions, classes, and attributes of
- classes, both built-in and user-defined. Completions are also provided
- for filenames.
-
- The AutoCompleteWindow (ACW) will open after a predefined delay
- (default is two seconds) after a '.' or (in a string) an os.sep is
- typed. If after one of those characters (plus zero or more other
- characters) you type a Tab the ACW will open immediately if a possible
- continuation is found.
-
- If there is only one possible completion for the characters entered, a
- Tab will supply that completion without opening the ACW.
-
- 'Show Completions' will force open a completions window. In an empty
- string, this will contain the files in the current directory. On a
- blank line, it will contain the built-in and user-defined functions and
- classes in the current name spaces, plus any modules imported. If some
- characters have been entered, the ACW will attempt to be more specific.
-
- If string of characters is typed, the ACW selection will jump to the
- entry most closely matching those characters. Entering a Tab will cause
- the longest non-ambiguous match to be entered in the Edit window or
- Shell. Two Tabs in a row will supply the current ACW selection, as
- will Return or a double click. Cursor keys, Page Up/Down, mouse
- selection, and the scrollwheel all operate on the ACW.
-
- 'Hidden' attributes can be accessed by typing the beginning of hidden
- name after a '.'. e.g. '_'. This allows access to modules with
- '__all__' set, or to class-private attributes.
-
- Completions and the 'Expand Word' facility can save a lot of typing!
-
- Completions are currently limited to those in the namespaces. Names in
- an Edit window which are not via __main__ or sys.modules will not be
- found. Run the module once with your imports to correct this
- situation. Note that IDLE itself places quite a few modules in
- sys.modules, so much can be found by default, e.g. the re module.
-
- If you don't like the ACW popping up unbidden, simply make the delay
- longer or disable the extension. OTOH, you could make the delay zero.
-
- You could also switch off the CallTips extension. (We will be adding
- a delay to the call tip window.)
+ Completions are supplied for functions, classes, and attributes of
+ classes, both built-in and user-defined. Completions are also provided
+ for filenames.
+
+ The AutoCompleteWindow (ACW) will open after a predefined delay
+ (default is two seconds) after a '.' or (in a string) an os.sep is
+ typed. If after one of those characters (plus zero or more other
+ characters) a tab is typed the ACW will open immediately if a possible
+ continuation is found.
+
+ If there is only one possible completion for the characters entered, a
+ tab will supply that completion without opening the ACW.
+
+ 'Show Completions' will force open a completions window, by default the
+ Control-space keys will open a completions window. In an empty
+ string, this will contain the files in the current directory. On a
+ blank line, it will contain the built-in and user-defined functions and
+ classes in the current name spaces, plus any modules imported. If some
+ characters have been entered, the ACW will attempt to be more specific.
+
+ If string of characters is typed, the ACW selection will jump to the
+ entry most closely matching those characters. Entering a tab will cause
+ the longest non-ambiguous match to be entered in the Edit window or
+ Shell. Two tabs in a row will supply the current ACW selection, as
+ will return or a double click. Cursor keys, Page Up/Down, mouse
+ selection, and the scroll wheel all operate on the ACW.
+
+ "Hidden" attributes can be accessed by typing the beginning of hidden
+ name after a '.', e.g. '_'. This allows access to modules with
+ '__all__' set, or to class-private attributes.
+
+ Completions and the 'Expand Word' facility can save a lot of typing!
+
+ Completions are currently limited to those in the namespaces. Names in
+ an Editor window which are not via __main__ or sys.modules will not be
+ found. Run the module once with your imports to correct this
+ situation. Note that IDLE itself places quite a few modules in
+ sys.modules, so much can be found by default, e.g. the re module.
+
+ If you don't like the ACW popping up unbidden, simply make the delay
+ longer or disable the extension. Or another option is the delay could
+ be set to zero. Another alternative to preventing ACW popups is to
+ disable the call tips extension.
Python Shell window:
- Control-c interrupts executing command.
- Control-d sends end-of-file; closes window if typed at >>> prompt.
+ Control-c interrupts executing command.
+ Control-d sends end-of-file; closes window if typed at >>> prompt.
+ Alt-/ expand word is also useful to reduce typing.
Command history:
- Alt-p retrieves previous command matching what you have typed.
- Alt-n retrieves next.
- (These are Control-p, Control-n on OS X)
- Return while cursor is on a previous command retrieves that command.
- Expand word is also useful to reduce typing.
+ Alt-p retrieves previous command matching what you have typed. On OS X
+ use Control-p.
+ Alt-n retrieves next. On OS X use Control-n.
+ Return while cursor is on a previous command retrieves that command.
Syntax colors:
- The coloring is applied in a background "thread", so you may
- occasionally see uncolorized text. To change the color
- scheme, use the Configure IDLE / Highlighting dialog.
+ The coloring is applied in a background "thread", so you may
+ occasionally see uncolorized text. To change the color
+ scheme, use the Configure IDLE / Highlighting dialog.
Python default syntax colors:
- Keywords orange
- Builtins royal purple
- Strings green
- Comments red
- Definitions blue
+ Keywords orange
+ Builtins royal purple
+ Strings green
+ Comments red
+ Definitions blue
Shell default colors:
- Console output brown
- stdout blue
- stderr red
- stdin black
+ Console output brown
+ stdout blue
+ stderr red
+ stdin black
Other preferences:
- The font preferences, keybinding, and startup preferences can
- be changed using the Settings dialog.
+ The font preferences, highlighting, keys, and general preferences can
+ be changed via the Configure IDLE menu option. Be sure to note that
+ keys can be user defined, IDLE ships with four built in key sets. In
+ addition a user can create a custom key set in the Configure IDLE
+ dialog under the keys tab.
Command line usage:
- Enter idle -h at the command prompt to get a usage message.
-
-Running without a subprocess:
-
- If IDLE is started with the -n command line switch it will run in a
- single process and will not create the subprocess which runs the RPC
- Python execution server. This can be useful if Python cannot create
- the subprocess or the RPC socket interface on your platform. However,
- in this mode user code is not isolated from IDLE itself. Also, the
- environment is not restarted when Run/Run Module (F5) is selected. If
- your code has been modified, you must reload() the affected modules and
- re-import any specific items (e.g. from foo import baz) if the changes
- are to take effect. For these reasons, it is preferable to run IDLE
- with the default subprocess if at all possible.
+ Enter idle -h at the command prompt to get a usage message.
+
+ idle.py [-c command] [-d] [-e] [-s] [-t title] [arg] ...
+
+ -c command run this command
+ -d enable debugger
+ -e edit mode; arguments are files to be edited
+ -s run $IDLESTARTUP or $PYTHONSTARTUP first
+ -t title set title of shell window
+
+ If there are arguments:
+ 1. If -e is used, arguments are files opened for editing and sys.argv
+ reflects the arguments passed to IDLE itself.
+ 2. Otherwise, if -c is used, all arguments are placed in
+ sys.argv[1:...], with sys.argv[0] set to -c.
+ 3. Otherwise, if neither -e nor -c is used, the first argument is a
+ script which is executed with the remaining arguments in
+ sys.argv[1:...] and sys.argv[0] set to the script name. If the
+ script name is -, no script is executed but an interactive Python
+ session is started; the arguments are still available in sys.argv.
+
+Running without a subprocess: (DEPRECATED in Python 3.4 see Issue 16123)
+
+ If IDLE is started with the -n command line switch it will run in a
+ single process and will not create the subprocess which runs the RPC
+ Python execution server. This can be useful if Python cannot create
+ the subprocess or the RPC socket interface on your platform. However,
+ in this mode user code is not isolated from IDLE itself. Also, the
+ environment is not restarted when Run/Run Module (F5) is selected. If
+ your code has been modified, you must reload() the affected modules and
+ re-import any specific items (e.g. from foo import baz) if the changes
+ are to take effect. For these reasons, it is preferable to run IDLE
+ with the default subprocess if at all possible.
Extensions:
- IDLE contains an extension facility. See the beginning of
- config-extensions.def in the idlelib directory for further information.
- The default extensions are currently:
-
- FormatParagraph
- AutoExpand
- ZoomHeight
- ScriptBinding
- CallTips
- ParenMatch
- AutoComplete
- CodeContext
+ IDLE contains an extension facility. See the beginning of
+ config-extensions.def in the idlelib directory for further information.
+ The default extensions are currently:
+
+ FormatParagraph
+ AutoExpand
+ ZoomHeight
+ ScriptBinding
+ CallTips
+ ParenMatch
+ AutoComplete
+ CodeContext
diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py
index f363764..4ee15ae 100644
--- a/Lib/idlelib/idle_test/test_calltips.py
+++ b/Lib/idlelib/idle_test/test_calltips.py
@@ -54,9 +54,9 @@ class Get_signatureTest(unittest.TestCase):
gtest(List, List.__doc__)
gtest(list.__new__,
- 'T.__new__(S, ...) -> a new object with type S, a subtype of T')
+ 'Create and return a new object. See help(type) for accurate signature.')
gtest(list.__init__,
- 'x.__init__(...) initializes x; see help(type(x)) for signature')
+ 'Initialize self. See help(type(self)) for accurate signature.')
append_doc = "L.append(object) -> None -- append object to end"
gtest(list.append, append_doc)
gtest([].append, append_doc)
@@ -69,7 +69,8 @@ class Get_signatureTest(unittest.TestCase):
self.assertEqual(signature(textwrap.TextWrapper), '''\
(width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
- drop_whitespace=True, break_on_hyphens=True, tabsize=8)''')
+ drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None,
+ placeholder=' [...]')''')
def test_docline_truncation(self):
def f(): pass
diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py
index 044bef9..efe37a1 100644
--- a/Lib/idlelib/idlever.py
+++ b/Lib/idlelib/idlever.py
@@ -1 +1 @@
-IDLE_VERSION = "3.3.5"
+IDLE_VERSION = "3.4.0"
diff --git a/Lib/idlelib/keybindingDialog.py b/Lib/idlelib/keybindingDialog.py
index 0f0da8c..db88cb4 100644
--- a/Lib/idlelib/keybindingDialog.py
+++ b/Lib/idlelib/keybindingDialog.py
@@ -4,7 +4,7 @@ Dialog for building Tkinter accelerator key bindings
from tkinter import *
import tkinter.messagebox as tkMessageBox
import string
-from idlelib import macosxSupport
+import sys
class GetKeysDialog(Toplevel):
def __init__(self,parent,title,action,currentKeySequences):
@@ -133,8 +133,7 @@ class GetKeysDialog(Toplevel):
order is also important: key binding equality depends on it, so
config-keys.def must use the same ordering.
"""
- import sys
- if macosxSupport.runningAsOSXApp():
+ if sys.platform == "darwin":
self.modifiers = ['Shift', 'Control', 'Option', 'Command']
else:
self.modifiers = ['Control', 'Alt', 'Shift']
diff --git a/Lib/idlelib/macosxSupport.py b/Lib/idlelib/macosxSupport.py
index 67069fa..b6488f8 100644
--- a/Lib/idlelib/macosxSupport.py
+++ b/Lib/idlelib/macosxSupport.py
@@ -1,48 +1,70 @@
"""
-A number of function that enhance IDLE on MacOSX when it used as a normal
-GUI application (as opposed to an X11 application).
+A number of functions that enhance IDLE on Mac OSX.
"""
import sys
import tkinter
from os import path
+import warnings
+def runningAsOSXApp():
+ warnings.warn("runningAsOSXApp() is deprecated, use isAquaTk()",
+ DeprecationWarning, stacklevel=2)
+ return isAquaTk()
-_appbundle = None
+def isCarbonAquaTk(root):
+ warnings.warn("isCarbonAquaTk(root) is deprecated, use isCarbonTk()",
+ DeprecationWarning, stacklevel=2)
+ return isCarbonTk()
-def runningAsOSXApp():
+_tk_type = None
+
+def _initializeTkVariantTests(root):
+ """
+ Initializes OS X Tk variant values for
+ isAquaTk(), isCarbonTk(), isCocoaTk(), and isXQuartz().
"""
- Returns True if Python is running from within an app on OSX.
- If so, the various OS X customizations will be triggered later (menu
- fixup, et al). (Originally, this test was supposed to condition
- behavior on whether IDLE was running under Aqua Tk rather than
- under X11 Tk but that does not work since a framework build
- could be linked with X11. For several releases, this test actually
- differentiates between whether IDLE is running from a framework or
- not. As a future enhancement, it should be considered whether there
- should be a difference based on framework and any needed X11 adaptions
- should be made dependent on a new function that actually tests for X11.)
- """
- global _appbundle
- if _appbundle is None:
- _appbundle = sys.platform == 'darwin'
- if _appbundle:
- import sysconfig
- _appbundle = bool(sysconfig.get_config_var('PYTHONFRAMEWORK'))
- return _appbundle
-
-_carbonaquatk = None
+ global _tk_type
+ if sys.platform == 'darwin':
+ ws = root.tk.call('tk', 'windowingsystem')
+ if 'x11' in ws:
+ _tk_type = "xquartz"
+ elif 'aqua' not in ws:
+ _tk_type = "other"
+ elif 'AppKit' in root.tk.call('winfo', 'server', '.'):
+ _tk_type = "cocoa"
+ else:
+ _tk_type = "carbon"
+ else:
+ _tk_type = "other"
-def isCarbonAquaTk(root):
+def isAquaTk():
+ """
+ Returns True if IDLE is using a native OS X Tk (Cocoa or Carbon).
+ """
+ assert _tk_type is not None
+ return _tk_type == "cocoa" or _tk_type == "carbon"
+
+def isCarbonTk():
"""
Returns True if IDLE is using a Carbon Aqua Tk (instead of the
newer Cocoa Aqua Tk).
"""
- global _carbonaquatk
- if _carbonaquatk is None:
- _carbonaquatk = (runningAsOSXApp() and
- 'aqua' in root.tk.call('tk', 'windowingsystem') and
- 'AppKit' not in root.tk.call('winfo', 'server', '.'))
- return _carbonaquatk
+ assert _tk_type is not None
+ return _tk_type == "carbon"
+
+def isCocoaTk():
+ """
+ Returns True if IDLE is using a Cocoa Aqua Tk.
+ """
+ assert _tk_type is not None
+ return _tk_type == "cocoa"
+
+def isXQuartz():
+ """
+ Returns True if IDLE is using an OS X X11 Tk.
+ """
+ assert _tk_type is not None
+ return _tk_type == "xquartz"
def tkVersionWarning(root):
"""
@@ -53,8 +75,7 @@ def tkVersionWarning(root):
can still crash unexpectedly.
"""
- if (runningAsOSXApp() and
- ('AppKit' in root.tk.call('winfo', 'server', '.')) ):
+ if isCocoaTk():
patchlevel = root.tk.call('info', 'patchlevel')
if patchlevel not in ('8.5.7', '8.5.9'):
return False
@@ -88,8 +109,8 @@ def hideTkConsole(root):
def overrideRootMenu(root, flist):
"""
- Replace the Tk root menu by something that's more appropriate for
- IDLE.
+ Replace the Tk root menu by something that is more appropriate for
+ IDLE with an Aqua Tk.
"""
# The menu that is attached to the Tk root (".") is also used by AquaTk for
# all windows that don't specify a menu of their own. The default menubar
@@ -108,6 +129,22 @@ def overrideRootMenu(root, flist):
from idlelib import WindowList
from idlelib.MultiCall import MultiCallCreator
+ closeItem = Bindings.menudefs[0][1][-2]
+
+ # Remove the last 3 items of the file menu: a separator, close window and
+ # quit. Close window will be reinserted just above the save item, where
+ # it should be according to the HIG. Quit is in the application menu.
+ del Bindings.menudefs[0][1][-3:]
+ Bindings.menudefs[0][1].insert(6, closeItem)
+
+ # Remove the 'About' entry from the help menu, it is in the application
+ # menu
+ del Bindings.menudefs[-1][1][0:2]
+
+ # Remove the 'Configure' entry from the options menu, it is in the
+ # application menu as 'Preferences'
+ del Bindings.menudefs[-2][1][0:2]
+
menubar = Menu(root)
root.configure(menu=menubar)
menudict = {}
@@ -156,7 +193,7 @@ def overrideRootMenu(root, flist):
# right thing for now.
root.createcommand('exit', flist.close_all_callback)
- if isCarbonAquaTk(root):
+ if isCarbonTk():
# for Carbon AquaTk, replace the default Tk apple menu
menudict['application'] = menu = Menu(menubar, name='apple')
menubar.add_cascade(label='IDLE', menu=menu)
@@ -171,8 +208,7 @@ def overrideRootMenu(root, flist):
Bindings.menudefs[0][1].append(
('_Preferences....', '<<open-config-dialog>>'),
)
- else:
- # assume Cocoa AquaTk
+ if isCocoaTk():
# replace default About dialog with About IDLE one
root.createcommand('tkAboutDialog', about_dialog)
# replace default "Help" item in Help menu
@@ -182,10 +218,22 @@ def overrideRootMenu(root, flist):
def setupApp(root, flist):
"""
- Perform setup for the OSX application bundle.
+ Perform initial OS X customizations if needed.
+ Called from PyShell.main() after initial calls to Tk()
+
+ There are currently three major versions of Tk in use on OS X:
+ 1. Aqua Cocoa Tk (native default since OS X 10.6)
+ 2. Aqua Carbon Tk (original native, 32-bit only, deprecated)
+ 3. X11 (supported by some third-party distributors, deprecated)
+ There are various differences among the three that affect IDLE
+ behavior, primarily with menus, mouse key events, and accelerators.
+ Some one-time customizations are performed here.
+ Others are dynamically tested throughout idlelib by calls to the
+ isAquaTk(), isCarbonTk(), isCocoaTk(), isXQuartz() functions which
+ are initialized here as well.
"""
- if not runningAsOSXApp(): return
-
- hideTkConsole(root)
- overrideRootMenu(root, flist)
- addOpenEventSupport(root, flist)
+ _initializeTkVariantTests(root)
+ if isAquaTk():
+ hideTkConsole(root)
+ overrideRootMenu(root, flist)
+ addOpenEventSupport(root, flist)
diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py
index ddce6e9..9c51b8f 100644
--- a/Lib/idlelib/rpc.py
+++ b/Lib/idlelib/rpc.py
@@ -199,7 +199,7 @@ class SocketIO(object):
raise
except KeyboardInterrupt:
raise
- except socket.error:
+ except OSError:
raise
except Exception as ex:
return ("CALLEXC", ex)
@@ -340,7 +340,7 @@ class SocketIO(object):
n = self.sock.send(s[:BUFSIZE])
except (AttributeError, TypeError):
raise OSError("socket no longer exists")
- except socket.error:
+ except OSError:
raise
else:
s = s[n:]
@@ -357,7 +357,7 @@ class SocketIO(object):
return None
try:
s = self.sock.recv(BUFSIZE)
- except socket.error:
+ except OSError:
raise EOFError
if len(s) == 0:
raise EOFError
@@ -537,7 +537,7 @@ class RPCClient(SocketIO):
SocketIO.__init__(self, working_sock)
else:
print("** Invalid host: ", address, file=sys.__stderr__)
- raise socket.error
+ raise OSError
def get_remote_proxy(self, oid):
return RPCProxy(self, oid)
diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py
index c1859b6..13cec62 100644
--- a/Lib/idlelib/run.py
+++ b/Lib/idlelib/run.py
@@ -150,8 +150,8 @@ def manage_socket(address):
try:
server = MyRPCServer(address, MyHandler)
break
- except socket.error as err:
- print("IDLE Subprocess: socket error: " + err.args[1] +
+ except OSError as err:
+ print("IDLE Subprocess: OSError: " + err.args[1] +
", retrying....", file=sys.__stderr__)
socket_error = err
else:
diff --git a/Lib/imaplib.py b/Lib/imaplib.py
index ad5f4e9..ad104fe 100644
--- a/Lib/imaplib.py
+++ b/Lib/imaplib.py
@@ -185,7 +185,7 @@ class IMAP4:
except Exception:
try:
self.shutdown()
- except socket.error:
+ except OSError:
pass
raise
@@ -281,7 +281,7 @@ class IMAP4:
self.file.close()
try:
self.sock.shutdown(socket.SHUT_RDWR)
- except socket.error as e:
+ except OSError as e:
# The server might already have closed the connection
if e.errno != errno.ENOTCONN:
raise
@@ -554,7 +554,7 @@ class IMAP4:
import hmac
pwd = (self.password.encode('ASCII') if isinstance(self.password, str)
else self.password)
- return self.user + " " + hmac.HMAC(pwd, challenge).hexdigest()
+ return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest()
def logout(self):
@@ -742,12 +742,12 @@ class IMAP4:
raise self.abort('TLS not supported by server')
# Generate a default SSL context if none was passed.
if ssl_context is None:
- ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
- # SSLv2 considered harmful.
- ssl_context.options |= ssl.OP_NO_SSLv2
+ ssl_context = ssl._create_stdlib_context()
typ, dat = self._simple_command(name)
if typ == 'OK':
- self.sock = ssl_context.wrap_socket(self.sock)
+ server_hostname = self.host if ssl.HAS_SNI else None
+ self.sock = ssl_context.wrap_socket(self.sock,
+ server_hostname=server_hostname)
self.file = self.sock.makefile('rb')
self._tls_established = True
self._get_capabilities()
@@ -915,7 +915,7 @@ class IMAP4:
try:
self.send(data + CRLF)
- except (socket.error, OSError) as val:
+ except OSError as val:
raise self.abort('socket error: %s' % val)
if literal is None:
@@ -940,7 +940,7 @@ class IMAP4:
try:
self.send(literal)
self.send(CRLF)
- except (socket.error, OSError) as val:
+ except OSError as val:
raise self.abort('socket error: %s' % val)
if not literator:
@@ -1090,7 +1090,7 @@ class IMAP4:
# Protocol mandates all lines terminated by CRLF
if not line.endswith(b'\r\n'):
- raise self.abort('socket error: unterminated line')
+ raise self.abort('socket error: unterminated line: %r' % line)
line = line[:-2]
if __debug__:
@@ -1215,15 +1215,17 @@ if HAVE_SSL:
self.keyfile = keyfile
self.certfile = certfile
+ if ssl_context is None:
+ ssl_context = ssl._create_stdlib_context(certfile=certfile,
+ keyfile=keyfile)
self.ssl_context = ssl_context
IMAP4.__init__(self, host, port)
def _create_socket(self):
sock = IMAP4._create_socket(self)
- if self.ssl_context:
- return self.ssl_context.wrap_socket(sock)
- else:
- return ssl.wrap_socket(sock, self.keyfile, self.certfile)
+ server_hostname = self.host if ssl.HAS_SNI else None
+ return self.ssl_context.wrap_socket(sock,
+ server_hostname=server_hostname)
def open(self, host='', port=IMAP4_SSL_PORT):
"""Setup connection to remote server on "host:port".
diff --git a/Lib/imghdr.py b/Lib/imghdr.py
index bdd47ee..add2ea8 100644
--- a/Lib/imghdr.py
+++ b/Lib/imghdr.py
@@ -147,7 +147,7 @@ def testall(list, recursive, toplevel):
sys.stdout.flush()
try:
print(what(filename))
- except IOError:
+ except OSError:
print('*** not found ***')
if __name__ == '__main__':
diff --git a/Lib/imp.py b/Lib/imp.py
index 4088383..c8449c6 100644
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -16,18 +16,20 @@ except ImportError:
# Platform doesn't support dynamic loading.
load_dynamic = None
-# Directly exposed by this module
-from importlib._bootstrap import new_module
-from importlib._bootstrap import cache_from_source, source_from_cache
+from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _SpecMethods
-
-from importlib import _bootstrap
from importlib import machinery
+from importlib import util
+import importlib
import os
import sys
import tokenize
+import types
import warnings
+warnings.warn("the imp module is deprecated in favour of importlib; "
+ "see the module's documentation for alternative uses",
+ PendingDeprecationWarning)
# DEPRECATED
SEARCH_ERROR = 0
@@ -42,9 +44,23 @@ PY_CODERESOURCE = 8
IMP_HOOK = 9
+def new_module(name):
+ """**DEPRECATED**
+
+ Create a new module.
+
+ The module is not entered into sys.modules.
+
+ """
+ return types.ModuleType(name)
+
+
def get_magic():
- """Return the magic number for .pyc or .pyo files."""
- return _bootstrap._MAGIC_BYTES
+ """**DEPRECATED**
+
+ Return the magic number for .pyc or .pyo files.
+ """
+ return util.MAGIC_NUMBER
def get_tag():
@@ -52,12 +68,42 @@ def get_tag():
return sys.implementation.cache_tag
+def cache_from_source(path, debug_override=None):
+ """**DEPRECATED**
+
+ Given the path to a .py file, return the path to its .pyc/.pyo file.
+
+ The .py file does not need to exist; this simply returns the path to the
+ .pyc/.pyo file calculated as if the .py file were imported. The extension
+ will be .pyc unless sys.flags.optimize is non-zero, then it will be .pyo.
+
+ If debug_override is not None, then it must be a boolean and is used in
+ place of sys.flags.optimize.
+
+ If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ return util.cache_from_source(path, debug_override)
+
+
+def source_from_cache(path):
+ """**DEPRECATED**
+
+ Given the path to a .pyc./.pyo file, return the path to its .py file.
+
+ The .pyc/.pyo file does not need to exist; this simply returns the path to
+ the .py file calculated to correspond to the .pyc/.pyo file. If path does
+ not conform to PEP 3147 format, ValueError will be raised. If
+ sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ return util.source_from_cache(path)
+
+
def get_suffixes():
- warnings.warn('imp.get_suffixes() is deprecated; use the constants '
- 'defined on importlib.machinery instead',
- DeprecationWarning, 2)
+ """**DEPRECATED**"""
extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
- source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
+ source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
return extensions + source + bytecode
@@ -65,7 +111,11 @@ def get_suffixes():
class NullImporter:
- """Null import object."""
+ """**DEPRECATED**
+
+ Null import object.
+
+ """
def __init__(self, path):
if path == '':
@@ -106,48 +156,49 @@ class _HackedGetData:
return super().get_data(path)
-class _LoadSourceCompatibility(_HackedGetData, _bootstrap.SourceFileLoader):
+class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
"""Compatibility support for implementing load_source()."""
def load_source(name, pathname, file=None):
- msg = ('imp.load_source() is deprecated; use '
- 'importlib.machinery.SourceFileLoader(name, pathname).load_module()'
- ' instead')
- warnings.warn(msg, DeprecationWarning, 2)
- _LoadSourceCompatibility(name, pathname, file).load_module(name)
- module = sys.modules[name]
+ loader = _LoadSourceCompatibility(name, pathname, file)
+ spec = util.spec_from_file_location(name, pathname, loader=loader)
+ methods = _SpecMethods(spec)
+ if name in sys.modules:
+ module = methods.exec(sys.modules[name])
+ else:
+ module = methods.load()
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
- module.__loader__ = _bootstrap.SourceFileLoader(name, pathname)
+ module.__loader__ = machinery.SourceFileLoader(name, pathname)
+ module.__spec__.loader = module.__loader__
return module
-class _LoadCompiledCompatibility(_HackedGetData,
- _bootstrap.SourcelessFileLoader):
+class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
"""Compatibility support for implementing load_compiled()."""
def load_compiled(name, pathname, file=None):
- msg = ('imp.load_compiled() is deprecated; use '
- 'importlib.machinery.SourcelessFileLoader(name, pathname).'
- 'load_module() instead ')
- warnings.warn(msg, DeprecationWarning, 2)
- _LoadCompiledCompatibility(name, pathname, file).load_module(name)
- module = sys.modules[name]
+ """**DEPRECATED**"""
+ loader = _LoadCompiledCompatibility(name, pathname, file)
+ spec = util.spec_from_file_location(name, pathname, loader=loader)
+ methods = _SpecMethods(spec)
+ if name in sys.modules:
+ module = methods.exec(sys.modules[name])
+ else:
+ module = methods.load()
# To allow reloading to potentially work, use a non-hacked loader which
# won't rely on a now-closed file object.
- module.__loader__ = _bootstrap.SourcelessFileLoader(name, pathname)
+ module.__loader__ = SourcelessFileLoader(name, pathname)
+ module.__spec__.loader = module.__loader__
return module
def load_package(name, path):
- msg = ('imp.load_package() is deprecated; use either '
- 'importlib.machinery.SourceFileLoader() or '
- 'importlib.machinery.SourcelessFileLoader() instead')
- warnings.warn(msg, DeprecationWarning, 2)
+ """**DEPRECATED**"""
if os.path.isdir(path):
extensions = (machinery.SOURCE_SUFFIXES[:] +
machinery.BYTECODE_SUFFIXES[:])
@@ -157,7 +208,13 @@ def load_package(name, path):
break
else:
raise ValueError('{!r} is not a package'.format(path))
- return _bootstrap.SourceFileLoader(name, path).load_module(name)
+ spec = util.spec_from_file_location(name, path,
+ submodule_search_locations=[])
+ methods = _SpecMethods(spec)
+ if name in sys.modules:
+ return methods.exec(sys.modules[name])
+ else:
+ return methods.load()
def load_module(name, file, filename, details):
@@ -169,32 +226,30 @@ def load_module(name, file, filename, details):
"""
suffix, mode, type_ = details
- with warnings.catch_warnings():
- warnings.simplefilter('ignore')
- if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
- raise ValueError('invalid file open mode {!r}'.format(mode))
- elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
- msg = 'file object required for import (type code {})'.format(type_)
- raise ValueError(msg)
- elif type_ == PY_SOURCE:
- return load_source(name, filename, file)
- elif type_ == PY_COMPILED:
- return load_compiled(name, filename, file)
- elif type_ == C_EXTENSION and load_dynamic is not None:
- if file is None:
- with open(filename, 'rb') as opened_file:
- return load_dynamic(name, filename, opened_file)
- else:
- return load_dynamic(name, filename, file)
- elif type_ == PKG_DIRECTORY:
- return load_package(name, filename)
- elif type_ == C_BUILTIN:
- return init_builtin(name)
- elif type_ == PY_FROZEN:
- return init_frozen(name)
+ if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+ raise ValueError('invalid file open mode {!r}'.format(mode))
+ elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
+ msg = 'file object required for import (type code {})'.format(type_)
+ raise ValueError(msg)
+ elif type_ == PY_SOURCE:
+ return load_source(name, filename, file)
+ elif type_ == PY_COMPILED:
+ return load_compiled(name, filename, file)
+ elif type_ == C_EXTENSION and load_dynamic is not None:
+ if file is None:
+ with open(filename, 'rb') as opened_file:
+ return load_dynamic(name, filename, opened_file)
else:
- msg = "Don't know how to import {} (type code {})".format(name, type_)
- raise ImportError(msg, name=name)
+ return load_dynamic(name, filename, file)
+ elif type_ == PKG_DIRECTORY:
+ return load_package(name, filename)
+ elif type_ == C_BUILTIN:
+ return init_builtin(name)
+ elif type_ == PY_FROZEN:
+ return init_frozen(name)
+ else:
+ msg = "Don't know how to import {} (type code {})".format(name, type_)
+ raise ImportError(msg, name=name)
def find_module(name, path=None):
@@ -230,54 +285,31 @@ def find_module(name, path=None):
file_path = os.path.join(package_directory, package_file_name)
if os.path.isfile(file_path):
return None, package_directory, ('', '', PKG_DIRECTORY)
- with warnings.catch_warnings():
- warnings.simplefilter('ignore')
- for suffix, mode, type_ in get_suffixes():
- file_name = name + suffix
- file_path = os.path.join(entry, file_name)
- if os.path.isfile(file_path):
- break
- else:
- continue
- break # Break out of outer loop when breaking out of inner loop.
+ for suffix, mode, type_ in get_suffixes():
+ file_name = name + suffix
+ file_path = os.path.join(entry, file_name)
+ if os.path.isfile(file_path):
+ break
+ else:
+ continue
+ break # Break out of outer loop when breaking out of inner loop.
else:
- raise ImportError(_bootstrap._ERR_MSG.format(name), name=name)
+ raise ImportError(_ERR_MSG.format(name), name=name)
encoding = None
- if mode == 'U':
+ if 'b' not in mode:
with open(file_path, 'rb') as file:
encoding = tokenize.detect_encoding(file.readline)[0]
file = open(file_path, mode, encoding=encoding)
return file, file_path, (suffix, mode, type_)
-_RELOADING = {}
-
def reload(module):
- """Reload the module and return it.
+ """**DEPRECATED**
+
+ Reload the module and return it.
The module must have been successfully imported before.
"""
- if not module or type(module) != type(sys):
- raise TypeError("reload() argument must be module")
- name = module.__name__
- if name not in sys.modules:
- msg = "module {} not in sys.modules"
- raise ImportError(msg.format(name), name=name)
- if name in _RELOADING:
- return _RELOADING[name]
- _RELOADING[name] = module
- try:
- parent_name = name.rpartition('.')[0]
- if parent_name and parent_name not in sys.modules:
- msg = "parent {!r} not in sys.modules"
- raise ImportError(msg.format(parent_name), name=parent_name)
- module.__loader__.load_module(name)
- # The module may have replaced itself in sys.modules!
- return sys.modules[module.__name__]
- finally:
- try:
- del _RELOADING[name]
- except KeyError:
- pass
+ return importlib.reload(module)
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 22c90f2..1bc9947 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -1,5 +1,5 @@
"""A pure Python implementation of import."""
-__all__ = ['__import__', 'import_module', 'invalidate_caches']
+__all__ = ['__import__', 'import_module', 'invalidate_caches', 'reload']
# Bootstrap help #####################################################
@@ -22,7 +22,12 @@ else:
# a second copy of the module.
_bootstrap.__name__ = 'importlib._bootstrap'
_bootstrap.__package__ = 'importlib'
- _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py')
+ try:
+ _bootstrap.__file__ = __file__.replace('__init__.py', '_bootstrap.py')
+ except NameError:
+ # __file__ is not guaranteed to be defined, e.g. if this code gets
+ # frozen by a tool like cx_Freeze.
+ pass
sys.modules['importlib._bootstrap'] = _bootstrap
# To simplify imports in test code
@@ -32,6 +37,10 @@ _r_long = _bootstrap._r_long
# Fully bootstrapped at this point, import whatever you like, circular
# dependencies and startup overhead minimisation permitting :)
+import types
+import warnings
+
+
# Public API #########################################################
from ._bootstrap import __import__
@@ -46,20 +55,15 @@ def invalidate_caches():
def find_loader(name, path=None):
- """Find the loader for the specified module.
+ """Return the loader for the specified module.
- First, sys.modules is checked to see if the module was already imported. If
- so, then sys.modules[name].__loader__ is returned. If that happens to be
- set to None, then ValueError is raised. If the module is not in
- sys.modules, then sys.meta_path is searched for a suitable loader with the
- value of 'path' given to the finders. None is returned if no loader could
- be found.
+ This is a backward-compatible wrapper around find_spec().
- Dotted names do not have their parent packages implicitly imported. You will
- most likely need to explicitly import all parent packages in the proper
- order for a submodule to get the correct loader.
+ This function is deprecated in favor of importlib.util.find_spec().
"""
+ warnings.warn('Use importlib.util.find_spec() instead.',
+ DeprecationWarning, stacklevel=2)
try:
loader = sys.modules[name].__loader__
if loader is None:
@@ -68,7 +72,20 @@ def find_loader(name, path=None):
return loader
except KeyError:
pass
- return _bootstrap._find_module(name, path)
+ except AttributeError:
+ raise ValueError('{}.__loader__ is not set'.format(name))
+
+ spec = _bootstrap._find_spec(name, path)
+ # We won't worry about malformed specs (missing attributes).
+ if spec is None:
+ return None
+ if spec.loader is None:
+ if spec.submodule_search_locations is None:
+ raise ImportError('spec for {} missing loader'.format(name),
+ name=name)
+ raise ImportError('namespace packages do not have loaders',
+ name=name)
+ return spec.loader
def import_module(name, package=None):
@@ -82,9 +99,58 @@ def import_module(name, package=None):
level = 0
if name.startswith('.'):
if not package:
- raise TypeError("relative imports require the 'package' argument")
+ msg = ("the 'package' argument is required to perform a relative "
+ "import for {!r}")
+ raise TypeError(msg.format(name))
for character in name:
if character != '.':
break
level += 1
return _bootstrap._gcd_import(name[level:], package, level)
+
+
+_RELOADING = {}
+
+
+def reload(module):
+ """Reload the module and return it.
+
+ The module must have been successfully imported before.
+
+ """
+ if not module or not isinstance(module, types.ModuleType):
+ raise TypeError("reload() argument must be module")
+ try:
+ name = module.__spec__.name
+ except AttributeError:
+ name = module.__name__
+
+ if sys.modules.get(name) is not module:
+ msg = "module {} not in sys.modules"
+ raise ImportError(msg.format(name), name=name)
+ if name in _RELOADING:
+ return _RELOADING[name]
+ _RELOADING[name] = module
+ try:
+ parent_name = name.rpartition('.')[0]
+ if parent_name:
+ try:
+ parent = sys.modules[parent_name]
+ except KeyError:
+ msg = "parent {!r} not in sys.modules"
+ raise ImportError(msg.format(parent_name), name=parent_name)
+ else:
+ pkgpath = parent.__path__
+ else:
+ pkgpath = None
+ target = module
+ spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target)
+ methods = _bootstrap._SpecMethods(spec)
+ methods.exec(module)
+ # The module may have replaced itself in sys.modules!
+ return sys.modules[name]
+ finally:
+ try:
+ del _RELOADING[name]
+ except KeyError:
+ pass
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index e40ec92..beaa9b3 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module.
#
# IMPORTANT: Whenever making changes to this module, be sure to run
# a top-level make in order to get the frozen version of the module
-# update. Not doing so, will result in the Makefile to fail for
+# update. Not doing so will result in the Makefile to fail for
# all others who don't have a ./python around to freeze the module
# in the early stages of compilation.
#
@@ -20,10 +20,6 @@ work. One should use importlib as the public-facing version of this module.
# reference any injected objects! This includes not only global code but also
# anything specified at the class level.
-# XXX Make sure all public names have no single leading underscore and all
-# others do.
-
-
# Bootstrap-related code ######################################################
_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
@@ -41,76 +37,58 @@ def _make_relax_case():
return _relax_case
-# TODO: Expose from marshal
def _w_long(x):
- """Convert a 32-bit integer to little-endian.
+ """Convert a 32-bit integer to little-endian."""
+ return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
- XXX Temporary until marshal's long functions are exposed.
-
- """
- x = int(x)
- int_bytes = []
- int_bytes.append(x & 0xFF)
- int_bytes.append((x >> 8) & 0xFF)
- int_bytes.append((x >> 16) & 0xFF)
- int_bytes.append((x >> 24) & 0xFF)
- return bytearray(int_bytes)
-
-# TODO: Expose from marshal
def _r_long(int_bytes):
- """Convert 4 bytes in little-endian to an integer.
-
- XXX Temporary until marshal's long function are exposed.
-
- """
- x = int_bytes[0]
- x |= int_bytes[1] << 8
- x |= int_bytes[2] << 16
- x |= int_bytes[3] << 24
- return x
+ """Convert 4 bytes in little-endian to an integer."""
+ return int.from_bytes(int_bytes, 'little')
def _path_join(*path_parts):
"""Replacement for os.path.join()."""
- new_parts = []
- for part in path_parts:
- if not part:
- continue
- new_parts.append(part)
- if part[-1] not in path_separators:
- new_parts.append(path_sep)
- return ''.join(new_parts[:-1]) # Drop superfluous path separator.
+ return path_sep.join([part.rstrip(path_separators)
+ for part in path_parts if part])
def _path_split(path):
"""Replacement for os.path.split()."""
+ if len(path_separators) == 1:
+ front, _, tail = path.rpartition(path_sep)
+ return front, tail
for x in reversed(path):
if x in path_separators:
- sep = x
- break
- else:
- sep = path_sep
- front, _, tail = path.rpartition(sep)
- return front, tail
+ front, tail = path.rsplit(x, maxsplit=1)
+ return front, tail
+ return '', path
+
+
+def _path_stat(path):
+ """Stat the path.
+
+ Made a separate function to make it easier to override in experiments
+ (e.g. cache stat results).
+
+ """
+ return _os.stat(path)
def _path_is_mode_type(path, mode):
"""Test whether the path is the specified mode type."""
try:
- stat_info = _os.stat(path)
+ stat_info = _path_stat(path)
except OSError:
return False
return (stat_info.st_mode & 0o170000) == mode
-# XXX Could also expose Modules/getpath.c:isfile()
def _path_isfile(path):
"""Replacement for os.path.isfile."""
return _path_is_mode_type(path, 0o100000)
-# XXX Could also expose Modules/getpath.c:isdir()
def _path_isdir(path):
"""Replacement for os.path.isdir."""
if not path:
@@ -148,17 +126,30 @@ def _wrap(new, old):
new.__dict__.update(old.__dict__)
+def _new_module(name):
+ return type(sys)(name)
+
+
_code_type = type(_wrap.__code__)
-def new_module(name):
- """Create a new module.
- The module is not entered into sys.modules.
+class _ManageReload:
- """
- return type(_io)(name)
+ """Manages the possible clean-up of sys.modules for load_module()."""
+
+ def __init__(self, name):
+ self._name = name
+ def __enter__(self):
+ self._is_reload = self._name in sys.modules
+
+ def __exit__(self, *args):
+ if any(arg is not None for arg in args) and not self._is_reload:
+ try:
+ del sys.modules[self._name]
+ except KeyError:
+ pass
# Module-level locking ########################################################
@@ -214,7 +205,7 @@ class _ModuleLock:
self.count += 1
return True
if self.has_deadlock():
- raise _DeadlockError("deadlock detected by %r" % self)
+ raise _DeadlockError('deadlock detected by %r' % self)
if self.wakeup.acquire(False):
self.waiters += 1
# Wait for a release() call
@@ -227,7 +218,7 @@ class _ModuleLock:
tid = _thread.get_ident()
with self.lock:
if self.owner != tid:
- raise RuntimeError("cannot release un-acquired lock")
+ raise RuntimeError('cannot release un-acquired lock')
assert self.count > 0
self.count -= 1
if self.count == 0:
@@ -237,7 +228,7 @@ class _ModuleLock:
self.wakeup.release()
def __repr__(self):
- return "_ModuleLock(%r) at %d" % (self.name, id(self))
+ return '_ModuleLock({!r}) at {}'.format(self.name, id(self))
class _DummyModuleLock:
@@ -254,11 +245,28 @@ class _DummyModuleLock:
def release(self):
if self.count == 0:
- raise RuntimeError("cannot release un-acquired lock")
+ raise RuntimeError('cannot release un-acquired lock')
self.count -= 1
def __repr__(self):
- return "_DummyModuleLock(%r) at %d" % (self.name, id(self))
+ return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self))
+
+
+class _ModuleLockManager:
+
+ def __init__(self, name):
+ self._name = name
+ self._lock = None
+
+ def __enter__(self):
+ try:
+ self._lock = _get_module_lock(self._name)
+ finally:
+ _imp.release_lock()
+ self._lock.acquire()
+
+ def __exit__(self, *args, **kwargs):
+ self._lock.release()
# The following two functions are for consumption by Python/import.c.
@@ -315,95 +323,109 @@ def _call_with_frames_removed(f, *args, **kwds):
# Finder/loader utility code ###############################################
-"""Magic word to reject .pyc files generated by other Python versions.
-It should change for each incompatible change to the bytecode.
-
-The value of CR and LF is incorporated so if you ever read or write
-a .pyc file in text mode the magic number will be wrong; also, the
-Apple MPW compiler swaps their values, botching string constants.
-
-The magic numbers must be spaced apart at least 2 values, as the
--U interpeter flag will cause MAGIC+1 being used. They have been
-odd numbers for some time now.
-
-There were a variety of old schemes for setting the magic number.
-The current working scheme is to increment the previous value by
-10.
-
-Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
-number also includes a new "magic tag", i.e. a human readable string used
-to represent the magic number in __pycache__ directories. When you change
-the magic number, you must also set a new unique magic tag. Generally this
-can be named after the Python major version of the magic number bump, but
-it can really be anything, as long as it's different than anything else
-that's come before. The tags are included in the following table, starting
-with Python 3.2a0.
-
-Known values:
- Python 1.5: 20121
- Python 1.5.1: 20121
- Python 1.5.2: 20121
- Python 1.6: 50428
- Python 2.0: 50823
- Python 2.0.1: 50823
- Python 2.1: 60202
- Python 2.1.1: 60202
- Python 2.1.2: 60202
- Python 2.2: 60717
- Python 2.3a0: 62011
- Python 2.3a0: 62021
- Python 2.3a0: 62011 (!)
- Python 2.4a0: 62041
- Python 2.4a3: 62051
- Python 2.4b1: 62061
- Python 2.5a0: 62071
- Python 2.5a0: 62081 (ast-branch)
- Python 2.5a0: 62091 (with)
- Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
- Python 2.5b3: 62101 (fix wrong code: for x, in ...)
- Python 2.5b3: 62111 (fix wrong code: x += yield)
- Python 2.5c1: 62121 (fix wrong lnotab with for loops and
- storing constants that should have been removed)
- Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
- Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
- Python 2.6a1: 62161 (WITH_CLEANUP optimization)
- Python 3000: 3000
- 3010 (removed UNARY_CONVERT)
- 3020 (added BUILD_SET)
- 3030 (added keyword-only parameters)
- 3040 (added signature annotations)
- 3050 (print becomes a function)
- 3060 (PEP 3115 metaclass syntax)
- 3061 (string literals become unicode)
- 3071 (PEP 3109 raise changes)
- 3081 (PEP 3137 make __file__ and __name__ unicode)
- 3091 (kill str8 interning)
- 3101 (merge from 2.6a0, see 62151)
- 3103 (__file__ points to source file)
- Python 3.0a4: 3111 (WITH_CLEANUP optimization).
- Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
- Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
- change LIST_APPEND and SET_ADD, add MAP_ADD)
- Python 3.1a0: 3151 (optimize conditional branches:
- introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
- Python 3.2a0: 3160 (add SETUP_WITH)
- tag: cpython-32
- Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
- tag: cpython-32
- Python 3.2a2 3180 (add DELETE_DEREF)
- Python 3.3a0 3190 __class__ super closure changed
- Python 3.3a0 3200 (__qualname__ added)
- 3210 (added size modulo 2**32 to the pyc header)
- Python 3.3a1 3220 (changed PEP 380 implementation)
- Python 3.3a4 3230 (revert changes to implicit __class__ closure)
-
-MAGIC must change whenever the bytecode emitted by the compiler may no
-longer be understood by older implementations of the eval loop (usually
-due to the addition of new opcodes).
+# Magic word to reject .pyc files generated by other Python versions.
+# It should change for each incompatible change to the bytecode.
+#
+# The value of CR and LF is incorporated so if you ever read or write
+# a .pyc file in text mode the magic number will be wrong; also, the
+# Apple MPW compiler swaps their values, botching string constants.
+#
+# The magic numbers must be spaced apart at least 2 values, as the
+# -U interpeter flag will cause MAGIC+1 being used. They have been
+# odd numbers for some time now.
+#
+# There were a variety of old schemes for setting the magic number.
+# The current working scheme is to increment the previous value by
+# 10.
+#
+# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
+# number also includes a new "magic tag", i.e. a human readable string used
+# to represent the magic number in __pycache__ directories. When you change
+# the magic number, you must also set a new unique magic tag. Generally this
+# can be named after the Python major version of the magic number bump, but
+# it can really be anything, as long as it's different than anything else
+# that's come before. The tags are included in the following table, starting
+# with Python 3.2a0.
+#
+# Known values:
+# Python 1.5: 20121
+# Python 1.5.1: 20121
+# Python 1.5.2: 20121
+# Python 1.6: 50428
+# Python 2.0: 50823
+# Python 2.0.1: 50823
+# Python 2.1: 60202
+# Python 2.1.1: 60202
+# Python 2.1.2: 60202
+# Python 2.2: 60717
+# Python 2.3a0: 62011
+# Python 2.3a0: 62021
+# Python 2.3a0: 62011 (!)
+# Python 2.4a0: 62041
+# Python 2.4a3: 62051
+# Python 2.4b1: 62061
+# Python 2.5a0: 62071
+# Python 2.5a0: 62081 (ast-branch)
+# Python 2.5a0: 62091 (with)
+# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
+# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
+# Python 2.5b3: 62111 (fix wrong code: x += yield)
+# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
+# storing constants that should have been removed)
+# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
+# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
+# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
+# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
+# Python 2.7a0: 62181 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 2.7a0 62191 (introduce SETUP_WITH)
+# Python 2.7a0 62201 (introduce BUILD_SET)
+# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
+# Python 3000: 3000
+# 3010 (removed UNARY_CONVERT)
+# 3020 (added BUILD_SET)
+# 3030 (added keyword-only parameters)
+# 3040 (added signature annotations)
+# 3050 (print becomes a function)
+# 3060 (PEP 3115 metaclass syntax)
+# 3061 (string literals become unicode)
+# 3071 (PEP 3109 raise changes)
+# 3081 (PEP 3137 make __file__ and __name__ unicode)
+# 3091 (kill str8 interning)
+# 3101 (merge from 2.6a0, see 62151)
+# 3103 (__file__ points to source file)
+# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
+# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
+# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
+# change LIST_APPEND and SET_ADD, add MAP_ADD)
+# Python 3.1a0: 3151 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 3.2a0: 3160 (add SETUP_WITH)
+# tag: cpython-32
+# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
+# tag: cpython-32
+# Python 3.2a2 3180 (add DELETE_DEREF)
+# Python 3.3a0 3190 __class__ super closure changed
+# Python 3.3a0 3200 (__qualname__ added)
+# 3210 (added size modulo 2**32 to the pyc header)
+# Python 3.3a1 3220 (changed PEP 380 implementation)
+# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
+# Python 3.4a1 3250 (evaluate positional default arguments before
+# keyword-only defaults)
+# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
+# free vars)
+# Python 3.4a1 3270 (various tweaks to the __class__ closure)
+# Python 3.4a1 3280 (remove implicit class argument)
+# Python 3.4a4 3290 (changes to __qualname__ computation)
+# Python 3.4a4 3300 (more changes to __qualname__ computation)
+# Python 3.4rc2 3310 (alter __qualname__ computation)
+#
+# MAGIC must change whenever the bytecode emitted by the compiler may no
+# longer be understood by older implementations of the eval loop (usually
+# due to the addition of new opcodes).
-"""
-_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24
-_MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8))
+MAGIC_NUMBER = (3310).to_bytes(2, 'little') + b'\r\n'
+_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
_PYCACHE = '__pycache__'
@@ -481,6 +503,18 @@ def _get_sourcefile(bytecode_path):
return source_path if _path_isfile(source_path) else bytecode_path
+def _calc_mode(path):
+ """Calculate the mode permissions for a bytecode file."""
+ try:
+ mode = _path_stat(path).st_mode
+ except OSError:
+ mode = 0o666
+ # We always ensure write access so we can update cached files
+ # later even when the source files are read-only on Windows (#6074)
+ mode |= 0o200
+ return mode
+
+
def _verbose_message(message, *args, verbosity=1):
"""Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
if sys.flags.verbose >= verbosity:
@@ -489,85 +523,6 @@ def _verbose_message(message, *args, verbosity=1):
print(message.format(*args), file=sys.stderr)
-def set_package(fxn):
- """Set __package__ on the returned module."""
- def set_package_wrapper(*args, **kwargs):
- module = fxn(*args, **kwargs)
- if getattr(module, '__package__', None) is None:
- module.__package__ = module.__name__
- if not hasattr(module, '__path__'):
- module.__package__ = module.__package__.rpartition('.')[0]
- return module
- _wrap(set_package_wrapper, fxn)
- return set_package_wrapper
-
-
-def set_loader(fxn):
- """Set __loader__ on the returned module."""
- def set_loader_wrapper(self, *args, **kwargs):
- module = fxn(self, *args, **kwargs)
- if not hasattr(module, '__loader__'):
- module.__loader__ = self
- return module
- _wrap(set_loader_wrapper, fxn)
- return set_loader_wrapper
-
-
-def module_for_loader(fxn):
- """Decorator to handle selecting the proper module for loaders.
-
- The decorated function is passed the module to use instead of the module
- name. The module passed in to the function is either from sys.modules if
- it already exists or is a new module. If the module is new, then __name__
- is set the first argument to the method, __loader__ is set to self, and
- __package__ is set accordingly (if self.is_package() is defined) will be set
- before it is passed to the decorated function (if self.is_package() does
- not work for the module it will be set post-load).
-
- If an exception is raised and the decorator created the module it is
- subsequently removed from sys.modules.
-
- The decorator assumes that the decorated function takes the module name as
- the second argument.
-
- """
- def module_for_loader_wrapper(self, fullname, *args, **kwargs):
- module = sys.modules.get(fullname)
- is_reload = module is not None
- if not is_reload:
- # This must be done before open() is called as the 'io' module
- # implicitly imports 'locale' and would otherwise trigger an
- # infinite loop.
- module = new_module(fullname)
- # This must be done before putting the module in sys.modules
- # (otherwise an optimization shortcut in import.c becomes wrong)
- module.__initializing__ = True
- sys.modules[fullname] = module
- module.__loader__ = self
- try:
- is_package = self.is_package(fullname)
- except (ImportError, AttributeError):
- pass
- else:
- if is_package:
- module.__package__ = fullname
- else:
- module.__package__ = fullname.rpartition('.')[0]
- else:
- module.__initializing__ = True
- try:
- # If __package__ was not set above, __import__() will do it later.
- return fxn(self, module, *args, **kwargs)
- except:
- if not is_reload:
- del sys.modules[fullname]
- raise
- finally:
- module.__initializing__ = False
- _wrap(module_for_loader_wrapper, fxn)
- return module_for_loader_wrapper
-
-
def _check_name(method):
"""Decorator to verify that the module being requested matches the one the
loader can handle.
@@ -580,7 +535,7 @@ def _check_name(method):
if name is None:
name = self.name
elif self.name != name:
- raise ImportError("loader cannot handle %s" % name, name=name)
+ raise ImportError('loader cannot handle %s' % name, name=name)
return method(self, name, *args, **kwargs)
_wrap(_check_name_wrapper, method)
return _check_name_wrapper
@@ -590,7 +545,7 @@ def _requires_builtin(fxn):
"""Decorator to verify the named module is built-in."""
def _requires_builtin_wrapper(self, fullname):
if fullname not in sys.builtin_module_names:
- raise ImportError("{} is not a built-in module".format(fullname),
+ raise ImportError('{!r} is not a built-in module'.format(fullname),
name=fullname)
return fxn(self, fullname)
_wrap(_requires_builtin_wrapper, fxn)
@@ -601,7 +556,7 @@ def _requires_frozen(fxn):
"""Decorator to verify the named module is frozen."""
def _requires_frozen_wrapper(self, fullname):
if not _imp.is_frozen(fullname):
- raise ImportError("{} is not a frozen module".format(fullname),
+ raise ImportError('{!r} is not a frozen module'.format(fullname),
name=fullname)
return fxn(self, fullname)
_wrap(_requires_frozen_wrapper, fxn)
@@ -610,17 +565,659 @@ def _requires_frozen(fxn):
def _find_module_shim(self, fullname):
"""Try to find a loader for the specified module by delegating to
- self.find_loader()."""
+ self.find_loader().
+
+ This method is deprecated in favor of finder.find_spec().
+
+ """
# Call find_loader(). If it returns a string (indicating this
# is a namespace package portion), generate a warning and
# return None.
loader, portions = self.find_loader(fullname)
if loader is None and len(portions):
- msg = "Not importing directory {}: missing __init__"
+ msg = 'Not importing directory {}: missing __init__'
_warnings.warn(msg.format(portions[0]), ImportWarning)
return loader
+def _load_module_shim(self, fullname):
+ """Load the specified module into sys.modules and return it.
+
+ This method is deprecated. Use loader.exec_module instead.
+
+ """
+ spec = spec_from_loader(fullname, self)
+ methods = _SpecMethods(spec)
+ if fullname in sys.modules:
+ module = sys.modules[fullname]
+ methods.exec(module)
+ return sys.modules[fullname]
+ else:
+ return methods.load()
+
+
+def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
+ """Validate the header of the passed-in bytecode against source_stats (if
+ given) and returning the bytecode that can be compiled by compile().
+
+ All other arguments are used to enhance error reporting.
+
+ ImportError is raised when the magic number is incorrect or the bytecode is
+ found to be stale. EOFError is raised when the data is found to be
+ truncated.
+
+ """
+ exc_details = {}
+ if name is not None:
+ exc_details['name'] = name
+ else:
+ # To prevent having to make all messages have a conditional name.
+ name = '<bytecode>'
+ if path is not None:
+ exc_details['path'] = path
+ magic = data[:4]
+ raw_timestamp = data[4:8]
+ raw_size = data[8:12]
+ if magic != MAGIC_NUMBER:
+ message = 'bad magic number in {!r}: {!r}'.format(name, magic)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ elif len(raw_timestamp) != 4:
+ message = 'reached EOF while reading timestamp in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ elif len(raw_size) != 4:
+ message = 'reached EOF while reading size of source in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ if source_stats is not None:
+ try:
+ source_mtime = int(source_stats['mtime'])
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_timestamp) != source_mtime:
+ message = 'bytecode is stale for {!r}'.format(name)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ try:
+ source_size = source_stats['size'] & 0xFFFFFFFF
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_size) != source_size:
+ raise ImportError('bytecode is stale for {!r}'.format(name),
+ **exc_details)
+ return data[12:]
+
+
+def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
+ """Compile bytecode as returned by _validate_bytecode_header()."""
+ code = marshal.loads(data)
+ if isinstance(code, _code_type):
+ _verbose_message('code object from {!r}', bytecode_path)
+ if source_path is not None:
+ _imp._fix_co_filename(code, source_path)
+ return code
+ else:
+ raise ImportError('Non-code object in {!r}'.format(bytecode_path),
+ name=name, path=bytecode_path)
+
+def _code_to_bytecode(code, mtime=0, source_size=0):
+ """Compile a code object into bytecode for writing out to a byte-compiled
+ file."""
+ data = bytearray(MAGIC_NUMBER)
+ data.extend(_w_long(mtime))
+ data.extend(_w_long(source_size))
+ data.extend(marshal.dumps(code))
+ return data
+
+
+def decode_source(source_bytes):
+ """Decode bytes representing source code and return the string.
+
+ Universal newline support is used in the decoding.
+ """
+ import tokenize # To avoid bootstrap issues.
+ source_bytes_readline = _io.BytesIO(source_bytes).readline
+ encoding = tokenize.detect_encoding(source_bytes_readline)
+ newline_decoder = _io.IncrementalNewlineDecoder(None, True)
+ return newline_decoder.decode(source_bytes.decode(encoding[0]))
+
+
+# Module specifications #######################################################
+
+def _module_repr(module):
+ # The implementation of ModuleType__repr__().
+ loader = getattr(module, '__loader__', None)
+ if hasattr(loader, 'module_repr'):
+ # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader
+ # drop their implementations for module_repr. we can add a
+ # deprecation warning here.
+ try:
+ return loader.module_repr(module)
+ except Exception:
+ pass
+ try:
+ spec = module.__spec__
+ except AttributeError:
+ pass
+ else:
+ if spec is not None:
+ return _SpecMethods(spec).module_repr()
+
+ # We could use module.__class__.__name__ instead of 'module' in the
+ # various repr permutations.
+ try:
+ name = module.__name__
+ except AttributeError:
+ name = '?'
+ try:
+ filename = module.__file__
+ except AttributeError:
+ if loader is None:
+ return '<module {!r}>'.format(name)
+ else:
+ return '<module {!r} ({!r})>'.format(name, loader)
+ else:
+ return '<module {!r} from {!r}>'.format(name, filename)
+
+
+class _installed_safely:
+
+ def __init__(self, module):
+ self._module = module
+ self._spec = module.__spec__
+
+ def __enter__(self):
+ # This must be done before putting the module in sys.modules
+ # (otherwise an optimization shortcut in import.c becomes
+ # wrong)
+ self._spec._initializing = True
+ sys.modules[self._spec.name] = self._module
+
+ def __exit__(self, *args):
+ try:
+ spec = self._spec
+ if any(arg is not None for arg in args):
+ try:
+ del sys.modules[spec.name]
+ except KeyError:
+ pass
+ else:
+ _verbose_message('import {!r} # {!r}', spec.name, spec.loader)
+ finally:
+ self._spec._initializing = False
+
+
+class ModuleSpec:
+ """The specification for a module, used for loading.
+
+ A module's spec is the source for information about the module. For
+ data associated with the module, including source, use the spec's
+ loader.
+
+ `name` is the absolute name of the module. `loader` is the loader
+ to use when loading the module. `parent` is the name of the
+ package the module is in. The parent is derived from the name.
+
+ `is_package` determines if the module is considered a package or
+ not. On modules this is reflected by the `__path__` attribute.
+
+ `origin` is the specific location used by the loader from which to
+ load the module, if that information is available. When filename is
+ set, origin will match.
+
+ `has_location` indicates that a spec's "origin" reflects a location.
+ When this is True, `__file__` attribute of the module is set.
+
+ `cached` is the location of the cached bytecode file, if any. It
+ corresponds to the `__cached__` attribute.
+
+ `submodule_search_locations` is the sequence of path entries to
+ search when importing submodules. If set, is_package should be
+ True--and False otherwise.
+
+ Packages are simply modules that (may) have submodules. If a spec
+ has a non-None value in `submodule_search_locations`, the import
+ system will consider modules loaded from the spec as packages.
+
+ Only finders (see importlib.abc.MetaPathFinder and
+ importlib.abc.PathEntryFinder) should modify ModuleSpec instances.
+
+ """
+
+ def __init__(self, name, loader, *, origin=None, loader_state=None,
+ is_package=None):
+ self.name = name
+ self.loader = loader
+ self.origin = origin
+ self.loader_state = loader_state
+ self.submodule_search_locations = [] if is_package else None
+
+ # file-location attributes
+ self._set_fileattr = False
+ self._cached = None
+
+ def __repr__(self):
+ args = ['name={!r}'.format(self.name),
+ 'loader={!r}'.format(self.loader)]
+ if self.origin is not None:
+ args.append('origin={!r}'.format(self.origin))
+ if self.submodule_search_locations is not None:
+ args.append('submodule_search_locations={}'
+ .format(self.submodule_search_locations))
+ return '{}({})'.format(self.__class__.__name__, ', '.join(args))
+
+ def __eq__(self, other):
+ smsl = self.submodule_search_locations
+ try:
+ return (self.name == other.name and
+ self.loader == other.loader and
+ self.origin == other.origin and
+ smsl == other.submodule_search_locations and
+ self.cached == other.cached and
+ self.has_location == other.has_location)
+ except AttributeError:
+ return False
+
+ @property
+ def cached(self):
+ if self._cached is None:
+ if self.origin is not None and self._set_fileattr:
+ filename = self.origin
+ if filename.endswith(tuple(SOURCE_SUFFIXES)):
+ try:
+ self._cached = cache_from_source(filename)
+ except NotImplementedError:
+ pass
+ elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
+ self._cached = filename
+ return self._cached
+
+ @cached.setter
+ def cached(self, cached):
+ self._cached = cached
+
+ @property
+ def parent(self):
+ """The name of the module's parent."""
+ if self.submodule_search_locations is None:
+ return self.name.rpartition('.')[0]
+ else:
+ return self.name
+
+ @property
+ def has_location(self):
+ return self._set_fileattr
+
+ @has_location.setter
+ def has_location(self, value):
+ self._set_fileattr = bool(value)
+
+
+def spec_from_loader(name, loader, *, origin=None, is_package=None):
+ """Return a module spec based on various loader methods."""
+ if hasattr(loader, 'get_filename'):
+ if is_package is None:
+ return spec_from_file_location(name, loader=loader)
+ search = [] if is_package else None
+ return spec_from_file_location(name, loader=loader,
+ submodule_search_locations=search)
+
+ if is_package is None:
+ if hasattr(loader, 'is_package'):
+ try:
+ is_package = loader.is_package(name)
+ except ImportError:
+ is_package = None # aka, undefined
+ else:
+ # the default
+ is_package = False
+
+ return ModuleSpec(name, loader, origin=origin, is_package=is_package)
+
+
+_POPULATE = object()
+
+
+def spec_from_file_location(name, location=None, *, loader=None,
+ submodule_search_locations=_POPULATE):
+ """Return a module spec based on a file location.
+
+ To indicate that the module is a package, set
+ submodule_search_locations to a list of directory paths. An
+ empty list is sufficient, though its not otherwise useful to the
+ import system.
+
+ The loader must take a spec as its only __init__() arg.
+
+ """
+ if location is None:
+ # The caller may simply want a partially populated location-
+ # oriented spec. So we set the location to a bogus value and
+ # fill in as much as we can.
+ location = '<unknown>'
+ if hasattr(loader, 'get_filename'):
+ # ExecutionLoader
+ try:
+ location = loader.get_filename(name)
+ except ImportError:
+ pass
+
+ # If the location is on the filesystem, but doesn't actually exist,
+ # we could return None here, indicating that the location is not
+ # valid. However, we don't have a good way of testing since an
+ # indirect location (e.g. a zip file or URL) will look like a
+ # non-existent file relative to the filesystem.
+
+ spec = ModuleSpec(name, loader, origin=location)
+ spec._set_fileattr = True
+
+ # Pick a loader if one wasn't provided.
+ if loader is None:
+ for loader_class, suffixes in _get_supported_file_loaders():
+ if location.endswith(tuple(suffixes)):
+ loader = loader_class(name, location)
+ spec.loader = loader
+ break
+ else:
+ return None
+
+ # Set submodule_search_paths appropriately.
+ if submodule_search_locations is _POPULATE:
+ # Check the loader.
+ if hasattr(loader, 'is_package'):
+ try:
+ is_package = loader.is_package(name)
+ except ImportError:
+ pass
+ else:
+ if is_package:
+ spec.submodule_search_locations = []
+ else:
+ spec.submodule_search_locations = submodule_search_locations
+ if spec.submodule_search_locations == []:
+ if location:
+ dirname = _path_split(location)[0]
+ spec.submodule_search_locations.append(dirname)
+
+ return spec
+
+
+def _spec_from_module(module, loader=None, origin=None):
+ # This function is meant for use in _setup().
+ try:
+ spec = module.__spec__
+ except AttributeError:
+ pass
+ else:
+ if spec is not None:
+ return spec
+
+ name = module.__name__
+ if loader is None:
+ try:
+ loader = module.__loader__
+ except AttributeError:
+ # loader will stay None.
+ pass
+ try:
+ location = module.__file__
+ except AttributeError:
+ location = None
+ if origin is None:
+ if location is None:
+ try:
+ origin = loader._ORIGIN
+ except AttributeError:
+ origin = None
+ else:
+ origin = location
+ try:
+ cached = module.__cached__
+ except AttributeError:
+ cached = None
+ try:
+ submodule_search_locations = list(module.__path__)
+ except AttributeError:
+ submodule_search_locations = None
+
+ spec = ModuleSpec(name, loader, origin=origin)
+ spec._set_fileattr = False if location is None else True
+ spec.cached = cached
+ spec.submodule_search_locations = submodule_search_locations
+ return spec
+
+
+class _SpecMethods:
+
+ """Convenience wrapper around spec objects to provide spec-specific
+ methods."""
+
+ # The various spec_from_* functions could be made factory methods here.
+
+ def __init__(self, spec):
+ self.spec = spec
+
+ def module_repr(self):
+ """Return the repr to use for the module."""
+ # We mostly replicate _module_repr() using the spec attributes.
+ spec = self.spec
+ name = '?' if spec.name is None else spec.name
+ if spec.origin is None:
+ if spec.loader is None:
+ return '<module {!r}>'.format(name)
+ else:
+ return '<module {!r} ({!r})>'.format(name, spec.loader)
+ else:
+ if spec.has_location:
+ return '<module {!r} from {!r}>'.format(name, spec.origin)
+ else:
+ return '<module {!r} ({})>'.format(spec.name, spec.origin)
+
+ def init_module_attrs(self, module, *, _override=False, _force_name=True):
+ """Set the module's attributes.
+
+ All missing import-related module attributes will be set. Here
+ is how the spec attributes map onto the module:
+
+ spec.name -> module.__name__
+ spec.loader -> module.__loader__
+ spec.parent -> module.__package__
+ spec -> module.__spec__
+
+ Optional:
+ spec.origin -> module.__file__ (if spec.set_fileattr is true)
+ spec.cached -> module.__cached__ (if __file__ also set)
+ spec.submodule_search_locations -> module.__path__ (if set)
+
+ """
+ spec = self.spec
+
+ # The passed in module may be not support attribute assignment,
+ # in which case we simply don't set the attributes.
+
+ # __name__
+ if (_override or _force_name or
+ getattr(module, '__name__', None) is None):
+ try:
+ module.__name__ = spec.name
+ except AttributeError:
+ pass
+
+ # __loader__
+ if _override or getattr(module, '__loader__', None) is None:
+ loader = spec.loader
+ if loader is None:
+ # A backward compatibility hack.
+ if spec.submodule_search_locations is not None:
+ loader = _NamespaceLoader.__new__(_NamespaceLoader)
+ loader._path = spec.submodule_search_locations
+ try:
+ module.__loader__ = loader
+ except AttributeError:
+ pass
+
+ # __package__
+ if _override or getattr(module, '__package__', None) is None:
+ try:
+ module.__package__ = spec.parent
+ except AttributeError:
+ pass
+
+ # __spec__
+ try:
+ module.__spec__ = spec
+ except AttributeError:
+ pass
+
+ # __path__
+ if _override or getattr(module, '__path__', None) is None:
+ if spec.submodule_search_locations is not None:
+ try:
+ module.__path__ = spec.submodule_search_locations
+ except AttributeError:
+ pass
+
+ if spec.has_location:
+ # __file__
+ if _override or getattr(module, '__file__', None) is None:
+ try:
+ module.__file__ = spec.origin
+ except AttributeError:
+ pass
+
+ # __cached__
+ if _override or getattr(module, '__cached__', None) is None:
+ if spec.cached is not None:
+ try:
+ module.__cached__ = spec.cached
+ except AttributeError:
+ pass
+
+ def create(self):
+ """Return a new module to be loaded.
+
+ The import-related module attributes are also set with the
+ appropriate values from the spec.
+
+ """
+ spec = self.spec
+ # Typically loaders will not implement create_module().
+ if hasattr(spec.loader, 'create_module'):
+ # If create_module() returns `None` it means the default
+ # module creation should be used.
+ module = spec.loader.create_module(spec)
+ else:
+ module = None
+ if module is None:
+ # This must be done before open() is ever called as the 'io'
+ # module implicitly imports 'locale' and would otherwise
+ # trigger an infinite loop.
+ module = _new_module(spec.name)
+ self.init_module_attrs(module)
+ return module
+
+ def _exec(self, module):
+ """Do everything necessary to execute the module.
+
+ The namespace of `module` is used as the target of execution.
+ This method uses the loader's `exec_module()` method.
+
+ """
+ self.spec.loader.exec_module(module)
+
+ # Used by importlib.reload() and _load_module_shim().
+ def exec(self, module):
+ """Execute the spec in an existing module's namespace."""
+ name = self.spec.name
+ _imp.acquire_lock()
+ with _ModuleLockManager(name):
+ if sys.modules.get(name) is not module:
+ msg = 'module {!r} not in sys.modules'.format(name)
+ raise ImportError(msg, name=name)
+ if self.spec.loader is None:
+ if self.spec.submodule_search_locations is None:
+ raise ImportError('missing loader', name=self.spec.name)
+ # namespace package
+ self.init_module_attrs(module, _override=True)
+ return module
+ self.init_module_attrs(module, _override=True)
+ if not hasattr(self.spec.loader, 'exec_module'):
+ # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+ # have exec_module() implemented, we can add a deprecation
+ # warning here.
+ self.spec.loader.load_module(name)
+ else:
+ self._exec(module)
+ return sys.modules[name]
+
+ def _load_backward_compatible(self):
+ # (issue19713) Once BuiltinImporter and ExtensionFileLoader
+ # have exec_module() implemented, we can add a deprecation
+ # warning here.
+ spec = self.spec
+ spec.loader.load_module(spec.name)
+ # The module must be in sys.modules at this point!
+ module = sys.modules[spec.name]
+ if getattr(module, '__loader__', None) is None:
+ try:
+ module.__loader__ = spec.loader
+ except AttributeError:
+ pass
+ if getattr(module, '__package__', None) is None:
+ try:
+ # Since module.__path__ may not line up with
+ # spec.submodule_search_paths, we can't necessarily rely
+ # on spec.parent here.
+ module.__package__ = module.__name__
+ if not hasattr(module, '__path__'):
+ module.__package__ = spec.name.rpartition('.')[0]
+ except AttributeError:
+ pass
+ if getattr(module, '__spec__', None) is None:
+ try:
+ module.__spec__ = spec
+ except AttributeError:
+ pass
+ return module
+
+ def _load_unlocked(self):
+ # A helper for direct use by the import system.
+ if self.spec.loader is not None:
+ # not a namespace package
+ if not hasattr(self.spec.loader, 'exec_module'):
+ return self._load_backward_compatible()
+
+ module = self.create()
+ with _installed_safely(module):
+ if self.spec.loader is None:
+ if self.spec.submodule_search_locations is None:
+ raise ImportError('missing loader', name=self.spec.name)
+ # A namespace package so do nothing.
+ else:
+ self._exec(module)
+
+ # We don't ensure that the import-related module attributes get
+ # set in the sys.modules replacement case. Such modules are on
+ # their own.
+ return sys.modules[self.spec.name]
+
+ # A method used during testing of _load_unlocked() and by
+ # _load_module_shim().
+ def load(self):
+ """Return a new module object, loaded by the spec's loader.
+
+ The module is not added to its parent.
+
+ If a module is already in sys.modules, that existing module gets
+ clobbered.
+
+ """
+ _imp.acquire_lock()
+ with _ModuleLockManager(self.spec.name):
+ return self._load_unlocked()
# Loaders #####################################################################
@@ -634,9 +1231,23 @@ class BuiltinImporter:
"""
+ @staticmethod
+ def module_repr(module):
+ """Return repr for the module.
+
+ The method is deprecated. The import machinery does the job itself.
+
+ """
+ return '<module {!r} (built-in)>'.format(module.__name__)
+
@classmethod
- def module_repr(cls, module):
- return "<module '{}' (built-in)>".format(module.__name__)
+ def find_spec(cls, fullname, path=None, target=None):
+ if path is not None:
+ return None
+ if _imp.is_builtin(fullname):
+ return spec_from_loader(fullname, cls, origin='built-in')
+ else:
+ return None
@classmethod
def find_module(cls, fullname, path=None):
@@ -644,24 +1255,23 @@ class BuiltinImporter:
If 'path' is ever specified then the search is considered a failure.
+ This method is deprecated. Use find_spec() instead.
+
"""
- if path is not None:
- return None
- return cls if _imp.is_builtin(fullname) else None
+ spec = cls.find_spec(fullname, path)
+ return spec.loader if spec is not None else None
@classmethod
- @set_package
- @set_loader
@_requires_builtin
def load_module(cls, fullname):
"""Load a built-in module."""
- is_reload = fullname in sys.modules
- try:
- return _call_with_frames_removed(_imp.init_builtin, fullname)
- except:
- if not is_reload and fullname in sys.modules:
- del sys.modules[fullname]
- raise
+ # Once an exec_module() implementation is added we can also
+ # add a deprecation warning here.
+ with _ManageReload(fullname):
+ module = _call_with_frames_removed(_imp.init_builtin, fullname)
+ module.__loader__ = cls
+ module.__package__ = ''
+ return module
@classmethod
@_requires_builtin
@@ -691,31 +1301,48 @@ class FrozenImporter:
"""
+ @staticmethod
+ def module_repr(m):
+ """Return repr for the module.
+
+ The method is deprecated. The import machinery does the job itself.
+
+ """
+ return '<module {!r} (frozen)>'.format(m.__name__)
+
@classmethod
- def module_repr(cls, m):
- return "<module '{}' (frozen)>".format(m.__name__)
+ def find_spec(cls, fullname, path=None, target=None):
+ if _imp.is_frozen(fullname):
+ return spec_from_loader(fullname, cls, origin='frozen')
+ else:
+ return None
@classmethod
def find_module(cls, fullname, path=None):
- """Find a frozen module."""
+ """Find a frozen module.
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
return cls if _imp.is_frozen(fullname) else None
+ @staticmethod
+ def exec_module(module):
+ name = module.__spec__.name
+ if not _imp.is_frozen(name):
+ raise ImportError('{!r} is not a frozen module'.format(name),
+ name=name)
+ code = _call_with_frames_removed(_imp.get_frozen_object, name)
+ exec(code, module.__dict__)
+
@classmethod
- @set_package
- @set_loader
- @_requires_frozen
def load_module(cls, fullname):
- """Load a frozen module."""
- is_reload = fullname in sys.modules
- try:
- m = _call_with_frames_removed(_imp.init_frozen, fullname)
- # Let our own module_repr() method produce a suitable repr.
- del m.__file__
- return m
- except:
- if not is_reload and fullname in sys.modules:
- del sys.modules[fullname]
- raise
+ """Load a frozen module.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ return _load_module_shim(cls, fullname)
@classmethod
@_requires_frozen
@@ -738,22 +1365,21 @@ class FrozenImporter:
class WindowsRegistryFinder:
- """Meta path finder for modules declared in the Windows registry.
- """
+ """Meta path finder for modules declared in the Windows registry."""
REGISTRY_KEY = (
- "Software\\Python\\PythonCore\\{sys_version}"
- "\\Modules\\{fullname}")
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}')
REGISTRY_KEY_DEBUG = (
- "Software\\Python\\PythonCore\\{sys_version}"
- "\\Modules\\{fullname}\\Debug")
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}\\Debug')
DEBUG_BUILD = False # Changed in _setup()
@classmethod
def _open_registry(cls, key):
try:
return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
- except WindowsError:
+ except OSError:
return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
@classmethod
@@ -766,24 +1392,38 @@ class WindowsRegistryFinder:
sys_version=sys.version[:3])
try:
with cls._open_registry(key) as hkey:
- filepath = _winreg.QueryValue(hkey, "")
- except WindowsError:
+ filepath = _winreg.QueryValue(hkey, '')
+ except OSError:
return None
return filepath
@classmethod
- def find_module(cls, fullname, path=None):
- """Find module named in the registry."""
+ def find_spec(cls, fullname, path=None, target=None):
filepath = cls._search_registry(fullname)
if filepath is None:
return None
try:
- _os.stat(filepath)
+ _path_stat(filepath)
except OSError:
return None
for loader, suffixes in _get_supported_file_loaders():
if filepath.endswith(tuple(suffixes)):
- return loader(fullname, filepath)
+ spec = spec_from_loader(fullname, loader(fullname, filepath),
+ origin=filepath)
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """Find module named in the registry.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is not None:
+ return spec.loader
+ else:
+ return None
class _LoaderBasics:
@@ -799,74 +1439,15 @@ class _LoaderBasics:
tail_name = fullname.rpartition('.')[2]
return filename_base == '__init__' and tail_name != '__init__'
- def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats):
- """Return the marshalled bytes from bytecode, verifying the magic
- number, timestamp and source size along the way.
+ def exec_module(self, module):
+ """Execute the module."""
+ code = self.get_code(module.__name__)
+ if code is None:
+ raise ImportError('cannot load module {!r} when get_code() '
+ 'returns None'.format(module.__name__))
+ _call_with_frames_removed(exec, code, module.__dict__)
- If source_stats is None then skip the timestamp check.
-
- """
- magic = data[:4]
- raw_timestamp = data[4:8]
- raw_size = data[8:12]
- if magic != _MAGIC_BYTES:
- msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic)
- _verbose_message(msg)
- raise ImportError(msg, name=fullname, path=bytecode_path)
- elif len(raw_timestamp) != 4:
- message = 'bad timestamp in {}'.format(fullname)
- _verbose_message(message)
- raise EOFError(message)
- elif len(raw_size) != 4:
- message = 'bad size in {}'.format(fullname)
- _verbose_message(message)
- raise EOFError(message)
- if source_stats is not None:
- try:
- source_mtime = int(source_stats['mtime'])
- except KeyError:
- pass
- else:
- if _r_long(raw_timestamp) != source_mtime:
- message = 'bytecode is stale for {}'.format(fullname)
- _verbose_message(message)
- raise ImportError(message, name=fullname,
- path=bytecode_path)
- try:
- source_size = source_stats['size'] & 0xFFFFFFFF
- except KeyError:
- pass
- else:
- if _r_long(raw_size) != source_size:
- raise ImportError(
- "bytecode is stale for {}".format(fullname),
- name=fullname, path=bytecode_path)
- # Can't return the code object as errors from marshal loading need to
- # propagate even when source is available.
- return data[12:]
-
- @module_for_loader
- def _load_module(self, module, *, sourceless=False):
- """Helper for load_module able to handle either source or sourceless
- loading."""
- name = module.__name__
- code_object = self.get_code(name)
- module.__file__ = self.get_filename(name)
- if not sourceless:
- try:
- module.__cached__ = cache_from_source(module.__file__)
- except NotImplementedError:
- module.__cached__ = module.__file__
- else:
- module.__cached__ = module.__file__
- module.__package__ = name
- if self.is_package(name):
- module.__path__ = [_path_split(module.__file__)[0]]
- else:
- module.__package__ = module.__package__.rpartition('.')[0]
- module.__loader__ = self
- _call_with_frames_removed(exec, code_object, module.__dict__)
- return module
+ load_module = _load_module_shim
class SourceLoader(_LoaderBasics):
@@ -874,8 +1455,10 @@ class SourceLoader(_LoaderBasics):
def path_mtime(self, path):
"""Optional method that returns the modification time (an int) for the
specified path, where path is a str.
+
+ Raises IOError when the path cannot be handled.
"""
- raise NotImplementedError
+ raise IOError
def path_stats(self, path):
"""Optional method returning a metadata dict for the specified path
@@ -886,6 +1469,7 @@ class SourceLoader(_LoaderBasics):
- 'size' (optional) is the size in bytes of the source code.
Implementing this method allows the loader to read bytecode files.
+ Raises IOError when the path cannot be handled.
"""
return {'mtime': self.path_mtime(path)}
@@ -903,32 +1487,26 @@ class SourceLoader(_LoaderBasics):
"""Optional method which writes data (bytes) to a file path (a str).
Implementing this method allows for the writing of bytecode files.
-
"""
- raise NotImplementedError
def get_source(self, fullname):
"""Concrete implementation of InspectLoader.get_source."""
- import tokenize
path = self.get_filename(fullname)
try:
source_bytes = self.get_data(path)
- except IOError as exc:
- raise ImportError("source not available through get_data()",
- name=fullname) from exc
- readsource = _io.BytesIO(source_bytes).readline
- try:
- encoding = tokenize.detect_encoding(readsource)
- except SyntaxError as exc:
- raise ImportError("Failed to detect encoding",
- name=fullname) from exc
- newline_decoder = _io.IncrementalNewlineDecoder(None, True)
- try:
- return newline_decoder.decode(source_bytes.decode(encoding[0]))
- except UnicodeDecodeError as exc:
- raise ImportError("Failed to decode source file",
+ except OSError as exc:
+ raise ImportError('source not available through get_data()',
name=fullname) from exc
+ return decode_source(source_bytes)
+
+ def source_to_code(self, data, path, *, _optimize=-1):
+ """Return the code object compiled from source.
+
+ The 'data' argument can be any object type that compile() supports.
+ """
+ return _call_with_frames_removed(compile, data, path, 'exec',
+ dont_inherit=True, optimize=_optimize)
def get_code(self, fullname):
"""Concrete implementation of InspectLoader.get_code.
@@ -946,45 +1524,34 @@ class SourceLoader(_LoaderBasics):
else:
try:
st = self.path_stats(source_path)
- except NotImplementedError:
+ except IOError:
pass
else:
source_mtime = int(st['mtime'])
try:
data = self.get_data(bytecode_path)
- except IOError:
+ except OSError:
pass
else:
try:
- bytes_data = self._bytes_from_bytecode(fullname, data,
- bytecode_path,
- st)
+ bytes_data = _validate_bytecode_header(data,
+ source_stats=st, name=fullname,
+ path=bytecode_path)
except (ImportError, EOFError):
pass
else:
_verbose_message('{} matches {}', bytecode_path,
source_path)
- found = marshal.loads(bytes_data)
- if isinstance(found, _code_type):
- _imp._fix_co_filename(found, source_path)
- _verbose_message('code object from {}',
- bytecode_path)
- return found
- else:
- msg = "Non-code object in {}"
- raise ImportError(msg.format(bytecode_path),
- name=fullname, path=bytecode_path)
+ return _compile_bytecode(bytes_data, name=fullname,
+ bytecode_path=bytecode_path,
+ source_path=source_path)
source_bytes = self.get_data(source_path)
- code_object = _call_with_frames_removed(compile,
- source_bytes, source_path, 'exec',
- dont_inherit=True)
+ code_object = self.source_to_code(source_bytes, source_path)
_verbose_message('code object from {}', source_path)
if (not sys.dont_write_bytecode and bytecode_path is not None and
- source_mtime is not None):
- data = bytearray(_MAGIC_BYTES)
- data.extend(_w_long(source_mtime))
- data.extend(_w_long(len(source_bytes)))
- data.extend(marshal.dumps(code_object))
+ source_mtime is not None):
+ data = _code_to_bytecode(code_object, source_mtime,
+ len(source_bytes))
try:
self._cache_bytecode(source_path, bytecode_path, data)
_verbose_message('wrote {!r}', bytecode_path)
@@ -992,16 +1559,6 @@ class SourceLoader(_LoaderBasics):
pass
return code_object
- def load_module(self, fullname):
- """Concrete implementation of Loader.load_module.
-
- Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be
- implemented to load source code. Use of bytecode is dictated by whether
- get_code uses/writes bytecode.
-
- """
- return self._load_module(fullname)
-
class FileLoader:
@@ -1014,10 +1571,22 @@ class FileLoader:
self.name = fullname
self.path = path
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
@_check_name
def load_module(self, fullname):
- """Load a module from a file."""
- # Issue #14857: Avoid the zero-argument form so the implementation
+ """Load a module from a file.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The only reason for this method is for the name check.
+ # Issue #14857: Avoid the zero-argument form of super so the implementation
# of that form can be updated without breaking the frozen module
return super(FileLoader, self).load_module(fullname)
@@ -1038,18 +1607,12 @@ class SourceFileLoader(FileLoader, SourceLoader):
def path_stats(self, path):
"""Return the metadata for the path."""
- st = _os.stat(path)
+ st = _path_stat(path)
return {'mtime': st.st_mtime, 'size': st.st_size}
def _cache_bytecode(self, source_path, bytecode_path, data):
# Adapt between the two APIs
- try:
- mode = _os.stat(source_path).st_mode
- except OSError:
- mode = 0o666
- # We always ensure write access so we can update cached files
- # later even when the source files are read-only on Windows (#6074)
- mode |= 0o200
+ mode = _calc_mode(source_path)
return self.set_data(bytecode_path, data, _mode=mode)
def set_data(self, path, data, *, _mode=0o666):
@@ -1085,20 +1648,11 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
"""Loader which handles sourceless file imports."""
- def load_module(self, fullname):
- return self._load_module(fullname, sourceless=True)
-
def get_code(self, fullname):
path = self.get_filename(fullname)
data = self.get_data(path)
- bytes_data = self._bytes_from_bytecode(fullname, data, path, None)
- found = marshal.loads(bytes_data)
- if isinstance(found, _code_type):
- _verbose_message('code object from {!r}', path)
- return found
- else:
- raise ImportError("Non-code object in {}".format(path),
- name=fullname, path=path)
+ bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
+ return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
def get_source(self, fullname):
"""Return None as there is no source code."""
@@ -1121,23 +1675,30 @@ class ExtensionFileLoader:
self.name = name
self.path = path
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
@_check_name
- @set_package
- @set_loader
def load_module(self, fullname):
"""Load an extension module."""
- is_reload = fullname in sys.modules
- try:
+ # Once an exec_module() implementation is added we can also
+ # add a deprecation warning here.
+ with _ManageReload(fullname):
module = _call_with_frames_removed(_imp.load_dynamic,
fullname, self.path)
- _verbose_message('extension module loaded from {!r}', self.path)
- if self.is_package(fullname) and not hasattr(module, '__path__'):
- module.__path__ = [_path_split(self.path)[0]]
- return module
- except:
- if not is_reload and fullname in sys.modules:
- del sys.modules[fullname]
- raise
+ _verbose_message('extension module loaded from {!r}', self.path)
+ is_package = self.is_package(fullname)
+ if is_package and not hasattr(module, '__path__'):
+ module.__path__ = [_path_split(self.path)[0]]
+ module.__loader__ = self
+ module.__package__ = module.__name__
+ if not is_package:
+ module.__package__ = module.__package__.rpartition('.')[0]
+ return module
def is_package(self, fullname):
"""Return True if the extension module is a package."""
@@ -1153,6 +1714,11 @@ class ExtensionFileLoader:
"""Return None as extension modules have no source code."""
return None
+ @_check_name
+ def get_filename(self, fullname):
+ """Return the path to the source file as found by the finder."""
+ return self.path
+
class _NamespacePath:
"""Represents a namespace package's path. It uses the module name
@@ -1185,11 +1751,12 @@ class _NamespacePath:
# If the parent's path has changed, recalculate _path
parent_path = tuple(self._get_parent_path()) # Make a copy
if parent_path != self._last_parent_path:
- loader, new_path = self._path_finder(self._name, parent_path)
+ spec = self._path_finder(self._name, parent_path)
# Note that no changes are made if a loader is returned, but we
# do remember the new parent path
- if loader is None:
- self._path = new_path
+ if spec is not None and spec.loader is None:
+ if spec.submodule_search_locations:
+ self._path = spec.submodule_search_locations
self._last_parent_path = parent_path # Save the copy
return self._path
@@ -1200,7 +1767,7 @@ class _NamespacePath:
return len(self._recalculate())
def __repr__(self):
- return "_NamespacePath({!r})".format(self._path)
+ return '_NamespacePath({!r})'.format(self._path)
def __contains__(self, item):
return item in self._recalculate()
@@ -1209,20 +1776,41 @@ class _NamespacePath:
self._path.append(item)
-class NamespaceLoader:
+# We use this exclusively in init_module_attrs() for backward-compatibility.
+class _NamespaceLoader:
def __init__(self, name, path, path_finder):
self._path = _NamespacePath(name, path, path_finder)
@classmethod
def module_repr(cls, module):
- return "<module '{}' (namespace)>".format(module.__name__)
+ """Return repr for the module.
+
+ The method is deprecated. The import machinery does the job itself.
+
+ """
+ return '<module {!r} (namespace)>'.format(module.__name__)
+
+ def is_package(self, fullname):
+ return True
+
+ def get_source(self, fullname):
+ return ''
+
+ def get_code(self, fullname):
+ return compile('', '<string>', 'exec', dont_inherit=True)
- @module_for_loader
- def load_module(self, module):
- """Load a namespace module."""
+ def exec_module(self, module):
+ pass
+
+ def load_module(self, fullname):
+ """Load a namespace module.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The import system never calls this method.
_verbose_message('namespace module loaded with path {!r}', self._path)
- module.__path__ = self._path
- return module
+ return _load_module_shim(self, fullname)
# Finders #####################################################################
@@ -1265,7 +1853,7 @@ class PathFinder:
"""
if path == '':
- path = '.'
+ path = _os.getcwd()
try:
finder = sys.path_importer_cache[path]
except KeyError:
@@ -1274,7 +1862,22 @@ class PathFinder:
return finder
@classmethod
- def _get_loader(cls, fullname, path):
+ def _legacy_get_spec(cls, fullname, finder):
+ # This would be a good place for a DeprecationWarning if
+ # we ended up going that route.
+ if hasattr(finder, 'find_loader'):
+ loader, portions = finder.find_loader(fullname)
+ else:
+ loader = finder.find_module(fullname)
+ portions = []
+ if loader is not None:
+ return spec_from_loader(fullname, loader)
+ spec = ModuleSpec(fullname, None)
+ spec.submodule_search_locations = portions
+ return spec
+
+ @classmethod
+ def _get_spec(cls, fullname, path, target=None):
"""Find the loader or namespace_path for this module/package name."""
# If this ends up being a namespace package, namespace_path is
# the list of paths that will become its __path__
@@ -1284,38 +1887,61 @@ class PathFinder:
continue
finder = cls._path_importer_cache(entry)
if finder is not None:
- if hasattr(finder, 'find_loader'):
- loader, portions = finder.find_loader(fullname)
+ if hasattr(finder, 'find_spec'):
+ spec = finder.find_spec(fullname, target)
else:
- loader = finder.find_module(fullname)
- portions = []
- if loader is not None:
- # We found a loader: return it immediately.
- return loader, namespace_path
+ spec = cls._legacy_get_spec(fullname, finder)
+ if spec is None:
+ continue
+ if spec.loader is not None:
+ return spec
+ portions = spec.submodule_search_locations
+ if portions is None:
+ raise ImportError('spec missing loader')
# This is possibly part of a namespace package.
# Remember these path entries (if any) for when we
# create a namespace package, and continue iterating
# on path.
namespace_path.extend(portions)
else:
- return None, namespace_path
+ spec = ModuleSpec(fullname, None)
+ spec.submodule_search_locations = namespace_path
+ return spec
@classmethod
- def find_module(cls, fullname, path=None):
- """Find the module on sys.path or 'path' based on sys.path_hooks and
+ def find_spec(cls, fullname, path=None, target=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
sys.path_importer_cache."""
if path is None:
path = sys.path
- loader, namespace_path = cls._get_loader(fullname, path)
- if loader is not None:
- return loader
- else:
+ spec = cls._get_spec(fullname, path, target)
+ if spec is None:
+ return None
+ elif spec.loader is None:
+ namespace_path = spec.submodule_search_locations
if namespace_path:
# We found at least one namespace path. Return a
- # loader which can create the namespace package.
- return NamespaceLoader(fullname, namespace_path, cls._get_loader)
+ # spec which can create the namespace package.
+ spec.origin = 'namespace'
+ spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
+ return spec
else:
return None
+ else:
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
+ sys.path_importer_cache.
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is None:
+ return None
+ return spec.loader
class FileFinder:
@@ -1349,11 +1975,28 @@ class FileFinder:
def find_loader(self, fullname):
"""Try to find a loader for the specified module, or the namespace
+ package portions. Returns (loader, list-of-portions).
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = self.find_spec(fullname)
+ if spec is None:
+ return None, []
+ return spec.loader, spec.submodule_search_locations or []
+
+ def _get_spec(self, loader_class, fullname, path, smsl, target):
+ loader = loader_class(fullname, path)
+ return spec_from_file_location(fullname, path, loader=loader,
+ submodule_search_locations=smsl)
+
+ def find_spec(self, fullname, target=None):
+ """Try to find a loader for the specified module, or the namespace
package portions. Returns (loader, list-of-portions)."""
is_namespace = False
tail_module = fullname.rpartition('.')[2]
try:
- mtime = _os.stat(self.path).st_mtime
+ mtime = _path_stat(self.path or _os.getcwd()).st_mtime
except OSError:
mtime = -1
if mtime != self._path_mtime:
@@ -1369,33 +2012,34 @@ class FileFinder:
# Check if the module is the name of a directory (and thus a package).
if cache_module in cache:
base_path = _path_join(self.path, tail_module)
- if _path_isdir(base_path):
- for suffix, loader in self._loaders:
- init_filename = '__init__' + suffix
- full_path = _path_join(base_path, init_filename)
- if _path_isfile(full_path):
- return (loader(fullname, full_path), [base_path])
- else:
- # A namespace package, return the path if we don't also
- # find a module in the next section.
- is_namespace = True
+ for suffix, loader_class in self._loaders:
+ init_filename = '__init__' + suffix
+ full_path = _path_join(base_path, init_filename)
+ if _path_isfile(full_path):
+ return self._get_spec(loader_class, fullname, full_path, [base_path], target)
+ else:
+ # If a namespace package, return the path if we don't
+ # find a module in the next section.
+ is_namespace = _path_isdir(base_path)
# Check for a file w/ a proper suffix exists.
- for suffix, loader in self._loaders:
+ for suffix, loader_class in self._loaders:
full_path = _path_join(self.path, tail_module + suffix)
_verbose_message('trying {}'.format(full_path), verbosity=2)
if cache_module + suffix in cache:
if _path_isfile(full_path):
- return (loader(fullname, full_path), [])
+ return self._get_spec(loader_class, fullname, full_path, None, target)
if is_namespace:
_verbose_message('possible namespace for {}'.format(base_path))
- return (None, [base_path])
- return (None, [])
+ spec = ModuleSpec(fullname, None)
+ spec.submodule_search_locations = [base_path]
+ return spec
+ return None
def _fill_cache(self):
"""Fill the cache of potential modules and packages for this directory."""
path = self.path
try:
- contents = _os.listdir(path)
+ contents = _os.listdir(path or _os.getcwd())
except (FileNotFoundError, PermissionError, NotADirectoryError):
# Directory has either been removed, turned into a file, or made
# unreadable.
@@ -1420,7 +2064,7 @@ class FileFinder:
lower_suffix_contents.add(new_name)
self._path_cache = lower_suffix_contents
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
- self._relaxed_path_cache = set(fn.lower() for fn in contents)
+ self._relaxed_path_cache = {fn.lower() for fn in contents}
@classmethod
def path_hook(cls, *loader_details):
@@ -1435,13 +2079,13 @@ class FileFinder:
def path_hook_for_FileFinder(path):
"""Path hook for importlib.machinery.FileFinder."""
if not _path_isdir(path):
- raise ImportError("only directories are supported", path=path)
+ raise ImportError('only directories are supported', path=path)
return cls(path, *loader_details)
return path_hook_for_FileFinder
def __repr__(self):
- return "FileFinder(%r)" % (self.path,)
+ return 'FileFinder({!r})'.format(self.path)
# Import itself ###############################################################
@@ -1468,19 +2112,51 @@ def _resolve_name(name, package, level):
return '{}.{}'.format(base, name) if name else base
-def _find_module(name, path):
+def _find_spec_legacy(finder, name, path):
+ # This would be a good place for a DeprecationWarning if
+ # we ended up going that route.
+ loader = finder.find_module(name, path)
+ if loader is None:
+ return None
+ return spec_from_loader(name, loader)
+
+
+def _find_spec(name, path, target=None):
"""Find a module's loader."""
if not sys.meta_path:
_warnings.warn('sys.meta_path is empty', ImportWarning)
+ # We check sys.modules here for the reload case. While a passed-in
+ # target will usually indicate a reload there is no guarantee, whereas
+ # sys.modules provides one.
+ is_reload = name in sys.modules
for finder in sys.meta_path:
with _ImportLockContext():
- loader = finder.find_module(name, path)
- if loader is not None:
+ try:
+ find_spec = finder.find_spec
+ except AttributeError:
+ spec = _find_spec_legacy(finder, name, path)
+ if spec is None:
+ continue
+ else:
+ spec = find_spec(name, path, target)
+ if spec is not None:
# The parent import may have already imported this module.
- if name not in sys.modules:
- return loader
+ if not is_reload and name in sys.modules:
+ module = sys.modules[name]
+ try:
+ __spec__ = module.__spec__
+ except AttributeError:
+ # We use the found spec since that is the one that
+ # we would have used if the parent module hadn't
+ # beaten us to the punch.
+ return spec
+ else:
+ if __spec__ is None:
+ return spec
+ else:
+ return __spec__
else:
- return sys.modules[name].__loader__
+ return spec
else:
return None
@@ -1488,21 +2164,22 @@ def _find_module(name, path):
def _sanity_check(name, package, level):
"""Verify arguments are "sane"."""
if not isinstance(name, str):
- raise TypeError("module name must be str, not {}".format(type(name)))
+ raise TypeError('module name must be str, not {}'.format(type(name)))
if level < 0:
raise ValueError('level must be >= 0')
if package:
if not isinstance(package, str):
- raise TypeError("__package__ not set to a string")
+ raise TypeError('__package__ not set to a string')
elif package not in sys.modules:
- msg = ("Parent module {!r} not loaded, cannot perform relative "
- "import")
+ msg = ('Parent module {!r} not loaded, cannot perform relative '
+ 'import')
raise SystemError(msg.format(package))
if not name and level == 0:
- raise ValueError("Empty module name")
+ raise ValueError('Empty module name')
-_ERR_MSG = 'No module named {!r}'
+_ERR_MSG_PREFIX = 'No module named '
+_ERR_MSG = _ERR_MSG_PREFIX + '{!r}'
def _find_and_load_unlocked(name, import_):
path = None
@@ -1513,58 +2190,28 @@ def _find_and_load_unlocked(name, import_):
# Crazy side-effects!
if name in sys.modules:
return sys.modules[name]
- # Backwards-compatibility; be nicer to skip the dict lookup.
parent_module = sys.modules[parent]
try:
path = parent_module.__path__
except AttributeError:
- msg = (_ERR_MSG + '; {} is not a package').format(name, parent)
+ msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent)
raise ImportError(msg, name=name)
- loader = _find_module(name, path)
- if loader is None:
- exc = ImportError(_ERR_MSG.format(name), name=name)
- # TODO(brett): switch to a proper ModuleNotFound exception in Python
- # 3.4.
- exc._not_found = True
- raise exc
- elif name not in sys.modules:
- # The parent import may have already imported this module.
- loader.load_module(name)
- _verbose_message('import {!r} # {!r}', name, loader)
- # Backwards-compatibility; be nicer to skip the dict lookup.
- module = sys.modules[name]
+ spec = _find_spec(name, path)
+ if spec is None:
+ raise ImportError(_ERR_MSG.format(name), name=name)
+ else:
+ module = _SpecMethods(spec)._load_unlocked()
if parent:
# Set the module as an attribute on its parent.
parent_module = sys.modules[parent]
setattr(parent_module, name.rpartition('.')[2], module)
- # Set __package__ if the loader did not.
- if getattr(module, '__package__', None) is None:
- try:
- module.__package__ = module.__name__
- if not hasattr(module, '__path__'):
- module.__package__ = module.__package__.rpartition('.')[0]
- except AttributeError:
- pass
- # Set loader if need be.
- if not hasattr(module, '__loader__'):
- try:
- module.__loader__ = loader
- except AttributeError:
- pass
return module
def _find_and_load(name, import_):
"""Find and load the module, and release the import lock."""
- try:
- lock = _get_module_lock(name)
- finally:
- _imp.release_lock()
- lock.acquire()
- try:
+ with _ModuleLockManager(name):
return _find_and_load_unlocked(name, import_)
- finally:
- lock.release()
def _gcd_import(name, package=None, level=0):
@@ -1585,8 +2232,8 @@ def _gcd_import(name, package=None, level=0):
module = sys.modules[name]
if module is None:
_imp.release_lock()
- message = ("import of {} halted; "
- "None in sys.modules".format(name))
+ message = ('import of {} halted; '
+ 'None in sys.modules'.format(name))
raise ImportError(message, name=name)
_lock_unlock_module(name)
return module
@@ -1616,9 +2263,7 @@ def _handle_fromlist(module, fromlist, import_):
# Backwards-compatibility dictates we ignore failed
# imports triggered by fromlist for modules that don't
# exist.
- # TODO(brett): In Python 3.4, have import raise
- # ModuleNotFound and catch that.
- if getattr(exc, '_not_found', False):
+ if str(exc).startswith(_ERR_MSG_PREFIX):
if exc.name == from_name:
continue
raise
@@ -1686,6 +2331,13 @@ def __import__(name, globals=None, locals=None, fromlist=(), level=0):
return _handle_fromlist(module, fromlist, _gcd_import)
+def _builtin_from_name(name):
+ spec = BuiltinImporter.find_spec(name)
+ if spec is None:
+ raise ImportError('no built-in module named ' + name)
+ methods = _SpecMethods(spec)
+ return methods._load_unlocked()
+
def _setup(sys_module, _imp_module):
"""Setup importlib by importing needed built-in modules and injecting them
@@ -1704,24 +2356,31 @@ def _setup(sys_module, _imp_module):
else:
BYTECODE_SUFFIXES = DEBUG_BYTECODE_SUFFIXES
+ # Set up the spec for existing builtin/frozen modules.
module_type = type(sys)
for name, module in sys.modules.items():
if isinstance(module, module_type):
- if not hasattr(module, '__loader__'):
- if name in sys.builtin_module_names:
- module.__loader__ = BuiltinImporter
- elif _imp.is_frozen(name):
- module.__loader__ = FrozenImporter
+ if name in sys.builtin_module_names:
+ loader = BuiltinImporter
+ elif _imp.is_frozen(name):
+ loader = FrozenImporter
+ else:
+ continue
+ spec = _spec_from_module(module, loader)
+ methods = _SpecMethods(spec)
+ methods.init_module_attrs(module)
+ # Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]
for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
if builtin_name not in sys.modules:
- builtin_module = BuiltinImporter.load_module(builtin_name)
+ builtin_module = _builtin_from_name(builtin_name)
else:
builtin_module = sys.modules[builtin_name]
setattr(self_module, builtin_name, builtin_module)
- os_details = ('posix', ['/']), ('nt', ['\\', '/']), ('os2', ['\\', '/'])
+ # Directly load the os module (needed during bootstrap).
+ os_details = ('posix', ['/']), ('nt', ['\\', '/'])
for builtin_os, path_separators in os_details:
# Assumption made in _path_join()
assert all(len(sep) == 1 for sep in path_separators)
@@ -1731,32 +2390,33 @@ def _setup(sys_module, _imp_module):
break
else:
try:
- os_module = BuiltinImporter.load_module(builtin_os)
- # TODO: rip out os2 code after 3.3 is released as per PEP 11
- if builtin_os == 'os2' and 'EMX GCC' in sys.version:
- path_sep = path_separators[1]
+ os_module = _builtin_from_name(builtin_os)
break
except ImportError:
continue
else:
raise ImportError('importlib requires posix or nt')
+ setattr(self_module, '_os', os_module)
+ setattr(self_module, 'path_sep', path_sep)
+ setattr(self_module, 'path_separators', ''.join(path_separators))
+ # Directly load the _thread module (needed during bootstrap).
try:
- thread_module = BuiltinImporter.load_module('_thread')
+ thread_module = _builtin_from_name('_thread')
except ImportError:
# Python was built without threads
thread_module = None
- weakref_module = BuiltinImporter.load_module('_weakref')
+ setattr(self_module, '_thread', thread_module)
+ # Directly load the _weakref module (needed during bootstrap).
+ weakref_module = _builtin_from_name('_weakref')
+ setattr(self_module, '_weakref', weakref_module)
+
+ # Directly load the winreg module (needed during bootstrap).
if builtin_os == 'nt':
- winreg_module = BuiltinImporter.load_module('winreg')
+ winreg_module = _builtin_from_name('winreg')
setattr(self_module, '_winreg', winreg_module)
- setattr(self_module, '_os', os_module)
- setattr(self_module, '_thread', thread_module)
- setattr(self_module, '_weakref', weakref_module)
- setattr(self_module, 'path_sep', path_sep)
- setattr(self_module, 'path_separators', set(path_separators))
# Constants
setattr(self_module, '_relax_case', _make_relax_case())
EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index 387567a..558abd3 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -8,11 +8,6 @@ except ImportError as exc:
raise
_frozen_importlib = None
import abc
-import imp
-import marshal
-import sys
-import tokenize
-import warnings
def _register(abstract_cls, *classes):
@@ -37,28 +32,37 @@ class Finder(metaclass=abc.ABCMeta):
def find_module(self, fullname, path=None):
"""An abstract method that should find a module.
The fullname is a str and the optional path is a str or None.
- Returns a Loader object.
+ Returns a Loader object or None.
"""
- raise NotImplementedError
class MetaPathFinder(Finder):
"""Abstract base class for import finders on sys.meta_path."""
- @abc.abstractmethod
+ # We don't define find_spec() here since that would break
+ # hasattr checks we do to support backward compatibility.
+
def find_module(self, fullname, path):
- """Abstract method which, when implemented, should find a module.
- The fullname is a str and the path is a str or None.
- Returns a Loader object.
+ """Return a loader for the module.
+
+ If no module is found, return None. The fullname is a str and
+ the path is a list of strings or None.
+
+ This method is deprecated in favor of finder.find_spec(). If find_spec()
+ exists then backwards-compatible functionality is provided for this
+ method.
+
"""
- raise NotImplementedError
+ if not hasattr(self, 'find_spec'):
+ return None
+ found = self.find_spec(fullname, path)
+ return found.loader if found is not None else None
def invalidate_caches(self):
"""An optional method for clearing the finder's cache, if any.
This method is used by importlib.invalidate_caches().
"""
- return NotImplemented
_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
machinery.PathFinder, machinery.WindowsRegistryFinder)
@@ -68,15 +72,35 @@ class PathEntryFinder(Finder):
"""Abstract base class for path entry finders used by PathFinder."""
- @abc.abstractmethod
+ # We don't define find_spec() here since that would break
+ # hasattr checks we do to support backward compatibility.
+
def find_loader(self, fullname):
- """Abstract method which, when implemented, returns a module loader.
- The fullname is a str. Returns a 2-tuple of (Loader, portion) where
- portion is a sequence of file system locations contributing to part of
- a namespace package. The sequence may be empty and the loader may be
- None.
+ """Return (loader, namespace portion) for the path entry.
+
+ The fullname is a str. The namespace portion is a sequence of
+ path entries contributing to part of a namespace package. The
+ sequence may be empty. If loader is not None, the portion will
+ be ignored.
+
+ The portion will be discarded if another path entry finder
+ locates the module as a normal module or package.
+
+ This method is deprecated in favor of finder.find_spec(). If find_spec()
+ is provided than backwards-compatible functionality is provided.
+
"""
- raise NotImplementedError
+ if not hasattr(self, 'find_spec'):
+ return None, []
+ found = self.find_spec(fullname)
+ if found is not None:
+ if not found.submodule_search_locations:
+ portions = []
+ else:
+ portions = found.submodule_search_locations
+ return found.loader, portions
+ else:
+ return None, []
find_module = _bootstrap._find_module_shim
@@ -84,7 +108,6 @@ class PathEntryFinder(Finder):
"""An optional method for clearing the finder's cache, if any.
This method is used by PathFinder.invalidate_caches().
"""
- return NotImplemented
_register(PathEntryFinder, machinery.FileFinder)
@@ -93,16 +116,49 @@ class Loader(metaclass=abc.ABCMeta):
"""Abstract base class for import loaders."""
- @abc.abstractmethod
+ def create_module(self, spec):
+ """Return a module to initialize and into which to load.
+
+ This method should raise ImportError if anything prevents it
+ from creating a new module. It may return None to indicate
+ that the spec should create the new module.
+
+ create_module() is optional.
+
+ """
+ # By default, defer to _SpecMethods.create() for the new module.
+ return None
+
+ # We don't define exec_module() here since that would break
+ # hasattr checks we do to support backward compatibility.
+
def load_module(self, fullname):
- """Abstract method which when implemented should load a module.
- The fullname is a str."""
- raise NotImplementedError
+ """Return the loaded module.
+
+ The module must be added to sys.modules and have import-related
+ attributes set properly. The fullname is a str.
+
+ ImportError is raised on failure.
+
+ This method is deprecated in favor of loader.exec_module(). If
+ exec_module() exists then it is used to provide a backwards-compatible
+ functionality for this method.
+
+ """
+ if not hasattr(self, 'exec_module'):
+ raise ImportError
+ return _bootstrap._load_module_shim(self, fullname)
- @abc.abstractmethod
def module_repr(self, module):
- """Abstract method which when implemented calculates and returns the
- given module's repr."""
+ """Return a module's repr.
+
+ Used by the module type when the method does not raise
+ NotImplementedError.
+
+ This method is deprecated.
+
+ """
+ # The exception will cause ModuleType.__repr__ to ignore this method.
raise NotImplementedError
@@ -119,7 +175,7 @@ class ResourceLoader(Loader):
def get_data(self, path):
"""Abstract method which when implemented should return the bytes for
the specified path. The path must be a str."""
- raise NotImplementedError
+ raise IOError
class InspectLoader(Loader):
@@ -131,26 +187,47 @@ class InspectLoader(Loader):
"""
- @abc.abstractmethod
def is_package(self, fullname):
- """Abstract method which when implemented should return whether the
- module is a package. The fullname is a str. Returns a bool."""
- raise NotImplementedError
+ """Optional method which when implemented should return whether the
+ module is a package. The fullname is a str. Returns a bool.
+
+ Raises ImportError if the module cannot be found.
+ """
+ raise ImportError
- @abc.abstractmethod
def get_code(self, fullname):
- """Abstract method which when implemented should return the code object
- for the module. The fullname is a str. Returns a types.CodeType."""
- raise NotImplementedError
+ """Method which returns the code object for the module.
+
+ The fullname is a str. Returns a types.CodeType if possible, else
+ returns None if a code object does not make sense
+ (e.g. built-in module). Raises ImportError if the module cannot be
+ found.
+ """
+ source = self.get_source(fullname)
+ if source is None:
+ return None
+ return self.source_to_code(source)
@abc.abstractmethod
def get_source(self, fullname):
"""Abstract method which should return the source code for the
- module. The fullname is a str. Returns a str."""
- raise NotImplementedError
+ module. The fullname is a str. Returns a str.
+
+ Raises ImportError if the module cannot be found.
+ """
+ raise ImportError
+
+ def source_to_code(self, data, path='<string>'):
+ """Compile 'data' into a code object.
+
+ The 'data' argument can be anything that compile() can handle. The'path'
+ argument should be where the data was retrieved (when applicable)."""
+ return compile(data, path, 'exec', dont_inherit=True)
-_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter,
- machinery.ExtensionFileLoader)
+ exec_module = _bootstrap._LoaderBasics.exec_module
+ load_module = _bootstrap._LoaderBasics.load_module
+
+_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter)
class ExecutionLoader(InspectLoader):
@@ -165,8 +242,29 @@ class ExecutionLoader(InspectLoader):
@abc.abstractmethod
def get_filename(self, fullname):
"""Abstract method which should return the value that __file__ is to be
- set to."""
- raise NotImplementedError
+ set to.
+
+ Raises ImportError if the module cannot be found.
+ """
+ raise ImportError
+
+ def get_code(self, fullname):
+ """Method to return the code object for fullname.
+
+ Should return None if not applicable (e.g. built-in module).
+ Raise ImportError if the module cannot be found.
+ """
+ source = self.get_source(fullname)
+ if source is None:
+ return None
+ try:
+ path = self.get_filename(fullname)
+ except ImportError:
+ return self.source_to_code(source)
+ else:
+ return self.source_to_code(source, path)
+
+_register(ExecutionLoader, machinery.ExtensionFileLoader)
class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
@@ -198,7 +296,7 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
def path_mtime(self, path):
"""Return the (int) modification time for the path (str)."""
if self.path_stats.__func__ is SourceLoader.path_stats:
- raise NotImplementedError
+ raise IOError
return int(self.path_stats(path)['mtime'])
def path_stats(self, path):
@@ -209,7 +307,7 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
- 'size' (optional) is the size in bytes of the source code.
"""
if self.path_mtime.__func__ is SourceLoader.path_mtime:
- raise NotImplementedError
+ raise IOError
return {'mtime': self.path_mtime(path)}
def set_data(self, path, data):
@@ -220,185 +318,6 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
Any needed intermediary directories are to be created. If for some
reason the file cannot be written because of permissions, fail
silently.
-
"""
- raise NotImplementedError
_register(SourceLoader, machinery.SourceFileLoader)
-
-class PyLoader(SourceLoader):
-
- """Implement the deprecated PyLoader ABC in terms of SourceLoader.
-
- This class has been deprecated! It is slated for removal in Python 3.4.
- If compatibility with Python 3.1 is not needed then implement the
- SourceLoader ABC instead of this class. If Python 3.1 compatibility is
- needed, then use the following idiom to have a single class that is
- compatible with Python 3.1 onwards::
-
- try:
- from importlib.abc import SourceLoader
- except ImportError:
- from importlib.abc import PyLoader as SourceLoader
-
-
- class CustomLoader(SourceLoader):
- def get_filename(self, fullname):
- # Implement ...
-
- def source_path(self, fullname):
- '''Implement source_path in terms of get_filename.'''
- try:
- return self.get_filename(fullname)
- except ImportError:
- return None
-
- def is_package(self, fullname):
- filename = os.path.basename(self.get_filename(fullname))
- return os.path.splitext(filename)[0] == '__init__'
-
- """
-
- @abc.abstractmethod
- def is_package(self, fullname):
- raise NotImplementedError
-
- @abc.abstractmethod
- def source_path(self, fullname):
- """Abstract method. Accepts a str module name and returns the path to
- the source code for the module."""
- raise NotImplementedError
-
- def get_filename(self, fullname):
- """Implement get_filename in terms of source_path.
-
- As get_filename should only return a source file path there is no
- chance of the path not existing but loading still being possible, so
- ImportError should propagate instead of being turned into returning
- None.
-
- """
- warnings.warn("importlib.abc.PyLoader is deprecated and is "
- "slated for removal in Python 3.4; "
- "use SourceLoader instead. "
- "See the importlib documentation on how to be "
- "compatible with Python 3.1 onwards.",
- DeprecationWarning)
- path = self.source_path(fullname)
- if path is None:
- raise ImportError(name=fullname)
- else:
- return path
-
-
-class PyPycLoader(PyLoader):
-
- """Abstract base class to assist in loading source and bytecode by
- requiring only back-end storage methods to be implemented.
-
- This class has been deprecated! Removal is slated for Python 3.4. Implement
- the SourceLoader ABC instead. If Python 3.1 compatibility is needed, see
- PyLoader.
-
- The methods get_code, get_source, and load_module are implemented for the
- user.
-
- """
-
- def get_filename(self, fullname):
- """Return the source or bytecode file path."""
- path = self.source_path(fullname)
- if path is not None:
- return path
- path = self.bytecode_path(fullname)
- if path is not None:
- return path
- raise ImportError("no source or bytecode path available for "
- "{0!r}".format(fullname), name=fullname)
-
- def get_code(self, fullname):
- """Get a code object from source or bytecode."""
- warnings.warn("importlib.abc.PyPycLoader is deprecated and slated for "
- "removal in Python 3.4; use SourceLoader instead. "
- "If Python 3.1 compatibility is required, see the "
- "latest documentation for PyLoader.",
- DeprecationWarning)
- source_timestamp = self.source_mtime(fullname)
- # Try to use bytecode if it is available.
- bytecode_path = self.bytecode_path(fullname)
- if bytecode_path:
- data = self.get_data(bytecode_path)
- try:
- magic = data[:4]
- if len(magic) < 4:
- raise ImportError(
- "bad magic number in {}".format(fullname),
- name=fullname, path=bytecode_path)
- raw_timestamp = data[4:8]
- if len(raw_timestamp) < 4:
- raise EOFError("bad timestamp in {}".format(fullname))
- pyc_timestamp = _bootstrap._r_long(raw_timestamp)
- raw_source_size = data[8:12]
- if len(raw_source_size) != 4:
- raise EOFError("bad file size in {}".format(fullname))
- # Source size is unused as the ABC does not provide a way to
- # get the size of the source ahead of reading it.
- bytecode = data[12:]
- # Verify that the magic number is valid.
- if imp.get_magic() != magic:
- raise ImportError(
- "bad magic number in {}".format(fullname),
- name=fullname, path=bytecode_path)
- # Verify that the bytecode is not stale (only matters when
- # there is source to fall back on.
- if source_timestamp:
- if pyc_timestamp < source_timestamp:
- raise ImportError("bytecode is stale", name=fullname,
- path=bytecode_path)
- except (ImportError, EOFError):
- # If source is available give it a shot.
- if source_timestamp is not None:
- pass
- else:
- raise
- else:
- # Bytecode seems fine, so try to use it.
- return marshal.loads(bytecode)
- elif source_timestamp is None:
- raise ImportError("no source or bytecode available to create code "
- "object for {0!r}".format(fullname),
- name=fullname)
- # Use the source.
- source_path = self.source_path(fullname)
- if source_path is None:
- message = "a source path must exist to load {0}".format(fullname)
- raise ImportError(message, name=fullname)
- source = self.get_data(source_path)
- code_object = compile(source, source_path, 'exec', dont_inherit=True)
- # Generate bytecode and write it out.
- if not sys.dont_write_bytecode:
- data = bytearray(imp.get_magic())
- data.extend(_bootstrap._w_long(source_timestamp))
- data.extend(_bootstrap._w_long(len(source) & 0xFFFFFFFF))
- data.extend(marshal.dumps(code_object))
- self.write_bytecode(fullname, data)
- return code_object
-
- @abc.abstractmethod
- def source_mtime(self, fullname):
- """Abstract method. Accepts a str filename and returns an int
- modification time for the source of the module."""
- raise NotImplementedError
-
- @abc.abstractmethod
- def bytecode_path(self, fullname):
- """Abstract method. Accepts a str filename and returns the str pathname
- to the bytecode for the module."""
- raise NotImplementedError
-
- @abc.abstractmethod
- def write_bytecode(self, fullname, bytecode):
- """Abstract method. Accepts a str filename and bytes object
- representing the bytecode for the module. Returns a boolean
- representing whether the bytecode was written or not."""
- raise NotImplementedError
diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py
index ff826e4..2e1b2d7 100644
--- a/Lib/importlib/machinery.py
+++ b/Lib/importlib/machinery.py
@@ -5,6 +5,7 @@ import _imp
from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
EXTENSION_SUFFIXES)
+from ._bootstrap import ModuleSpec
from ._bootstrap import BuiltinImporter
from ._bootstrap import FrozenImporter
from ._bootstrap import WindowsRegistryFinder
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 1316437..6d73b1d 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -1,9 +1,18 @@
"""Utility code for constructing importers, etc."""
-from ._bootstrap import module_for_loader
-from ._bootstrap import set_loader
-from ._bootstrap import set_package
+from ._bootstrap import MAGIC_NUMBER
+from ._bootstrap import cache_from_source
+from ._bootstrap import decode_source
+from ._bootstrap import source_from_cache
+from ._bootstrap import spec_from_loader
+from ._bootstrap import spec_from_file_location
from ._bootstrap import _resolve_name
+from ._bootstrap import _find_spec
+
+from contextlib import contextmanager
+import functools
+import sys
+import warnings
def resolve_name(name, package):
@@ -19,3 +28,175 @@ def resolve_name(name, package):
break
level += 1
return _resolve_name(name[level:], package, level)
+
+
+def _find_spec_from_path(name, path=None):
+ """Return the spec for the specified module.
+
+ First, sys.modules is checked to see if the module was already imported. If
+ so, then sys.modules[name].__spec__ is returned. If that happens to be
+ set to None, then ValueError is raised. If the module is not in
+ sys.modules, then sys.meta_path is searched for a suitable spec with the
+ value of 'path' given to the finders. None is returned if no spec could
+ be found.
+
+ Dotted names do not have their parent packages implicitly imported. You will
+ most likely need to explicitly import all parent packages in the proper
+ order for a submodule to get the correct spec.
+
+ """
+ if name not in sys.modules:
+ return _find_spec(name, path)
+ else:
+ module = sys.modules[name]
+ if module is None:
+ return None
+ try:
+ spec = module.__spec__
+ except AttributeError:
+ raise ValueError('{}.__spec__ is not set'.format(name))
+ else:
+ if spec is None:
+ raise ValueError('{}.__spec__ is None'.format(name))
+ return spec
+
+
+def find_spec(name, package=None):
+ """Return the spec for the specified module.
+
+ First, sys.modules is checked to see if the module was already imported. If
+ so, then sys.modules[name].__spec__ is returned. If that happens to be
+ set to None, then ValueError is raised. If the module is not in
+ sys.modules, then sys.meta_path is searched for a suitable spec with the
+ value of 'path' given to the finders. None is returned if no spec could
+ be found.
+
+ If the name is for submodule (contains a dot), the parent module is
+ automatically imported.
+
+ The name and package arguments work the same as importlib.import_module().
+ In other words, relative module names (with leading dots) work.
+
+ """
+ fullname = resolve_name(name, package) if name.startswith('.') else name
+ if fullname not in sys.modules:
+ parent_name = fullname.rpartition('.')[0]
+ if parent_name:
+ # Use builtins.__import__() in case someone replaced it.
+ parent = __import__(parent_name, fromlist=['__path__'])
+ return _find_spec(fullname, parent.__path__)
+ else:
+ return _find_spec(fullname, None)
+ else:
+ module = sys.modules[fullname]
+ if module is None:
+ return None
+ try:
+ spec = module.__spec__
+ except AttributeError:
+ raise ValueError('{}.__spec__ is not set'.format(name))
+ else:
+ if spec is None:
+ raise ValueError('{}.__spec__ is None'.format(name))
+ return spec
+
+
+@contextmanager
+def _module_to_load(name):
+ is_reload = name in sys.modules
+
+ module = sys.modules.get(name)
+ if not is_reload:
+ # This must be done before open() is called as the 'io' module
+ # implicitly imports 'locale' and would otherwise trigger an
+ # infinite loop.
+ module = type(sys)(name)
+ # This must be done before putting the module in sys.modules
+ # (otherwise an optimization shortcut in import.c becomes wrong)
+ module.__initializing__ = True
+ sys.modules[name] = module
+ try:
+ yield module
+ except Exception:
+ if not is_reload:
+ try:
+ del sys.modules[name]
+ except KeyError:
+ pass
+ finally:
+ module.__initializing__ = False
+
+
+def set_package(fxn):
+ """Set __package__ on the returned module.
+
+ This function is deprecated.
+
+ """
+ @functools.wraps(fxn)
+ def set_package_wrapper(*args, **kwargs):
+ warnings.warn('The import system now takes care of this automatically.',
+ DeprecationWarning, stacklevel=2)
+ module = fxn(*args, **kwargs)
+ if getattr(module, '__package__', None) is None:
+ module.__package__ = module.__name__
+ if not hasattr(module, '__path__'):
+ module.__package__ = module.__package__.rpartition('.')[0]
+ return module
+ return set_package_wrapper
+
+
+def set_loader(fxn):
+ """Set __loader__ on the returned module.
+
+ This function is deprecated.
+
+ """
+ @functools.wraps(fxn)
+ def set_loader_wrapper(self, *args, **kwargs):
+ warnings.warn('The import system now takes care of this automatically.',
+ DeprecationWarning, stacklevel=2)
+ module = fxn(self, *args, **kwargs)
+ if getattr(module, '__loader__', None) is None:
+ module.__loader__ = self
+ return module
+ return set_loader_wrapper
+
+
+def module_for_loader(fxn):
+ """Decorator to handle selecting the proper module for loaders.
+
+ The decorated function is passed the module to use instead of the module
+ name. The module passed in to the function is either from sys.modules if
+ it already exists or is a new module. If the module is new, then __name__
+ is set the first argument to the method, __loader__ is set to self, and
+ __package__ is set accordingly (if self.is_package() is defined) will be set
+ before it is passed to the decorated function (if self.is_package() does
+ not work for the module it will be set post-load).
+
+ If an exception is raised and the decorator created the module it is
+ subsequently removed from sys.modules.
+
+ The decorator assumes that the decorated function takes the module name as
+ the second argument.
+
+ """
+ warnings.warn('The import system now takes care of this automatically.',
+ DeprecationWarning, stacklevel=2)
+ @functools.wraps(fxn)
+ def module_for_loader_wrapper(self, fullname, *args, **kwargs):
+ with _module_to_load(fullname) as module:
+ module.__loader__ = self
+ try:
+ is_package = self.is_package(fullname)
+ except (ImportError, AttributeError):
+ pass
+ else:
+ if is_package:
+ module.__package__ = fullname
+ else:
+ module.__package__ = fullname.rpartition('.')[0]
+ # If __package__ was not set above, __import__() will do it later.
+ return fxn(self, module, *args, **kwargs)
+
+ return module_for_loader_wrapper
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 680623d..9f9a600 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -31,7 +31,7 @@ Here are some of the useful functions provided by this module:
__author__ = ('Ka-Ping Yee <ping@lfw.org>',
'Yury Selivanov <yselivanov@sprymix.com>')
-import imp
+import ast
import importlib.machinery
import itertools
import linecache
@@ -39,6 +39,7 @@ import os
import re
import sys
import tokenize
+import token
import types
import warnings
import functools
@@ -268,21 +269,40 @@ def getmembers(object, predicate=None):
else:
mro = ()
results = []
- for key in dir(object):
- # First try to get the value via __dict__. Some descriptors don't
- # like calling their __get__ (see bug #1785).
- for base in mro:
- if key in base.__dict__:
- value = base.__dict__[key]
- break
- else:
- try:
- value = getattr(object, key)
- except AttributeError:
+ processed = set()
+ names = dir(object)
+ # :dd any DynamicClassAttributes to the list of names if object is a class;
+ # this may result in duplicate entries if, for example, a virtual
+ # attribute with the same name as a DynamicClassAttribute exists
+ try:
+ for base in object.__bases__:
+ for k, v in base.__dict__.items():
+ if isinstance(v, types.DynamicClassAttribute):
+ names.append(k)
+ except AttributeError:
+ pass
+ for key in names:
+ # First try to get the value via getattr. Some descriptors don't
+ # like calling their __get__ (see bug #1785), so fall back to
+ # looking in the __dict__.
+ try:
+ value = getattr(object, key)
+ # handle the duplicate key
+ if key in processed:
+ raise AttributeError
+ except AttributeError:
+ for base in mro:
+ if key in base.__dict__:
+ value = base.__dict__[key]
+ break
+ else:
+ # could be a (currently) missing slot member, or a buggy
+ # __dir__; discard and move on
continue
if not predicate or predicate(value):
results.append((key, value))
- results.sort()
+ processed.add(key)
+ results.sort(key=lambda pair: pair[0])
return results
Attribute = namedtuple('Attribute', 'name kind defining_class object')
@@ -299,60 +319,106 @@ def classify_class_attrs(cls):
'class method' created via classmethod()
'static method' created via staticmethod()
'property' created via property()
- 'method' any other flavor of method
+ 'method' any other flavor of method or descriptor
'data' not a method
2. The class which defined this attribute (a class).
- 3. The object as obtained directly from the defining class's
- __dict__, not via getattr. This is especially important for
- data attributes: C.data is just a data object, but
- C.__dict__['data'] may be a data descriptor with additional
- info, like a __doc__ string.
+ 3. The object as obtained by calling getattr; if this fails, or if the
+ resulting object does not live anywhere in the class' mro (including
+ metaclasses) then the object is looked up in the defining class's
+ dict (found by walking the mro).
+
+ If one of the items in dir(cls) is stored in the metaclass it will now
+ be discovered and not have None be listed as the class in which it was
+ defined. Any items whose home class cannot be discovered are skipped.
"""
mro = getmro(cls)
+ metamro = getmro(type(cls)) # for attributes stored in the metaclass
+ metamro = tuple([cls for cls in metamro if cls not in (type, object)])
+ class_bases = (cls,) + mro
+ all_bases = class_bases + metamro
names = dir(cls)
+ # :dd any DynamicClassAttributes to the list of names;
+ # this may result in duplicate entries if, for example, a virtual
+ # attribute with the same name as a DynamicClassAttribute exists.
+ for base in mro:
+ for k, v in base.__dict__.items():
+ if isinstance(v, types.DynamicClassAttribute):
+ names.append(k)
result = []
+ processed = set()
+
for name in names:
# Get the object associated with the name, and where it was defined.
+ # Normal objects will be looked up with both getattr and directly in
+ # its class' dict (in case getattr fails [bug #1785], and also to look
+ # for a docstring).
+ # For DynamicClassAttributes on the second pass we only look in the
+ # class's dict.
+ #
# Getting an obj from the __dict__ sometimes reveals more than
# using getattr. Static and class methods are dramatic examples.
- # Furthermore, some objects may raise an Exception when fetched with
- # getattr(). This is the case with some descriptors (bug #1785).
- # Thus, we only use getattr() as a last resort.
homecls = None
- for base in (cls,) + mro:
+ get_obj = None
+ dict_obj = None
+ if name not in processed:
+ try:
+ if name == '__dict__':
+ raise Exception("__dict__ is special, don't want the proxy")
+ get_obj = getattr(cls, name)
+ except Exception as exc:
+ pass
+ else:
+ homecls = getattr(get_obj, "__objclass__", homecls)
+ if homecls not in class_bases:
+ # if the resulting object does not live somewhere in the
+ # mro, drop it and search the mro manually
+ homecls = None
+ last_cls = None
+ # first look in the classes
+ for srch_cls in class_bases:
+ srch_obj = getattr(srch_cls, name, None)
+ if srch_obj == get_obj:
+ last_cls = srch_cls
+ # then check the metaclasses
+ for srch_cls in metamro:
+ try:
+ srch_obj = srch_cls.__getattr__(cls, name)
+ except AttributeError:
+ continue
+ if srch_obj == get_obj:
+ last_cls = srch_cls
+ if last_cls is not None:
+ homecls = last_cls
+ for base in all_bases:
if name in base.__dict__:
- obj = base.__dict__[name]
- homecls = base
+ dict_obj = base.__dict__[name]
+ if homecls not in metamro:
+ homecls = base
break
- else:
- obj = getattr(cls, name)
- homecls = getattr(obj, "__objclass__", homecls)
-
- # Classify the object.
- if isinstance(obj, staticmethod):
+ if homecls is None:
+ # unable to locate the attribute anywhere, most likely due to
+ # buggy custom __dir__; discard and move on
+ continue
+ obj = get_obj or dict_obj
+ # Classify the object or its descriptor.
+ if isinstance(dict_obj, staticmethod):
kind = "static method"
- elif isinstance(obj, classmethod):
+ obj = dict_obj
+ elif isinstance(dict_obj, classmethod):
kind = "class method"
- elif isinstance(obj, property):
+ obj = dict_obj
+ elif isinstance(dict_obj, property):
kind = "property"
- elif ismethoddescriptor(obj):
+ obj = dict_obj
+ elif isroutine(obj):
kind = "method"
- elif isdatadescriptor(obj):
- kind = "data"
else:
- obj_via_getattr = getattr(cls, name)
- if (isfunction(obj_via_getattr) or
- ismethoddescriptor(obj_via_getattr)):
- kind = "method"
- else:
- kind = "data"
- obj = obj_via_getattr
-
+ kind = "data"
result.append(Attribute(name, kind, homecls, obj))
-
+ processed.add(name)
return result
# ----------------------------------------------------------- class helpers
@@ -361,6 +427,40 @@ def getmro(cls):
"Return tuple of base classes (including cls) in method resolution order."
return cls.__mro__
+# -------------------------------------------------------- function helpers
+
+def unwrap(func, *, stop=None):
+ """Get the object wrapped by *func*.
+
+ Follows the chain of :attr:`__wrapped__` attributes returning the last
+ object in the chain.
+
+ *stop* is an optional callback accepting an object in the wrapper chain
+ as its sole argument that allows the unwrapping to be terminated early if
+ the callback returns a true value. If the callback never returns a true
+ value, the last object in the chain is returned as usual. For example,
+ :func:`signature` uses this to stop unwrapping if any object in the
+ chain has a ``__signature__`` attribute defined.
+
+ :exc:`ValueError` is raised if a cycle is encountered.
+
+ """
+ if stop is None:
+ def _is_wrapper(f):
+ return hasattr(f, '__wrapped__')
+ else:
+ def _is_wrapper(f):
+ return hasattr(f, '__wrapped__') and not stop(f)
+ f = func # remember the original func for error reporting
+ memo = {id(f)} # Memoise by id to tolerate non-hashable objects
+ while _is_wrapper(func):
+ func = func.__wrapped__
+ id_func = id(func)
+ if id_func in memo:
+ raise ValueError('wrapper loop when unwrapping {!r}'.format(f))
+ memo.add(id_func)
+ return func
+
# -------------------------------------------------- source code extraction
def indentsize(line):
"""Return the indent size, in spaces, at the start of a line of text."""
@@ -417,9 +517,10 @@ def getfile(object):
return object.__file__
raise TypeError('{!r} is a built-in module'.format(object))
if isclass(object):
- object = sys.modules.get(object.__module__)
- if hasattr(object, '__file__'):
- return object.__file__
+ if hasattr(object, '__module__'):
+ object = sys.modules.get(object.__module__)
+ if hasattr(object, '__file__'):
+ return object.__file__
raise TypeError('{!r} is a built-in class'.format(object))
if ismethod(object):
object = object.__func__
@@ -440,6 +541,9 @@ def getmoduleinfo(path):
"""Get the module name, suffix, mode, and module type for a given file."""
warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning,
2)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', PendingDeprecationWarning)
+ import imp
filename = os.path.basename(path)
suffixes = [(-len(suffix), suffix, mode, mtype)
for suffix, mode, mtype in imp.get_suffixes()]
@@ -476,7 +580,7 @@ def getsourcefile(object):
if os.path.exists(filename):
return filename
# only return a non-existent filename if the module has a PEP 302 loader
- if hasattr(getmodule(object, filename), '__loader__'):
+ if getattr(getmodule(object, filename), '__loader__', None) is not None:
return filename
# or it is in the linecache
if filename in linecache.cache:
@@ -545,13 +649,13 @@ def findsource(object):
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a list of all the lines
- in the file and the line number indexes a line in that list. An IOError
+ in the file and the line number indexes a line in that list. An OSError
is raised if the source code cannot be retrieved."""
file = getfile(object)
sourcefile = getsourcefile(object)
if not sourcefile and file[:1] + file[-1:] != '<>':
- raise IOError('source code not available')
+ raise OSError('source code not available')
file = sourcefile if sourcefile else file
module = getmodule(object, file)
@@ -560,7 +664,7 @@ def findsource(object):
else:
lines = linecache.getlines(file)
if not lines:
- raise IOError('could not get source code')
+ raise OSError('could not get source code')
if ismodule(object):
return lines, 0
@@ -586,7 +690,7 @@ def findsource(object):
candidates.sort()
return lines, candidates[0][1]
else:
- raise IOError('could not find class definition')
+ raise OSError('could not find class definition')
if ismethod(object):
object = object.__func__
@@ -598,14 +702,14 @@ def findsource(object):
object = object.f_code
if iscode(object):
if not hasattr(object, 'co_firstlineno'):
- raise IOError('could not find function definition')
+ raise OSError('could not find function definition')
lnum = object.co_firstlineno - 1
pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
while lnum > 0:
if pat.match(lines[lnum]): break
lnum = lnum - 1
return lines, lnum
- raise IOError('could not find code object')
+ raise OSError('could not find code object')
def getcomments(object):
"""Get lines of comments immediately preceding an object's source code.
@@ -614,7 +718,7 @@ def getcomments(object):
"""
try:
lines, lnum = findsource(object)
- except (IOError, TypeError):
+ except (OSError, TypeError):
return None
if ismodule(object):
@@ -710,7 +814,7 @@ def getsourcelines(object):
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a list of the lines
corresponding to the object and the line number indicates where in the
- original source file the first line of code was found. An IOError is
+ original source file the first line of code was found. An OSError is
raised if the source code cannot be retrieved."""
lines, lnum = findsource(object)
@@ -722,7 +826,7 @@ def getsource(object):
The argument may be a module, class, method, function, traceback, frame,
or code object. The source code is returned as a single string. An
- IOError is raised if the source code cannot be retrieved."""
+ OSError is raised if the source code cannot be retrieved."""
lines, lnum = getsourcelines(object)
return ''.join(lines)
@@ -831,7 +935,7 @@ FullArgSpec = namedtuple('FullArgSpec',
'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations')
def getfullargspec(func):
- """Get the names and default values of a function's arguments.
+ """Get the names and default values of a callable object's arguments.
A tuple of seven things is returned:
(args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults annotations).
@@ -845,13 +949,78 @@ def getfullargspec(func):
The first four items in the tuple correspond to getargspec().
"""
- if ismethod(func):
- func = func.__func__
- if not isfunction(func):
- raise TypeError('{!r} is not a Python function'.format(func))
- args, varargs, kwonlyargs, varkw = _getfullargs(func.__code__)
- return FullArgSpec(args, varargs, varkw, func.__defaults__,
- kwonlyargs, func.__kwdefaults__, func.__annotations__)
+ try:
+ # Re: `skip_bound_arg=False`
+ #
+ # There is a notable difference in behaviour between getfullargspec
+ # and Signature: the former always returns 'self' parameter for bound
+ # methods, whereas the Signature always shows the actual calling
+ # signature of the passed object.
+ #
+ # To simulate this behaviour, we "unbind" bound methods, to trick
+ # inspect.signature to always return their first parameter ("self",
+ # usually)
+
+ # Re: `follow_wrapper_chains=False`
+ #
+ # getfullargspec() historically ignored __wrapped__ attributes,
+ # so we ensure that remains the case in 3.3+
+
+ sig = _signature_internal(func,
+ follow_wrapper_chains=False,
+ skip_bound_arg=False)
+ except Exception as ex:
+ # Most of the times 'signature' will raise ValueError.
+ # But, it can also raise AttributeError, and, maybe something
+ # else. So to be fully backwards compatible, we catch all
+ # possible exceptions here, and reraise a TypeError.
+ raise TypeError('unsupported callable') from ex
+
+ args = []
+ varargs = None
+ varkw = None
+ kwonlyargs = []
+ defaults = ()
+ annotations = {}
+ defaults = ()
+ kwdefaults = {}
+
+ if sig.return_annotation is not sig.empty:
+ annotations['return'] = sig.return_annotation
+
+ for param in sig.parameters.values():
+ kind = param.kind
+ name = param.name
+
+ if kind is _POSITIONAL_ONLY:
+ args.append(name)
+ elif kind is _POSITIONAL_OR_KEYWORD:
+ args.append(name)
+ if param.default is not param.empty:
+ defaults += (param.default,)
+ elif kind is _VAR_POSITIONAL:
+ varargs = name
+ elif kind is _KEYWORD_ONLY:
+ kwonlyargs.append(name)
+ if param.default is not param.empty:
+ kwdefaults[name] = param.default
+ elif kind is _VAR_KEYWORD:
+ varkw = name
+
+ if param.annotation is not param.empty:
+ annotations[name] = param.annotation
+
+ if not kwdefaults:
+ # compatibility with 'func.__kwdefaults__'
+ kwdefaults = None
+
+ if not defaults:
+ # compatibility with 'func.__defaults__'
+ defaults = None
+
+ return FullArgSpec(args, varargs, varkw, defaults,
+ kwonlyargs, kwdefaults, annotations)
+
ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals')
@@ -956,7 +1125,7 @@ def _missing_arguments(f_name, argnames, pos, values):
elif missing == 2:
s = "{} and {}".format(*names)
else:
- tail = ", {} and {}".format(names[-2:])
+ tail = ", {} and {}".format(*names[-2:])
del names[-2:]
s = ", ".join(names) + tail
raise TypeError("%s() missing %i required %s argument%s: %s" %
@@ -1039,7 +1208,7 @@ def getcallargs(*func_and_positional, **named):
missing = 0
for kwarg in kwonlyargs:
if kwarg not in arg2value:
- if kwarg in kwonlydefaults:
+ if kwonlydefaults and kwarg in kwonlydefaults:
arg2value[kwarg] = kwonlydefaults[kwarg]
else:
missing += 1
@@ -1125,7 +1294,7 @@ def getframeinfo(frame, context=1):
start = lineno - 1 - context//2
try:
lines, lnum = findsource(frame)
- except IOError:
+ except OSError:
lines = index = None
else:
start = max(start, 1)
@@ -1317,13 +1486,15 @@ def getgeneratorlocals(generator):
_WrapperDescriptor = type(type.__call__)
_MethodWrapper = type(all.__call__)
+_ClassMethodWrapper = type(int.__dict__['from_bytes'])
_NonUserDefinedCallables = (_WrapperDescriptor,
_MethodWrapper,
+ _ClassMethodWrapper,
types.BuiltinFunctionType)
-def _get_user_defined_method(cls, method_name):
+def _signature_get_user_defined_method(cls, method_name):
try:
meth = getattr(cls, method_name)
except AttributeError:
@@ -1335,8 +1506,361 @@ def _get_user_defined_method(cls, method_name):
return meth
-def signature(obj):
- '''Get a signature object for the passed callable.'''
+def _signature_get_partial(wrapped_sig, partial, extra_args=()):
+ # Internal helper to calculate how 'wrapped_sig' signature will
+ # look like after applying a 'functools.partial' object (or alike)
+ # on it.
+
+ new_params = OrderedDict(wrapped_sig.parameters.items())
+
+ partial_args = partial.args or ()
+ partial_keywords = partial.keywords or {}
+
+ if extra_args:
+ partial_args = extra_args + partial_args
+
+ try:
+ ba = wrapped_sig.bind_partial(*partial_args, **partial_keywords)
+ except TypeError as ex:
+ msg = 'partial object {!r} has incorrect arguments'.format(partial)
+ raise ValueError(msg) from ex
+
+ for arg_name, arg_value in ba.arguments.items():
+ param = new_params[arg_name]
+ if arg_name in partial_keywords:
+ # We set a new default value, because the following code
+ # is correct:
+ #
+ # >>> def foo(a): print(a)
+ # >>> print(partial(partial(foo, a=10), a=20)())
+ # 20
+ # >>> print(partial(partial(foo, a=10), a=20)(a=30))
+ # 30
+ #
+ # So, with 'partial' objects, passing a keyword argument is
+ # like setting a new default value for the corresponding
+ # parameter
+ #
+ # We also mark this parameter with '_partial_kwarg'
+ # flag. Later, in '_bind', the 'default' value of this
+ # parameter will be added to 'kwargs', to simulate
+ # the 'functools.partial' real call.
+ new_params[arg_name] = param.replace(default=arg_value,
+ _partial_kwarg=True)
+
+ elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
+ not param._partial_kwarg):
+ new_params.pop(arg_name)
+
+ return wrapped_sig.replace(parameters=new_params.values())
+
+
+def _signature_bound_method(sig):
+ # Internal helper to transform signatures for unbound
+ # functions to bound methods
+
+ params = tuple(sig.parameters.values())
+
+ if not params or params[0].kind in (_VAR_KEYWORD, _KEYWORD_ONLY):
+ raise ValueError('invalid method signature')
+
+ kind = params[0].kind
+ if kind in (_POSITIONAL_OR_KEYWORD, _POSITIONAL_ONLY):
+ # Drop first parameter:
+ # '(p1, p2[, ...])' -> '(p2[, ...])'
+ params = params[1:]
+ else:
+ if kind is not _VAR_POSITIONAL:
+ # Unless we add a new parameter type we never
+ # get here
+ raise ValueError('invalid argument type')
+ # It's a var-positional parameter.
+ # Do nothing. '(*args[, ...])' -> '(*args[, ...])'
+
+ return sig.replace(parameters=params)
+
+
+def _signature_is_builtin(obj):
+ # Internal helper to test if `obj` is a callable that might
+ # support Argument Clinic's __text_signature__ protocol.
+ return (isbuiltin(obj) or
+ ismethoddescriptor(obj) or
+ isinstance(obj, _NonUserDefinedCallables) or
+ # Can't test 'isinstance(type)' here, as it would
+ # also be True for regular python classes
+ obj in (type, object))
+
+
+def _signature_is_functionlike(obj):
+ # Internal helper to test if `obj` is a duck type of FunctionType.
+ # A good example of such objects are functions compiled with
+ # Cython, which have all attributes that a pure Python function
+ # would have, but have their code statically compiled.
+
+ if not callable(obj) or isclass(obj):
+ # All function-like objects are obviously callables,
+ # and not classes.
+ return False
+
+ name = getattr(obj, '__name__', None)
+ code = getattr(obj, '__code__', None)
+ defaults = getattr(obj, '__defaults__', _void) # Important to use _void ...
+ kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here
+ annotations = getattr(obj, '__annotations__', None)
+
+ return (isinstance(code, types.CodeType) and
+ isinstance(name, str) and
+ (defaults is None or isinstance(defaults, tuple)) and
+ (kwdefaults is None or isinstance(kwdefaults, dict)) and
+ isinstance(annotations, dict))
+
+
+def _signature_get_bound_param(spec):
+ # Internal helper to get first parameter name from a
+ # __text_signature__ of a builtin method, which should
+ # be in the following format: '($param1, ...)'.
+ # Assumptions are that the first argument won't have
+ # a default value or an annotation.
+
+ assert spec.startswith('($')
+
+ pos = spec.find(',')
+ if pos == -1:
+ pos = spec.find(')')
+
+ cpos = spec.find(':')
+ assert cpos == -1 or cpos > pos
+
+ cpos = spec.find('=')
+ assert cpos == -1 or cpos > pos
+
+ return spec[2:pos]
+
+
+def _signature_strip_non_python_syntax(signature):
+ """
+ Takes a signature in Argument Clinic's extended signature format.
+ Returns a tuple of three things:
+ * that signature re-rendered in standard Python syntax,
+ * the index of the "self" parameter (generally 0), or None if
+ the function does not have a "self" parameter, and
+ * the index of the last "positional only" parameter,
+ or None if the signature has no positional-only parameters.
+ """
+
+ if not signature:
+ return signature, None, None
+
+ self_parameter = None
+ last_positional_only = None
+
+ lines = [l.encode('ascii') for l in signature.split('\n')]
+ generator = iter(lines).__next__
+ token_stream = tokenize.tokenize(generator)
+
+ delayed_comma = False
+ skip_next_comma = False
+ text = []
+ add = text.append
+
+ current_parameter = 0
+ OP = token.OP
+ ERRORTOKEN = token.ERRORTOKEN
+
+ # token stream always starts with ENCODING token, skip it
+ t = next(token_stream)
+ assert t.type == tokenize.ENCODING
+
+ for t in token_stream:
+ type, string = t.type, t.string
+
+ if type == OP:
+ if string == ',':
+ if skip_next_comma:
+ skip_next_comma = False
+ else:
+ assert not delayed_comma
+ delayed_comma = True
+ current_parameter += 1
+ continue
+
+ if string == '/':
+ assert not skip_next_comma
+ assert last_positional_only is None
+ skip_next_comma = True
+ last_positional_only = current_parameter - 1
+ continue
+
+ if (type == ERRORTOKEN) and (string == '$'):
+ assert self_parameter is None
+ self_parameter = current_parameter
+ continue
+
+ if delayed_comma:
+ delayed_comma = False
+ if not ((type == OP) and (string == ')')):
+ add(', ')
+ add(string)
+ if (string == ','):
+ add(' ')
+ clean_signature = ''.join(text)
+ return clean_signature, self_parameter, last_positional_only
+
+
+def _signature_fromstr(cls, obj, s, skip_bound_arg=True):
+ # Internal helper to parse content of '__text_signature__'
+ # and return a Signature based on it
+ Parameter = cls._parameter_cls
+
+ clean_signature, self_parameter, last_positional_only = \
+ _signature_strip_non_python_syntax(s)
+
+ program = "def foo" + clean_signature + ": pass"
+
+ try:
+ module = ast.parse(program)
+ except SyntaxError:
+ module = None
+
+ if not isinstance(module, ast.Module):
+ raise ValueError("{!r} builtin has invalid signature".format(obj))
+
+ f = module.body[0]
+
+ parameters = []
+ empty = Parameter.empty
+ invalid = object()
+
+ module = None
+ module_dict = {}
+ module_name = getattr(obj, '__module__', None)
+ if module_name:
+ module = sys.modules.get(module_name, None)
+ if module:
+ module_dict = module.__dict__
+ sys_module_dict = sys.modules
+
+ def parse_name(node):
+ assert isinstance(node, ast.arg)
+ if node.annotation != None:
+ raise ValueError("Annotations are not currently supported")
+ return node.arg
+
+ def wrap_value(s):
+ try:
+ value = eval(s, module_dict)
+ except NameError:
+ try:
+ value = eval(s, sys_module_dict)
+ except NameError:
+ raise RuntimeError()
+
+ if isinstance(value, str):
+ return ast.Str(value)
+ if isinstance(value, (int, float)):
+ return ast.Num(value)
+ if isinstance(value, bytes):
+ return ast.Bytes(value)
+ if value in (True, False, None):
+ return ast.NameConstant(value)
+ raise RuntimeError()
+
+ class RewriteSymbolics(ast.NodeTransformer):
+ def visit_Attribute(self, node):
+ a = []
+ n = node
+ while isinstance(n, ast.Attribute):
+ a.append(n.attr)
+ n = n.value
+ if not isinstance(n, ast.Name):
+ raise RuntimeError()
+ a.append(n.id)
+ value = ".".join(reversed(a))
+ return wrap_value(value)
+
+ def visit_Name(self, node):
+ if not isinstance(node.ctx, ast.Load):
+ raise ValueError()
+ return wrap_value(node.id)
+
+ def p(name_node, default_node, default=empty):
+ name = parse_name(name_node)
+ if name is invalid:
+ return None
+ if default_node and default_node is not _empty:
+ try:
+ default_node = RewriteSymbolics().visit(default_node)
+ o = ast.literal_eval(default_node)
+ except ValueError:
+ o = invalid
+ if o is invalid:
+ return None
+ default = o if o is not invalid else default
+ parameters.append(Parameter(name, kind, default=default, annotation=empty))
+
+ # non-keyword-only parameters
+ args = reversed(f.args.args)
+ defaults = reversed(f.args.defaults)
+ iter = itertools.zip_longest(args, defaults, fillvalue=None)
+ if last_positional_only is not None:
+ kind = Parameter.POSITIONAL_ONLY
+ else:
+ kind = Parameter.POSITIONAL_OR_KEYWORD
+ for i, (name, default) in enumerate(reversed(list(iter))):
+ p(name, default)
+ if i == last_positional_only:
+ kind = Parameter.POSITIONAL_OR_KEYWORD
+
+ # *args
+ if f.args.vararg:
+ kind = Parameter.VAR_POSITIONAL
+ p(f.args.vararg, empty)
+
+ # keyword-only arguments
+ kind = Parameter.KEYWORD_ONLY
+ for name, default in zip(f.args.kwonlyargs, f.args.kw_defaults):
+ p(name, default)
+
+ # **kwargs
+ if f.args.kwarg:
+ kind = Parameter.VAR_KEYWORD
+ p(f.args.kwarg, empty)
+
+ if self_parameter is not None:
+ # Possibly strip the bound argument:
+ # - We *always* strip first bound argument if
+ # it is a module.
+ # - We don't strip first bound argument if
+ # skip_bound_arg is False.
+ assert parameters
+ _self = getattr(obj, '__self__', None)
+ self_isbound = _self is not None
+ self_ismodule = ismodule(_self)
+ if self_isbound and (self_ismodule or skip_bound_arg):
+ parameters.pop(0)
+ else:
+ # for builtins, self parameter is always positional-only!
+ p = parameters[0].replace(kind=Parameter.POSITIONAL_ONLY)
+ parameters[0] = p
+
+ return cls(parameters, return_annotation=cls.empty)
+
+
+def _signature_from_builtin(cls, func, skip_bound_arg=True):
+ # Internal helper function to get signature for
+ # builtin callables
+ if not _signature_is_builtin(func):
+ raise TypeError("{!r} is not a Python builtin "
+ "function".format(func))
+
+ s = getattr(func, "__text_signature__", None)
+ if not s:
+ raise ValueError("no signature found for builtin {!r}".format(func))
+
+ return _signature_fromstr(cls, func, s, skip_bound_arg)
+
+
+def _signature_internal(obj, follow_wrapper_chains=True, skip_bound_arg=True):
if not callable(obj):
raise TypeError('{!r} is not a callable object'.format(obj))
@@ -1344,8 +1868,17 @@ def signature(obj):
if isinstance(obj, types.MethodType):
# In this case we skip the first parameter of the underlying
# function (usually `self` or `cls`).
- sig = signature(obj.__func__)
- return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+ sig = _signature_internal(obj.__func__,
+ follow_wrapper_chains,
+ skip_bound_arg)
+ if skip_bound_arg:
+ return _signature_bound_method(sig)
+ else:
+ return sig
+
+ # Was this function wrapped by a decorator?
+ if follow_wrapper_chains:
+ obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__")))
try:
sig = obj.__signature__
@@ -1356,57 +1889,42 @@ def signature(obj):
return sig
try:
- # Was this function wrapped by a decorator?
- wrapped = obj.__wrapped__
+ partialmethod = obj._partialmethod
except AttributeError:
pass
else:
- return signature(wrapped)
-
- if isinstance(obj, types.FunctionType):
+ if isinstance(partialmethod, functools.partialmethod):
+ # Unbound partialmethod (see functools.partialmethod)
+ # This means, that we need to calculate the signature
+ # as if it's a regular partial object, but taking into
+ # account that the first positional argument
+ # (usually `self`, or `cls`) will not be passed
+ # automatically (as for boundmethods)
+
+ wrapped_sig = _signature_internal(partialmethod.func,
+ follow_wrapper_chains,
+ skip_bound_arg)
+ sig = _signature_get_partial(wrapped_sig, partialmethod, (None,))
+
+ first_wrapped_param = tuple(wrapped_sig.parameters.values())[0]
+ new_params = (first_wrapped_param,) + tuple(sig.parameters.values())
+
+ return sig.replace(parameters=new_params)
+
+ if isfunction(obj) or _signature_is_functionlike(obj):
+ # If it's a pure Python function, or an object that is duck type
+ # of a Python function (Cython functions, for instance), then:
return Signature.from_function(obj)
- if isinstance(obj, functools.partial):
- sig = signature(obj.func)
-
- new_params = OrderedDict(sig.parameters.items())
+ if _signature_is_builtin(obj):
+ return _signature_from_builtin(Signature, obj,
+ skip_bound_arg=skip_bound_arg)
- partial_args = obj.args or ()
- partial_keywords = obj.keywords or {}
- try:
- ba = sig.bind_partial(*partial_args, **partial_keywords)
- except TypeError as ex:
- msg = 'partial object {!r} has incorrect arguments'.format(obj)
- raise ValueError(msg) from ex
-
- for arg_name, arg_value in ba.arguments.items():
- param = new_params[arg_name]
- if arg_name in partial_keywords:
- # We set a new default value, because the following code
- # is correct:
- #
- # >>> def foo(a): print(a)
- # >>> print(partial(partial(foo, a=10), a=20)())
- # 20
- # >>> print(partial(partial(foo, a=10), a=20)(a=30))
- # 30
- #
- # So, with 'partial' objects, passing a keyword argument is
- # like setting a new default value for the corresponding
- # parameter
- #
- # We also mark this parameter with '_partial_kwarg'
- # flag. Later, in '_bind', the 'default' value of this
- # parameter will be added to 'kwargs', to simulate
- # the 'functools.partial' real call.
- new_params[arg_name] = param.replace(default=arg_value,
- _partial_kwarg=True)
-
- elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and
- not param._partial_kwarg):
- new_params.pop(arg_name)
-
- return sig.replace(parameters=new_params.values())
+ if isinstance(obj, functools.partial):
+ wrapped_sig = _signature_internal(obj.func,
+ follow_wrapper_chains,
+ skip_bound_arg)
+ return _signature_get_partial(wrapped_sig, obj)
sig = None
if isinstance(obj, type):
@@ -1414,32 +1932,80 @@ def signature(obj):
# First, let's see if it has an overloaded __call__ defined
# in its metaclass
- call = _get_user_defined_method(type(obj), '__call__')
+ call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
- sig = signature(call)
+ sig = _signature_internal(call,
+ follow_wrapper_chains,
+ skip_bound_arg)
else:
# Now we check if the 'obj' class has a '__new__' method
- new = _get_user_defined_method(obj, '__new__')
+ new = _signature_get_user_defined_method(obj, '__new__')
if new is not None:
- sig = signature(new)
+ sig = _signature_internal(new,
+ follow_wrapper_chains,
+ skip_bound_arg)
else:
# Finally, we should have at least __init__ implemented
- init = _get_user_defined_method(obj, '__init__')
+ init = _signature_get_user_defined_method(obj, '__init__')
if init is not None:
- sig = signature(init)
+ sig = _signature_internal(init,
+ follow_wrapper_chains,
+ skip_bound_arg)
+
+ if sig is None:
+ # At this point we know, that `obj` is a class, with no user-
+ # defined '__init__', '__new__', or class-level '__call__'
+
+ for base in obj.__mro__[:-1]:
+ # Since '__text_signature__' is implemented as a
+ # descriptor that extracts text signature from the
+ # class docstring, if 'obj' is derived from a builtin
+ # class, its own '__text_signature__' may be 'None'.
+ # Therefore, we go through the MRO (except the last
+ # class in there, which is 'object') to find the first
+ # class with non-empty text signature.
+ try:
+ text_sig = base.__text_signature__
+ except AttributeError:
+ pass
+ else:
+ if text_sig:
+ # If 'obj' class has a __text_signature__ attribute:
+ # return a signature based on it
+ return _signature_fromstr(Signature, obj, text_sig)
+
+ # No '__text_signature__' was found for the 'obj' class.
+ # Last option is to check if its '__init__' is
+ # object.__init__ or type.__init__.
+ if type not in obj.__mro__:
+ # We have a class (not metaclass), but no user-defined
+ # __init__ or __new__ for it
+ if obj.__init__ is object.__init__:
+ # Return a signature of 'object' builtin.
+ return signature(object)
+
elif not isinstance(obj, _NonUserDefinedCallables):
# An object with __call__
# We also check that the 'obj' is not an instance of
# _WrapperDescriptor or _MethodWrapper to avoid
# infinite recursion (and even potential segfault)
- call = _get_user_defined_method(type(obj), '__call__')
+ call = _signature_get_user_defined_method(type(obj), '__call__')
if call is not None:
- sig = signature(call)
+ try:
+ sig = _signature_internal(call,
+ follow_wrapper_chains,
+ skip_bound_arg)
+ except ValueError as ex:
+ msg = 'no signature found for {!r}'.format(obj)
+ raise ValueError(msg) from ex
if sig is not None:
# For classes and objects we skip the first parameter of their
# __call__, __new__, or __init__ methods
- return sig.replace(parameters=tuple(sig.parameters.values())[1:])
+ if skip_bound_arg:
+ return _signature_bound_method(sig)
+ else:
+ return sig
if isinstance(obj, types.BuiltinFunctionType):
# Raise a nicer error message for builtins
@@ -1448,6 +2014,10 @@ def signature(obj):
raise ValueError('callable {!r} is not supported by signature'.format(obj))
+def signature(obj):
+ '''Get a signature object for the passed callable.'''
+ return _signature_internal(obj)
+
class _void:
'''A private marker - used in Parameter & Signature'''
@@ -1486,10 +2056,12 @@ class Parameter:
The name of the parameter as a string.
* default : object
The default value for the parameter if specified. If the
- parameter has no default value, this attribute is not set.
+ parameter has no default value, this attribute is set to
+ `Parameter.empty`.
* annotation
The annotation for the parameter if specified. If the
- parameter has no annotation, this attribute is not set.
+ parameter has no annotation, this attribute is set to
+ `Parameter.empty`.
* kind : str
Describes how argument values are bound to the parameter.
Possible values: `Parameter.POSITIONAL_ONLY`,
@@ -1522,17 +2094,16 @@ class Parameter:
self._default = default
self._annotation = annotation
- if name is None:
- if kind != _POSITIONAL_ONLY:
- raise ValueError("None is not a valid name for a "
- "non-positional-only parameter")
- self._name = name
- else:
- name = str(name)
- if kind != _POSITIONAL_ONLY and not name.isidentifier():
- msg = '{!r} is not a valid parameter name'.format(name)
- raise ValueError(msg)
- self._name = name
+ if name is _empty:
+ raise ValueError('name is a required attribute for Parameter')
+
+ if not isinstance(name, str):
+ raise TypeError("name must be a str, not a {!r}".format(name))
+
+ if not name.isidentifier():
+ raise ValueError('{!r} is not a valid parameter name'.format(name))
+
+ self._name = name
self._partial_kwarg = _partial_kwarg
@@ -1576,12 +2147,7 @@ class Parameter:
def __str__(self):
kind = self.kind
-
formatted = self._name
- if kind == _POSITIONAL_ONLY:
- if formatted is None:
- formatted = ''
- formatted = '<{}>'.format(formatted)
# Add annotation and default value
if self._annotation is not _empty:
@@ -1603,6 +2169,17 @@ class Parameter:
id(self), self.name)
def __eq__(self, other):
+ # NB: We deliberately do not compare '_partial_kwarg' attributes
+ # here. Imagine we have a following situation:
+ #
+ # def foo(a, b=1): pass
+ # def bar(a, b): pass
+ # bar2 = functools.partial(bar, b=1)
+ #
+ # For the above scenario, signatures for `foo` and `bar2` should
+ # be equal. '_partial_kwarg' attribute is an internal flag, to
+ # distinguish between keyword parameters with defaults and
+ # keyword parameters which got their defaults from functools.partial
return (issubclass(other.__class__, Parameter) and
self._name == other._name and
self._kind == other._kind and
@@ -1720,7 +2297,7 @@ class Signature:
* return_annotation : object
The annotation for the return type of the function if specified.
If the function has no annotation for its return type, this
- attribute is not set.
+ attribute is set to `Signature.empty`.
* bind(*args, **kwargs) -> BoundArguments
Creates a mapping from positional and keyword arguments to
parameters.
@@ -1748,24 +2325,41 @@ class Signature:
if __validate_parameters__:
params = OrderedDict()
top_kind = _POSITIONAL_ONLY
+ kind_defaults = False
for idx, param in enumerate(parameters):
kind = param.kind
+ name = param.name
+
if kind < top_kind:
msg = 'wrong parameter order: {} before {}'
- msg = msg.format(top_kind, param.kind)
+ msg = msg.format(top_kind, kind)
raise ValueError(msg)
- else:
+ elif kind > top_kind:
+ kind_defaults = False
top_kind = kind
- name = param.name
- if name is None:
- name = str(idx)
- param = param.replace(name=name)
+ if (kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD) and
+ not param._partial_kwarg):
+ # If we have a positional-only or positional-or-keyword
+ # parameter, that does not have its default value set
+ # by 'functools.partial' or other "partial" signature:
+ if param.default is _empty:
+ if kind_defaults:
+ # No default for this parameter, but the
+ # previous parameter of the same kind had
+ # a default
+ msg = 'non-default argument follows default ' \
+ 'argument'
+ raise ValueError(msg)
+ else:
+ # There is a default for this parameter.
+ kind_defaults = True
if name in params:
msg = 'duplicate parameter name: {!r}'.format(name)
raise ValueError(msg)
+
params[name] = param
else:
params = OrderedDict(((param.name, param)
@@ -1778,8 +2372,14 @@ class Signature:
def from_function(cls, func):
'''Constructs Signature for the given python function'''
- if not isinstance(func, types.FunctionType):
- raise TypeError('{!r} is not a Python function'.format(func))
+ is_duck_function = False
+ if not isfunction(func):
+ if _signature_is_functionlike(func):
+ is_duck_function = True
+ else:
+ # If it's not a pure Python function, and not a duck type
+ # of pure function:
+ raise TypeError('{!r} is not a Python function'.format(func))
Parameter = cls._parameter_cls
@@ -1816,7 +2416,7 @@ class Signature:
default=defaults[offset]))
# *args
- if func_code.co_flags & 0x04:
+ if func_code.co_flags & CO_VARARGS:
name = arg_names[pos_count + keyword_only_count]
annotation = annotations.get(name, _empty)
parameters.append(Parameter(name, annotation=annotation,
@@ -1833,9 +2433,9 @@ class Signature:
kind=_KEYWORD_ONLY,
default=default))
# **kwargs
- if func_code.co_flags & 0x08:
+ if func_code.co_flags & CO_VARKEYWORDS:
index = pos_count + keyword_only_count
- if func_code.co_flags & 0x04:
+ if func_code.co_flags & CO_VARARGS:
index += 1
name = arg_names[index]
@@ -1843,9 +2443,15 @@ class Signature:
parameters.append(Parameter(name, annotation=annotation,
kind=_VAR_KEYWORD))
+ # Is 'func' is a pure Python function - don't validate the
+ # parameters list (for correct order and defaults), it should be OK.
return cls(parameters,
return_annotation=annotations.get('return', _empty),
- __validate_parameters__=False)
+ __validate_parameters__=is_duck_function)
+
+ @classmethod
+ def from_builtin(cls, func):
+ return _signature_from_builtin(cls, func)
@property
def parameters(self):
@@ -1955,6 +2561,8 @@ class Signature:
parameters_ex = (param,)
break
else:
+ # No default, not VAR_KEYWORD, not VAR_POSITIONAL,
+ # not in `kwargs`
if partial:
parameters_ex = (param,)
break
@@ -1993,19 +2601,17 @@ class Signature:
# keyword arguments
kwargs_param = None
for param in itertools.chain(parameters_ex, parameters):
- if param.kind == _POSITIONAL_ONLY:
- # This should never happen in case of a properly built
- # Signature object (but let's have this check here
- # to ensure correct behaviour just in case)
- raise TypeError('{arg!r} parameter is positional only, '
- 'but was passed as a keyword'. \
- format(arg=param.name))
-
if param.kind == _VAR_KEYWORD:
# Memorize that we have a '**kwargs'-like parameter
kwargs_param = param
continue
+ if param.kind == _VAR_POSITIONAL:
+ # Named arguments don't refer to '*args'-like parameters.
+ # We only arrive here if the positional arguments ended
+ # before reaching the last parameter before *args.
+ continue
+
param_name = param.name
try:
arg_val = kwargs.pop(param_name)
@@ -2020,6 +2626,14 @@ class Signature:
format(arg=param_name)) from None
else:
+ if param.kind == _POSITIONAL_ONLY:
+ # This should never happen in case of a properly built
+ # Signature object (but let's have this check here
+ # to ensure correct behaviour just in case)
+ raise TypeError('{arg!r} parameter is positional only, '
+ 'but was passed as a keyword'. \
+ format(arg=param.name))
+
arguments[param_name] = arg_val
if kwargs:
@@ -2031,27 +2645,37 @@ class Signature:
return self._bound_arguments_cls(self, arguments)
- def bind(__bind_self, *args, **kwargs):
+ def bind(*args, **kwargs):
'''Get a BoundArguments object, that maps the passed `args`
and `kwargs` to the function's signature. Raises `TypeError`
if the passed arguments can not be bound.
'''
- return __bind_self._bind(args, kwargs)
+ return args[0]._bind(args[1:], kwargs)
- def bind_partial(__bind_self, *args, **kwargs):
+ def bind_partial(*args, **kwargs):
'''Get a BoundArguments object, that partially maps the
passed `args` and `kwargs` to the function's signature.
Raises `TypeError` if the passed arguments can not be bound.
'''
- return __bind_self._bind(args, kwargs, partial=True)
+ return args[0]._bind(args[1:], kwargs, partial=True)
def __str__(self):
result = []
+ render_pos_only_separator = False
render_kw_only_separator = True
- for idx, param in enumerate(self.parameters.values()):
+ for param in self.parameters.values():
formatted = str(param)
kind = param.kind
+
+ if kind == _POSITIONAL_ONLY:
+ render_pos_only_separator = True
+ elif render_pos_only_separator:
+ # It's not a positional-only parameter, and the flag
+ # is set to 'True' (there were pos-only params before.)
+ result.append('/')
+ render_pos_only_separator = False
+
if kind == _VAR_POSITIONAL:
# OK, we have an '*args'-like parameter, so we won't need
# a '*' to separate keyword-only arguments
@@ -2067,6 +2691,11 @@ class Signature:
result.append(formatted)
+ if render_pos_only_separator:
+ # There were only positional-only parameters, hence the
+ # flag was not reset to 'False'
+ result.append('/')
+
rendered = '({})'.format(', '.join(result))
if self.return_annotation is not _empty:
@@ -2074,3 +2703,64 @@ class Signature:
rendered += ' -> {}'.format(anno)
return rendered
+
+def _main():
+ """ Logic for inspecting an object given at command line """
+ import argparse
+ import importlib
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ 'object',
+ help="The object to be analysed. "
+ "It supports the 'module:qualname' syntax")
+ parser.add_argument(
+ '-d', '--details', action='store_true',
+ help='Display info about the module rather than its source code')
+
+ args = parser.parse_args()
+
+ target = args.object
+ mod_name, has_attrs, attrs = target.partition(":")
+ try:
+ obj = module = importlib.import_module(mod_name)
+ except Exception as exc:
+ msg = "Failed to import {} ({}: {})".format(mod_name,
+ type(exc).__name__,
+ exc)
+ print(msg, file=sys.stderr)
+ exit(2)
+
+ if has_attrs:
+ parts = attrs.split(".")
+ obj = module
+ for part in parts:
+ obj = getattr(obj, part)
+
+ if module.__name__ in sys.builtin_module_names:
+ print("Can't get info for builtin modules.", file=sys.stderr)
+ exit(1)
+
+ if args.details:
+ print('Target: {}'.format(target))
+ print('Origin: {}'.format(getsourcefile(module)))
+ print('Cached: {}'.format(module.__cached__))
+ if obj is module:
+ print('Loader: {}'.format(repr(module.__loader__)))
+ if hasattr(module, '__path__'):
+ print('Submodule search path: {}'.format(module.__path__))
+ else:
+ try:
+ __, lineno = findsource(obj)
+ except Exception:
+ pass
+ else:
+ print('Line: {}'.format(lineno))
+
+ print('\n')
+ else:
+ print(getsource(obj))
+
+
+if __name__ == "__main__":
+ _main()
diff --git a/Lib/io.py b/Lib/io.py
index bda4def..39878b8 100644
--- a/Lib/io.py
+++ b/Lib/io.py
@@ -4,7 +4,7 @@ builtin open function is defined in this module.
At the top of the I/O hierarchy is the abstract base class IOBase. It
defines the basic interface to a stream. Note, however, that there is no
separation between reading and writing to streams; implementations are
-allowed to raise an IOError if they do not support a given operation.
+allowed to raise an OSError if they do not support a given operation.
Extending IOBase is RawIOBase which deals simply with the reading and
writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index ecf3f44..54df39a 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -1030,13 +1030,25 @@ class _BaseNetwork(_IPAddressBase):
"""Test if this address is allocated for private networks.
Returns:
- A boolean, True if the address is reserved per RFC 4193.
+ A boolean, True if the address is reserved per
+ iana-ipv4-special-registry or iana-ipv6-special-registry.
"""
return (self.network_address.is_private and
self.broadcast_address.is_private)
@property
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, True if the address is not reserved per
+ iana-ipv4-special-registry or iana-ipv6-special-registry.
+
+ """
+ return not self.is_private
+
+ @property
def is_unspecified(self):
"""Test if the address is unspecified.
@@ -1276,19 +1288,30 @@ class IPv4Address(_BaseV4, _BaseAddress):
return self in reserved_network
@property
+ @functools.lru_cache()
def is_private(self):
"""Test if this address is allocated for private networks.
Returns:
- A boolean, True if the address is reserved per RFC 1918.
+ A boolean, True if the address is reserved per
+ iana-ipv4-special-registry.
"""
- private_10 = IPv4Network('10.0.0.0/8')
- private_172 = IPv4Network('172.16.0.0/12')
- private_192 = IPv4Network('192.168.0.0/16')
- return (self in private_10 or
- self in private_172 or
- self in private_192)
+ return (self in IPv4Network('0.0.0.0/8') or
+ self in IPv4Network('10.0.0.0/8') or
+ self in IPv4Network('127.0.0.0/8') or
+ self in IPv4Network('169.254.0.0/16') or
+ self in IPv4Network('172.16.0.0/12') or
+ self in IPv4Network('192.0.0.0/29') or
+ self in IPv4Network('192.0.0.170/31') or
+ self in IPv4Network('192.0.2.0/24') or
+ self in IPv4Network('192.168.0.0/16') or
+ self in IPv4Network('198.18.0.0/15') or
+ self in IPv4Network('198.51.100.0/24') or
+ self in IPv4Network('203.0.113.0/24') or
+ self in IPv4Network('240.0.0.0/4') or
+ self in IPv4Network('255.255.255.255/32'))
+
@property
def is_multicast(self):
@@ -1504,6 +1527,21 @@ class IPv4Network(_BaseV4, _BaseNetwork):
if self._prefixlen == (self._max_prefixlen - 1):
self.hosts = self.__iter__
+ @property
+ @functools.lru_cache()
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, True if the address is not reserved per
+ iana-ipv4-special-registry.
+
+ """
+ return (not (self.network_address in IPv4Network('100.64.0.0/10') and
+ self.broadcast_address in IPv4Network('100.64.0.0/10')) and
+ not self.is_private)
+
+
class _BaseV6:
@@ -1860,15 +1898,36 @@ class IPv6Address(_BaseV6, _BaseAddress):
return self in sitelocal_network
@property
+ @functools.lru_cache()
def is_private(self):
"""Test if this address is allocated for private networks.
Returns:
- A boolean, True if the address is reserved per RFC 4193.
+ A boolean, True if the address is reserved per
+ iana-ipv6-special-registry.
"""
- private_network = IPv6Network('fc00::/7')
- return self in private_network
+ return (self in IPv6Network('::1/128') or
+ self in IPv6Network('::/128') or
+ self in IPv6Network('::ffff:0:0/96') or
+ self in IPv6Network('100::/64') or
+ self in IPv6Network('2001::/23') or
+ self in IPv6Network('2001:2::/48') or
+ self in IPv6Network('2001:db8::/32') or
+ self in IPv6Network('2001:10::/28') or
+ self in IPv6Network('fc00::/7') or
+ self in IPv6Network('fe80::/10'))
+
+ @property
+ def is_global(self):
+ """Test if this address is allocated for public networks.
+
+ Returns:
+ A boolean, true if the address is not reserved per
+ iana-ipv6-special-registry.
+
+ """
+ return not self.is_private
@property
def is_unspecified(self):
@@ -2096,6 +2155,18 @@ class IPv6Network(_BaseV6, _BaseNetwork):
if self._prefixlen == (self._max_prefixlen - 1):
self.hosts = self.__iter__
+ def hosts(self):
+ """Generate Iterator over usable hosts in a network.
+
+ This is like __iter__ except it doesn't return the
+ Subnet-Router anycast address.
+
+ """
+ network = int(self.network_address)
+ broadcast = int(self.broadcast_address)
+ for x in range(network + 1, broadcast + 1):
+ yield self._address_class(x)
+
@property
def is_site_local(self):
"""Test if the address is reserved for site-local.
diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py
index 6d9b30d..94f7d8c 100644
--- a/Lib/json/__init__.py
+++ b/Lib/json/__init__.py
@@ -36,8 +36,7 @@ Compact encoding::
Pretty printing::
>>> import json
- >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True,
- ... indent=4, separators=(',', ': ')))
+ >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4))
{
"4": 5,
"6": 7
@@ -143,13 +142,12 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
- representation. Since the default item separator is ``', '``, the
- output might include trailing whitespace when ``indent`` is specified.
- You can use ``separators=(',', ': ')`` to avoid this.
+ representation.
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
+ If specified, ``separators`` should be an ``(item_separator, key_separator)``
+ tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
+ ``(',', ': ')`` otherwise. To get the most compact JSON representation,
+ you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
@@ -206,13 +204,12 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
If ``indent`` is a non-negative integer, then JSON array elements and
object members will be pretty-printed with that indent level. An indent
level of 0 will only insert newlines. ``None`` is the most compact
- representation. Since the default item separator is ``', '``, the
- output might include trailing whitespace when ``indent`` is specified.
- You can use ``separators=(',', ': ')`` to avoid this.
+ representation.
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
+ If specified, ``separators`` should be an ``(item_separator, key_separator)``
+ tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and
+ ``(',', ': ')`` otherwise. To get the most compact JSON representation,
+ you should specify ``(',', ':')`` to eliminate whitespace.
``default(obj)`` is a function that should return a serializable version
of obj or raise TypeError. The default simply raises TypeError.
@@ -310,6 +307,11 @@ def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
The ``encoding`` argument is ignored and deprecated.
"""
+ if not isinstance(s, str):
+ raise TypeError('the JSON object must be str, not {!r}'.format(
+ s.__class__.__name__))
+ if s.startswith(u'\ufeff'):
+ raise ValueError("Unexpected UTF-8 BOM (decode using utf-8-sig)")
if (cls is None and object_hook is None and
parse_int is None and parse_float is None and
parse_constant is None and object_pairs_hook is None and not kw):
diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py
index 80d3420..59e5f41 100644
--- a/Lib/json/decoder.py
+++ b/Lib/json/decoder.py
@@ -1,9 +1,6 @@
"""Implementation of JSONDecoder
"""
-import binascii
import re
-import sys
-import struct
from json import scanner
try:
@@ -15,14 +12,9 @@ __all__ = ['JSONDecoder']
FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
-def _floatconstants():
- _BYTES = binascii.unhexlify(b'7FF80000000000007FF0000000000000')
- if sys.byteorder != 'big':
- _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
- nan, inf = struct.unpack('dd', _BYTES)
- return nan, inf, -inf
-
-NaN, PosInf, NegInf = _floatconstants()
+NaN = float('nan')
+PosInf = float('inf')
+NegInf = float('-inf')
def linecol(doc, pos):
@@ -195,8 +187,8 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook,
try:
value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
+ except StopIteration as err:
+ raise ValueError(errmsg("Expecting value", s, err.value)) from None
pairs_append((key, value))
try:
nextchar = s[end]
@@ -239,8 +231,8 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
while True:
try:
value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
+ except StopIteration as err:
+ raise ValueError(errmsg("Expecting value", s, err.value)) from None
_append(value)
nextchar = s[end:end + 1]
if nextchar in _ws:
@@ -250,7 +242,7 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
if nextchar == ']':
break
elif nextchar != ',':
- raise ValueError(errmsg("Expecting ',' delimiter", s, end))
+ raise ValueError(errmsg("Expecting ',' delimiter", s, end - 1))
try:
if s[end] in _ws:
end += 1
@@ -365,6 +357,6 @@ class JSONDecoder(object):
"""
try:
obj, end = self.scan_once(s, idx)
- except StopIteration:
- raise ValueError("No JSON object could be decoded")
+ except StopIteration as err:
+ raise ValueError(errmsg("Expecting value", s, err.value)) from None
return obj, end
diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py
index 1d8b20c..0513838 100644
--- a/Lib/json/encoder.py
+++ b/Lib/json/encoder.py
@@ -125,14 +125,12 @@ class JSONEncoder(object):
If indent is a non-negative integer, then JSON array
elements and object members will be pretty-printed with that
indent level. An indent level of 0 will only insert newlines.
- None is the most compact representation. Since the default
- item separator is ', ', the output might include trailing
- whitespace when indent is specified. You can use
- separators=(',', ': ') to avoid this.
+ None is the most compact representation.
- If specified, separators should be a (item_separator, key_separator)
- tuple. The default is (', ', ': '). To get the most compact JSON
- representation you should specify (',', ':') to eliminate whitespace.
+ If specified, separators should be an (item_separator, key_separator)
+ tuple. The default is (', ', ': ') if *indent* is ``None`` and
+ (',', ': ') otherwise. To get the most compact JSON representation,
+ you should specify (',', ':') to eliminate whitespace.
If specified, default is a function that gets called for objects
that can't otherwise be serialized. It should return a JSON encodable
@@ -148,6 +146,8 @@ class JSONEncoder(object):
self.indent = indent
if separators is not None:
self.item_separator, self.key_separator = separators
+ elif indent is not None:
+ self.item_separator = ','
if default is not None:
self.default = default
@@ -175,6 +175,7 @@ class JSONEncoder(object):
def encode(self, o):
"""Return a JSON string representation of a Python data structure.
+ >>> from json.encoder import JSONEncoder
>>> JSONEncoder().encode({"foo": ["bar", "baz"]})
'{"foo": ["bar", "baz"]}'
@@ -298,9 +299,13 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
elif value is False:
yield buf + 'false'
elif isinstance(value, int):
- yield buf + str(value)
+ # Subclasses of int/float may override __str__, but we still
+ # want to encode them as integers/floats in JSON. One example
+ # within the standard library is IntEnum.
+ yield buf + str(int(value))
elif isinstance(value, float):
- yield buf + _floatstr(value)
+ # see comment above for int
+ yield buf + _floatstr(float(value))
else:
yield buf
if isinstance(value, (list, tuple)):
@@ -309,8 +314,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
chunks = _iterencode_dict(value, _current_indent_level)
else:
chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
+ yield from chunks
if newline_indent is not None:
_current_indent_level -= 1
yield '\n' + _indent * _current_indent_level
@@ -347,7 +351,8 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
# JavaScript is weakly typed for these, so it makes sense to
# also allow them. Many encoders seem to do something like this.
elif isinstance(key, float):
- key = _floatstr(key)
+ # see comment for int/float in _make_iterencode
+ key = _floatstr(float(key))
elif key is True:
key = 'true'
elif key is False:
@@ -355,7 +360,8 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
elif key is None:
key = 'null'
elif isinstance(key, int):
- key = str(key)
+ # see comment for int/float in _make_iterencode
+ key = str(int(key))
elif _skipkeys:
continue
else:
@@ -375,9 +381,11 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
elif value is False:
yield 'false'
elif isinstance(value, int):
- yield str(value)
+ # see comment for int/float in _make_iterencode
+ yield str(int(value))
elif isinstance(value, float):
- yield _floatstr(value)
+ # see comment for int/float in _make_iterencode
+ yield _floatstr(float(value))
else:
if isinstance(value, (list, tuple)):
chunks = _iterencode_list(value, _current_indent_level)
@@ -385,8 +393,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
chunks = _iterencode_dict(value, _current_indent_level)
else:
chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
+ yield from chunks
if newline_indent is not None:
_current_indent_level -= 1
yield '\n' + _indent * _current_indent_level
@@ -404,15 +411,15 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
elif o is False:
yield 'false'
elif isinstance(o, int):
- yield str(o)
+ # see comment for int/float in _make_iterencode
+ yield str(int(o))
elif isinstance(o, float):
- yield _floatstr(o)
+ # see comment for int/float in _make_iterencode
+ yield _floatstr(float(o))
elif isinstance(o, (list, tuple)):
- for chunk in _iterencode_list(o, _current_indent_level):
- yield chunk
+ yield from _iterencode_list(o, _current_indent_level)
elif isinstance(o, dict):
- for chunk in _iterencode_dict(o, _current_indent_level):
- yield chunk
+ yield from _iterencode_dict(o, _current_indent_level)
else:
if markers is not None:
markerid = id(o)
@@ -420,8 +427,7 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
raise ValueError("Circular reference detected")
markers[markerid] = o
o = _default(o)
- for chunk in _iterencode(o, _current_indent_level):
- yield chunk
+ yield from _iterencode(o, _current_indent_level)
if markers is not None:
del markers[markerid]
return _iterencode
diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py
index 23eef61..86426cd 100644
--- a/Lib/json/scanner.py
+++ b/Lib/json/scanner.py
@@ -29,7 +29,7 @@ def py_make_scanner(context):
try:
nextchar = string[idx]
except IndexError:
- raise StopIteration
+ raise StopIteration(idx)
if nextchar == '"':
return parse_string(string, idx + 1, strict)
@@ -60,7 +60,7 @@ def py_make_scanner(context):
elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
return parse_constant('-Infinity'), idx + 9
else:
- raise StopIteration
+ raise StopIteration(idx)
def scan_once(string, idx):
try:
diff --git a/Lib/json/tool.py b/Lib/json/tool.py
index ecf9c47..7db4528 100644
--- a/Lib/json/tool.py
+++ b/Lib/json/tool.py
@@ -31,8 +31,7 @@ def main():
except ValueError as e:
raise SystemExit(e)
with outfile:
- json.dump(obj, outfile, sort_keys=True,
- indent=4, separators=(',', ': '))
+ json.dump(obj, outfile, sort_keys=True, indent=4)
outfile.write('\n')
diff --git a/Lib/keyword.py b/Lib/keyword.py
index dad39cc..6e1e882 100755
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -60,6 +60,12 @@ def main():
if len(args) > 1: optfile = args[1]
else: optfile = "Lib/keyword.py"
+ # load the output skeleton from the target, taking care to preserve its
+ # newline convention.
+ with open(optfile, newline='') as fp:
+ format = fp.readlines()
+ nl = format[0][len(format[0].strip()):] if format else '\n'
+
# scan the source file for keywords
with open(iptfile) as fp:
strprog = re.compile('"([^"]+)"')
@@ -68,26 +74,21 @@ def main():
if '{1, "' in line:
match = strprog.search(line)
if match:
- lines.append(" '" + match.group(1) + "',\n")
+ lines.append(" '" + match.group(1) + "'," + nl)
lines.sort()
- # load the output skeleton from the target
- with open(optfile) as fp:
- format = fp.readlines()
-
- # insert the lines of keywords
+ # insert the lines of keywords into the skeleton
try:
- start = format.index("#--start keywords--\n") + 1
- end = format.index("#--end keywords--\n")
+ start = format.index("#--start keywords--" + nl) + 1
+ end = format.index("#--end keywords--" + nl)
format[start:end] = lines
except ValueError:
sys.stderr.write("target does not contain format markers\n")
sys.exit(1)
# write the output file
- fp = open(optfile, 'w')
- fp.write(''.join(format))
- fp.close()
+ with open(optfile, 'w', newline='') as fp:
+ fp.writelines(format)
if __name__ == "__main__":
main()
diff --git a/Lib/lib2to3/btm_utils.py b/Lib/lib2to3/btm_utils.py
index 2276dc9..339750e 100644
--- a/Lib/lib2to3/btm_utils.py
+++ b/Lib/lib2to3/btm_utils.py
@@ -96,8 +96,7 @@ class MinNode(object):
def leaves(self):
"Generator that returns the leaves of the tree"
for child in self.children:
- for x in child.leaves():
- yield x
+ yield from child.leaves()
if not self.children:
yield self
@@ -277,7 +276,6 @@ def rec_test(sequence, test_func):
sub-iterables"""
for x in sequence:
if isinstance(x, (list, tuple)):
- for y in rec_test(x, test_func):
- yield y
+ yield from rec_test(x, test_func)
else:
yield test_func(x)
diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py
index 60d219f..6e259c5 100644
--- a/Lib/lib2to3/fixer_util.py
+++ b/Lib/lib2to3/fixer_util.py
@@ -129,6 +129,29 @@ def FromImport(package_name, name_leafs):
imp = Node(syms.import_from, children)
return imp
+def ImportAndCall(node, results, names):
+ """Returns an import statement and calls a method
+ of the module:
+
+ import module
+ module.name()"""
+ obj = results["obj"].clone()
+ if obj.type == syms.arglist:
+ newarglist = obj.clone()
+ else:
+ newarglist = Node(syms.arglist, [obj.clone()])
+ after = results["after"]
+ if after:
+ after = [n.clone() for n in after]
+ new = Node(syms.power,
+ Attr(Name(names[0]), Name(names[1])) +
+ [Node(syms.trailer,
+ [results["lpar"].clone(),
+ newarglist,
+ results["rpar"].clone()])] + after)
+ new.prefix = node.prefix
+ return new
+
###########################################################
### Determine whether a node represents a given literal
diff --git a/Lib/lib2to3/fixes/fix_asserts.py b/Lib/lib2to3/fixes/fix_asserts.py
new file mode 100644
index 0000000..5bcec88
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_asserts.py
@@ -0,0 +1,34 @@
+"""Fixer that replaces deprecated unittest method names."""
+
+# Author: Ezio Melotti
+
+from ..fixer_base import BaseFix
+from ..fixer_util import Name
+
+NAMES = dict(
+ assert_="assertTrue",
+ assertEquals="assertEqual",
+ assertNotEquals="assertNotEqual",
+ assertAlmostEquals="assertAlmostEqual",
+ assertNotAlmostEquals="assertNotAlmostEqual",
+ assertRegexpMatches="assertRegex",
+ assertRaisesRegexp="assertRaisesRegex",
+ failUnlessEqual="assertEqual",
+ failIfEqual="assertNotEqual",
+ failUnlessAlmostEqual="assertAlmostEqual",
+ failIfAlmostEqual="assertNotAlmostEqual",
+ failUnless="assertTrue",
+ failUnlessRaises="assertRaises",
+ failIf="assertFalse",
+)
+
+
+class FixAsserts(BaseFix):
+
+ PATTERN = """
+ power< any+ trailer< '.' meth=(%s)> any* >
+ """ % '|'.join(map(repr, NAMES))
+
+ def transform(self, node, results):
+ name = results["meth"][0]
+ name.replace(Name(NAMES[str(name)], prefix=name.prefix))
diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py
index 6be11cd..fb2973c 100644
--- a/Lib/lib2to3/fixes/fix_intern.py
+++ b/Lib/lib2to3/fixes/fix_intern.py
@@ -6,9 +6,8 @@
intern(s) -> sys.intern(s)"""
# Local imports
-from .. import pytree
from .. import fixer_base
-from ..fixer_util import Name, Attr, touch_import
+from ..fixer_util import ImportAndCall, touch_import
class FixIntern(fixer_base.BaseFix):
@@ -26,21 +25,7 @@ class FixIntern(fixer_base.BaseFix):
"""
def transform(self, node, results):
- syms = self.syms
- obj = results["obj"].clone()
- if obj.type == syms.arglist:
- newarglist = obj.clone()
- else:
- newarglist = pytree.Node(syms.arglist, [obj.clone()])
- after = results["after"]
- if after:
- after = [n.clone() for n in after]
- new = pytree.Node(syms.power,
- Attr(Name("sys"), Name("intern")) +
- [pytree.Node(syms.trailer,
- [results["lpar"].clone(),
- newarglist,
- results["rpar"].clone()])] + after)
- new.prefix = node.prefix
+ names = ('sys', 'intern')
+ new = ImportAndCall(node, results, names)
touch_import(None, 'sys', node)
return new
diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py
new file mode 100644
index 0000000..1855357
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_reload.py
@@ -0,0 +1,28 @@
+"""Fixer for reload().
+
+reload(s) -> imp.reload(s)"""
+
+# Local imports
+from .. import fixer_base
+from ..fixer_util import ImportAndCall, touch_import
+
+
+class FixReload(fixer_base.BaseFix):
+ BM_compatible = True
+ order = "pre"
+
+ PATTERN = """
+ power< 'reload'
+ trailer< lpar='('
+ ( not(arglist | argument<any '=' any>) obj=any
+ | obj=arglist<(not argument<any '=' any>) any ','> )
+ rpar=')' >
+ after=any*
+ >
+ """
+
+ def transform(self, node, results):
+ names = ('imp', 'reload')
+ new = ImportAndCall(node, results, names)
+ touch_import(None, 'imp', node)
+ return new
diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
index f9cc18b..93bae90 100644
--- a/Lib/lib2to3/main.py
+++ b/Lib/lib2to3/main.py
@@ -90,11 +90,11 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
if os.path.lexists(backup):
try:
os.remove(backup)
- except os.error as err:
+ except OSError as err:
self.log_message("Can't remove backup %s", backup)
try:
os.rename(filename, backup)
- except os.error as err:
+ except OSError as err:
self.log_message("Can't rename %s to %s", filename, backup)
# Actually write the new file
write = super(StdoutRefactoringTool, self).write_file
diff --git a/Lib/lib2to3/pgen2/conv.py b/Lib/lib2to3/pgen2/conv.py
index bf49762..ed0cac5 100644
--- a/Lib/lib2to3/pgen2/conv.py
+++ b/Lib/lib2to3/pgen2/conv.py
@@ -60,7 +60,7 @@ class Converter(grammar.Grammar):
"""
try:
f = open(filename)
- except IOError as err:
+ except OSError as err:
print("Can't open %s: %s" % (filename, err))
return False
self.symbol2number = {}
@@ -111,7 +111,7 @@ class Converter(grammar.Grammar):
"""
try:
f = open(filename)
- except IOError as err:
+ except OSError as err:
print("Can't open %s: %s" % (filename, err))
return False
# The code below essentially uses f's iterator-ness!
diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py
index 4c611c6..3ccc69d 100644
--- a/Lib/lib2to3/pgen2/driver.py
+++ b/Lib/lib2to3/pgen2/driver.py
@@ -123,7 +123,7 @@ def load_grammar(gt="Grammar.txt", gp=None,
logger.info("Writing grammar tables to %s", gp)
try:
g.dump(gp)
- except IOError as e:
+ except OSError as e:
logger.info("Writing failed:"+str(e))
else:
g = grammar.Grammar()
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
index 14c5f70..7f1c564 100644
--- a/Lib/lib2to3/pgen2/grammar.py
+++ b/Lib/lib2to3/pgen2/grammar.py
@@ -86,15 +86,13 @@ class Grammar(object):
def dump(self, filename):
"""Dump the grammar tables to a pickle file."""
- f = open(filename, "wb")
- pickle.dump(self.__dict__, f, 2)
- f.close()
+ with open(filename, "wb") as f:
+ pickle.dump(self.__dict__, f, 2)
def load(self, filename):
"""Load the grammar tables from a pickle file."""
- f = open(filename, "rb")
- d = pickle.load(f)
- f.close()
+ with open(filename, "rb") as f:
+ d = pickle.load(f)
self.__dict__.update(d)
def copy(self):
diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py
index 17cbf0a..c4a1be3 100644
--- a/Lib/lib2to3/pytree.py
+++ b/Lib/lib2to3/pytree.py
@@ -194,8 +194,7 @@ class Base(object):
def leaves(self):
for child in self.children:
- for x in child.leaves():
- yield x
+ yield from child.leaves()
def depth(self):
if self.parent is None:
@@ -274,16 +273,14 @@ class Node(Base):
def post_order(self):
"""Return a post-order iterator for the tree."""
for child in self.children:
- for node in child.post_order():
- yield node
+ yield from child.post_order()
yield self
def pre_order(self):
"""Return a pre-order iterator for the tree."""
yield self
for child in self.children:
- for node in child.pre_order():
- yield node
+ yield from child.pre_order()
def _prefix_getter(self):
"""
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index 201e193..8100317 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -326,7 +326,7 @@ class RefactoringTool(object):
"""
try:
f = open(filename, "rb")
- except IOError as err:
+ except OSError as err:
self.log_error("Can't open %s: %s", filename, err)
return None, None
try:
@@ -534,12 +534,12 @@ class RefactoringTool(object):
"""
try:
f = _open_with_encoding(filename, "w", encoding=encoding)
- except os.error as err:
+ except OSError as err:
self.log_error("Can't create %s: %s", filename, err)
return
try:
f.write(_to_system_newlines(new_text))
- except os.error as err:
+ except OSError as err:
self.log_error("Can't write %s: %s", filename, err)
finally:
f.close()
diff --git a/Lib/lib2to3/tests/pytree_idempotency.py b/Lib/lib2to3/tests/pytree_idempotency.py
index a02bbfe..731c403 100755
--- a/Lib/lib2to3/tests/pytree_idempotency.py
+++ b/Lib/lib2to3/tests/pytree_idempotency.py
@@ -53,7 +53,7 @@ def main():
for dir in sys.path:
try:
names = os.listdir(dir)
- except os.error:
+ except OSError:
continue
print("Scanning", dir, "...", file=sys.stderr)
for name in names:
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index 2f08f93..06b0033 100644
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -282,6 +282,65 @@ class Test_apply(FixerTestCase):
b = """f(*args, **kwds)"""
self.check(a, b)
+class Test_reload(FixerTestCase):
+ fixer = "reload"
+
+ def test(self):
+ b = """reload(a)"""
+ a = """import imp\nimp.reload(a)"""
+ self.check(b, a)
+
+ def test_comment(self):
+ b = """reload( a ) # comment"""
+ a = """import imp\nimp.reload( a ) # comment"""
+ self.check(b, a)
+
+ # PEP 8 comments
+ b = """reload( a ) # comment"""
+ a = """import imp\nimp.reload( a ) # comment"""
+ self.check(b, a)
+
+ def test_space(self):
+ b = """reload( a )"""
+ a = """import imp\nimp.reload( a )"""
+ self.check(b, a)
+
+ b = """reload( a)"""
+ a = """import imp\nimp.reload( a)"""
+ self.check(b, a)
+
+ b = """reload(a )"""
+ a = """import imp\nimp.reload(a )"""
+ self.check(b, a)
+
+ def test_unchanged(self):
+ s = """reload(a=1)"""
+ self.unchanged(s)
+
+ s = """reload(f, g)"""
+ self.unchanged(s)
+
+ s = """reload(f, *h)"""
+ self.unchanged(s)
+
+ s = """reload(f, *h, **i)"""
+ self.unchanged(s)
+
+ s = """reload(f, **i)"""
+ self.unchanged(s)
+
+ s = """reload(*h, **i)"""
+ self.unchanged(s)
+
+ s = """reload(*h)"""
+ self.unchanged(s)
+
+ s = """reload(**i)"""
+ self.unchanged(s)
+
+ s = """reload()"""
+ self.unchanged(s)
+
class Test_intern(FixerTestCase):
fixer = "intern"
@@ -4576,3 +4635,53 @@ class Test_exitfunc(FixerTestCase):
def test_unchanged(self):
s = """f(sys.exitfunc)"""
self.unchanged(s)
+
+
+class Test_asserts(FixerTestCase):
+
+ fixer = "asserts"
+
+ def test_deprecated_names(self):
+ tests = [
+ ('self.assert_(True)', 'self.assertTrue(True)'),
+ ('self.assertEquals(2, 2)', 'self.assertEqual(2, 2)'),
+ ('self.assertNotEquals(2, 3)', 'self.assertNotEqual(2, 3)'),
+ ('self.assertAlmostEquals(2, 3)', 'self.assertAlmostEqual(2, 3)'),
+ ('self.assertNotAlmostEquals(2, 8)', 'self.assertNotAlmostEqual(2, 8)'),
+ ('self.failUnlessEqual(2, 2)', 'self.assertEqual(2, 2)'),
+ ('self.failIfEqual(2, 3)', 'self.assertNotEqual(2, 3)'),
+ ('self.failUnlessAlmostEqual(2, 3)', 'self.assertAlmostEqual(2, 3)'),
+ ('self.failIfAlmostEqual(2, 8)', 'self.assertNotAlmostEqual(2, 8)'),
+ ('self.failUnless(True)', 'self.assertTrue(True)'),
+ ('self.failUnlessRaises(foo)', 'self.assertRaises(foo)'),
+ ('self.failIf(False)', 'self.assertFalse(False)'),
+ ]
+ for b, a in tests:
+ self.check(b, a)
+
+ def test_variants(self):
+ b = 'eq = self.assertEquals'
+ a = 'eq = self.assertEqual'
+ self.check(b, a)
+ b = 'self.assertEquals(2, 3, msg="fail")'
+ a = 'self.assertEqual(2, 3, msg="fail")'
+ self.check(b, a)
+ b = 'self.assertEquals(2, 3, msg="fail") # foo'
+ a = 'self.assertEqual(2, 3, msg="fail") # foo'
+ self.check(b, a)
+ b = 'self.assertEquals (2, 3)'
+ a = 'self.assertEqual (2, 3)'
+ self.check(b, a)
+ b = ' self.assertEquals (2, 3)'
+ a = ' self.assertEqual (2, 3)'
+ self.check(b, a)
+ b = 'with self.failUnlessRaises(Explosion): explode()'
+ a = 'with self.assertRaises(Explosion): explode()'
+ self.check(b, a)
+ b = 'with self.failUnlessRaises(Explosion) as cm: explode()'
+ a = 'with self.assertRaises(Explosion) as cm: explode()'
+ self.check(b, a)
+
+ def test_unchanged(self):
+ self.unchanged('self.assertEqualsOnSaturday')
+ self.unchanged('self.assertEqualsOnSaturday(3, 5)')
diff --git a/Lib/linecache.py b/Lib/linecache.py
index c3f2c3f..02a9eb5 100644
--- a/Lib/linecache.py
+++ b/Lib/linecache.py
@@ -59,7 +59,7 @@ def checkcache(filename=None):
continue # no-op for files loaded via a __loader__
try:
stat = os.stat(fullname)
- except os.error:
+ except OSError:
del cache[filename]
continue
if size != stat.st_size or mtime != stat.st_mtime:
@@ -91,7 +91,7 @@ def updatecache(filename, module_globals=None):
if name and get_source:
try:
data = get_source(name)
- except (ImportError, IOError):
+ except (ImportError, OSError):
pass
else:
if data is None:
@@ -118,14 +118,14 @@ def updatecache(filename, module_globals=None):
try:
stat = os.stat(fullname)
break
- except os.error:
+ except OSError:
pass
else:
return []
try:
with tokenize.open(fullname) as fp:
lines = fp.readlines()
- except IOError:
+ except OSError:
return []
if lines and not lines[-1].endswith('\n'):
lines[-1] += '\n'
diff --git a/Lib/locale.py b/Lib/locale.py
index 7cfea61..fa24829 100644
--- a/Lib/locale.py
+++ b/Lib/locale.py
@@ -344,14 +344,32 @@ def _replace_encoding(code, encoding):
# Convert the encoding to a C lib compatible encoding string
norm_encoding = encodings.normalize_encoding(encoding)
#print('norm encoding: %r' % norm_encoding)
- norm_encoding = encodings.aliases.aliases.get(norm_encoding,
+ norm_encoding = encodings.aliases.aliases.get(norm_encoding.lower(),
norm_encoding)
#print('aliased encoding: %r' % norm_encoding)
- encoding = locale_encoding_alias.get(norm_encoding,
- norm_encoding)
+ encoding = norm_encoding
+ norm_encoding = norm_encoding.lower()
+ if norm_encoding in locale_encoding_alias:
+ encoding = locale_encoding_alias[norm_encoding]
+ else:
+ norm_encoding = norm_encoding.replace('_', '')
+ norm_encoding = norm_encoding.replace('-', '')
+ if norm_encoding in locale_encoding_alias:
+ encoding = locale_encoding_alias[norm_encoding]
#print('found encoding %r' % encoding)
return langname + '.' + encoding
+def _append_modifier(code, modifier):
+ if modifier == 'euro':
+ if '.' not in code:
+ return code + '.ISO8859-15'
+ _, _, encoding = code.partition('.')
+ if encoding in ('ISO8859-15', 'UTF-8'):
+ return code
+ if encoding == 'ISO8859-1':
+ return _replace_encoding(code, 'ISO8859-15')
+ return code + '@' + modifier
+
def normalize(localename):
""" Returns a normalized locale code for the given locale
@@ -403,7 +421,7 @@ def normalize(localename):
if code is not None:
#print('lookup without modifier succeeded')
if '@' not in code:
- return code + '@' + modifier
+ return _append_modifier(code, modifier)
if code.split('@', 1)[1].lower() == modifier:
return code
#print('second lookup failed')
@@ -427,7 +445,8 @@ def normalize(localename):
if code is not None:
#print('lookup without modifier and encoding succeeded')
if '@' not in code:
- return _replace_encoding(code, encoding) + '@' + modifier
+ code = _replace_encoding(code, encoding)
+ return _append_modifier(code, modifier)
code, defmod = code.split('@', 1)
if defmod.lower() == modifier:
return _replace_encoding(code, encoding) + '@' + defmod
@@ -586,8 +605,8 @@ if sys.platform.startswith("win"):
# On Win32, this will return the ANSI code page
def getpreferredencoding(do_setlocale = True):
"""Return the charset that the user is likely using."""
- import _locale
- return _locale._getdefaultlocale()[1]
+ import _bootlocale
+ return _bootlocale.getpreferredencoding(False)
else:
# On Unix, if CODESET is available, use that.
try:
@@ -606,27 +625,16 @@ else:
def getpreferredencoding(do_setlocale = True):
"""Return the charset that the user is likely using,
according to the system configuration."""
+ import _bootlocale
if do_setlocale:
oldloc = setlocale(LC_CTYPE)
try:
setlocale(LC_CTYPE, "")
except Error:
pass
- result = nl_langinfo(CODESET)
- if not result and sys.platform == 'darwin':
- # nl_langinfo can return an empty string
- # when the setting has an invalid value.
- # Default to UTF-8 in that case because
- # UTF-8 is the default charset on OSX and
- # returning nothing will crash the
- # interpreter.
- result = 'UTF-8'
+ result = _bootlocale.getpreferredencoding(False)
+ if do_setlocale:
setlocale(LC_CTYPE, oldloc)
- else:
- result = nl_langinfo(CODESET)
- if not result and sys.platform == 'darwin':
- # See above for explanation
- result = 'UTF-8'
return result
@@ -654,6 +662,14 @@ locale_encoding_alias = {
'jis': 'JIS7',
'jis7': 'JIS7',
'ajec': 'eucJP',
+ 'koi8c': 'KOI8-C',
+ 'microsoftcp1251': 'CP1251',
+ 'microsoftcp1255': 'CP1255',
+ 'microsoftcp1256': 'CP1256',
+ '88591': 'ISO8859-1',
+ '88592': 'ISO8859-2',
+ '88595': 'ISO8859-5',
+ '885915': 'ISO8859-15',
# Mappings from Python codec names to C lib encoding names
'ascii': 'ISO8859-1',
@@ -681,10 +697,18 @@ locale_encoding_alias = {
'utf_8': 'UTF-8',
'koi8_r': 'KOI8-R',
'koi8_u': 'KOI8-U',
+ 'cp1251': 'CP1251',
+ 'cp1255': 'CP1255',
+ 'cp1256': 'CP1256',
+
# XXX This list is still incomplete. If you know more
# mappings, please file a bug report. Thanks.
}
+for k, v in sorted(locale_encoding_alias.items()):
+ k = k.replace('_', '')
+ locale_encoding_alias.setdefault(k, v)
+
#
# The locale_alias table maps lowercase alias names to C locale names
# (case-sensitive). Encodings are always separated from the locale
@@ -797,55 +821,33 @@ locale_encoding_alias = {
locale_alias = {
'a3': 'az_AZ.KOI8-C',
'a3_az': 'az_AZ.KOI8-C',
- 'a3_az.koi8c': 'az_AZ.KOI8-C',
'a3_az.koic': 'az_AZ.KOI8-C',
'af': 'af_ZA.ISO8859-1',
'af_za': 'af_ZA.ISO8859-1',
- 'af_za.iso88591': 'af_ZA.ISO8859-1',
'am': 'am_ET.UTF-8',
'am_et': 'am_ET.UTF-8',
'american': 'en_US.ISO8859-1',
- 'american.iso88591': 'en_US.ISO8859-1',
'ar': 'ar_AA.ISO8859-6',
'ar_aa': 'ar_AA.ISO8859-6',
- 'ar_aa.iso88596': 'ar_AA.ISO8859-6',
'ar_ae': 'ar_AE.ISO8859-6',
- 'ar_ae.iso88596': 'ar_AE.ISO8859-6',
'ar_bh': 'ar_BH.ISO8859-6',
- 'ar_bh.iso88596': 'ar_BH.ISO8859-6',
'ar_dz': 'ar_DZ.ISO8859-6',
- 'ar_dz.iso88596': 'ar_DZ.ISO8859-6',
'ar_eg': 'ar_EG.ISO8859-6',
- 'ar_eg.iso88596': 'ar_EG.ISO8859-6',
'ar_in': 'ar_IN.UTF-8',
'ar_iq': 'ar_IQ.ISO8859-6',
- 'ar_iq.iso88596': 'ar_IQ.ISO8859-6',
'ar_jo': 'ar_JO.ISO8859-6',
- 'ar_jo.iso88596': 'ar_JO.ISO8859-6',
'ar_kw': 'ar_KW.ISO8859-6',
- 'ar_kw.iso88596': 'ar_KW.ISO8859-6',
'ar_lb': 'ar_LB.ISO8859-6',
- 'ar_lb.iso88596': 'ar_LB.ISO8859-6',
'ar_ly': 'ar_LY.ISO8859-6',
- 'ar_ly.iso88596': 'ar_LY.ISO8859-6',
'ar_ma': 'ar_MA.ISO8859-6',
- 'ar_ma.iso88596': 'ar_MA.ISO8859-6',
'ar_om': 'ar_OM.ISO8859-6',
- 'ar_om.iso88596': 'ar_OM.ISO8859-6',
'ar_qa': 'ar_QA.ISO8859-6',
- 'ar_qa.iso88596': 'ar_QA.ISO8859-6',
'ar_sa': 'ar_SA.ISO8859-6',
- 'ar_sa.iso88596': 'ar_SA.ISO8859-6',
'ar_sd': 'ar_SD.ISO8859-6',
- 'ar_sd.iso88596': 'ar_SD.ISO8859-6',
'ar_sy': 'ar_SY.ISO8859-6',
- 'ar_sy.iso88596': 'ar_SY.ISO8859-6',
'ar_tn': 'ar_TN.ISO8859-6',
- 'ar_tn.iso88596': 'ar_TN.ISO8859-6',
'ar_ye': 'ar_YE.ISO8859-6',
- 'ar_ye.iso88596': 'ar_YE.ISO8859-6',
'arabic': 'ar_AA.ISO8859-6',
- 'arabic.iso88596': 'ar_AA.ISO8859-6',
'as': 'as_IN.UTF-8',
'as_in': 'as_IN.UTF-8',
'az': 'az_AZ.ISO8859-9E',
@@ -854,35 +856,20 @@ locale_alias = {
'be': 'be_BY.CP1251',
'be@latin': 'be_BY.UTF-8@latin',
'be_by': 'be_BY.CP1251',
- 'be_by.cp1251': 'be_BY.CP1251',
- 'be_by.microsoftcp1251': 'be_BY.CP1251',
- 'be_by.utf8@latin': 'be_BY.UTF-8@latin',
'be_by@latin': 'be_BY.UTF-8@latin',
'bg': 'bg_BG.CP1251',
'bg_bg': 'bg_BG.CP1251',
- 'bg_bg.cp1251': 'bg_BG.CP1251',
- 'bg_bg.iso88595': 'bg_BG.ISO8859-5',
- 'bg_bg.koi8r': 'bg_BG.KOI8-R',
- 'bg_bg.microsoftcp1251': 'bg_BG.CP1251',
'bn_in': 'bn_IN.UTF-8',
'bo_in': 'bo_IN.UTF-8',
'bokmal': 'nb_NO.ISO8859-1',
'bokm\xe5l': 'nb_NO.ISO8859-1',
'br': 'br_FR.ISO8859-1',
'br_fr': 'br_FR.ISO8859-1',
- 'br_fr.iso88591': 'br_FR.ISO8859-1',
- 'br_fr.iso885914': 'br_FR.ISO8859-14',
- 'br_fr.iso885915': 'br_FR.ISO8859-15',
- 'br_fr.iso885915@euro': 'br_FR.ISO8859-15',
- 'br_fr.utf8@euro': 'br_FR.UTF-8',
- 'br_fr@euro': 'br_FR.ISO8859-15',
'bs': 'bs_BA.ISO8859-2',
'bs_ba': 'bs_BA.ISO8859-2',
- 'bs_ba.iso88592': 'bs_BA.ISO8859-2',
'bulgarian': 'bg_BG.CP1251',
'c': 'C',
'c-french': 'fr_CA.ISO8859-1',
- 'c-french.iso88591': 'fr_CA.ISO8859-1',
'c.ascii': 'C',
'c.en': 'C',
'c.iso88591': 'en_US.ISO8859-1',
@@ -890,343 +877,131 @@ locale_alias = {
'c_c.c': 'C',
'ca': 'ca_ES.ISO8859-1',
'ca_ad': 'ca_AD.ISO8859-1',
- 'ca_ad.iso88591': 'ca_AD.ISO8859-1',
- 'ca_ad.iso885915': 'ca_AD.ISO8859-15',
- 'ca_ad.iso885915@euro': 'ca_AD.ISO8859-15',
- 'ca_ad.utf8@euro': 'ca_AD.UTF-8',
- 'ca_ad@euro': 'ca_AD.ISO8859-15',
'ca_es': 'ca_ES.ISO8859-1',
- 'ca_es.iso88591': 'ca_ES.ISO8859-1',
- 'ca_es.iso885915': 'ca_ES.ISO8859-15',
- 'ca_es.iso885915@euro': 'ca_ES.ISO8859-15',
- 'ca_es.utf8@euro': 'ca_ES.UTF-8',
- 'ca_es@euro': 'ca_ES.ISO8859-15',
'ca_fr': 'ca_FR.ISO8859-1',
- 'ca_fr.iso88591': 'ca_FR.ISO8859-1',
- 'ca_fr.iso885915': 'ca_FR.ISO8859-15',
- 'ca_fr.iso885915@euro': 'ca_FR.ISO8859-15',
- 'ca_fr.utf8@euro': 'ca_FR.UTF-8',
- 'ca_fr@euro': 'ca_FR.ISO8859-15',
'ca_it': 'ca_IT.ISO8859-1',
- 'ca_it.iso88591': 'ca_IT.ISO8859-1',
- 'ca_it.iso885915': 'ca_IT.ISO8859-15',
- 'ca_it.iso885915@euro': 'ca_IT.ISO8859-15',
- 'ca_it.utf8@euro': 'ca_IT.UTF-8',
- 'ca_it@euro': 'ca_IT.ISO8859-15',
'catalan': 'ca_ES.ISO8859-1',
'cextend': 'en_US.ISO8859-1',
- 'cextend.en': 'en_US.ISO8859-1',
'chinese-s': 'zh_CN.eucCN',
'chinese-t': 'zh_TW.eucTW',
'croatian': 'hr_HR.ISO8859-2',
'cs': 'cs_CZ.ISO8859-2',
'cs_cs': 'cs_CZ.ISO8859-2',
- 'cs_cs.iso88592': 'cs_CZ.ISO8859-2',
'cs_cz': 'cs_CZ.ISO8859-2',
- 'cs_cz.iso88592': 'cs_CZ.ISO8859-2',
'cy': 'cy_GB.ISO8859-1',
'cy_gb': 'cy_GB.ISO8859-1',
- 'cy_gb.iso88591': 'cy_GB.ISO8859-1',
- 'cy_gb.iso885914': 'cy_GB.ISO8859-14',
- 'cy_gb.iso885915': 'cy_GB.ISO8859-15',
- 'cy_gb@euro': 'cy_GB.ISO8859-15',
'cz': 'cs_CZ.ISO8859-2',
'cz_cz': 'cs_CZ.ISO8859-2',
'czech': 'cs_CZ.ISO8859-2',
'da': 'da_DK.ISO8859-1',
- 'da.iso885915': 'da_DK.ISO8859-15',
'da_dk': 'da_DK.ISO8859-1',
- 'da_dk.88591': 'da_DK.ISO8859-1',
- 'da_dk.885915': 'da_DK.ISO8859-15',
- 'da_dk.iso88591': 'da_DK.ISO8859-1',
- 'da_dk.iso885915': 'da_DK.ISO8859-15',
- 'da_dk@euro': 'da_DK.ISO8859-15',
'danish': 'da_DK.ISO8859-1',
- 'danish.iso88591': 'da_DK.ISO8859-1',
'dansk': 'da_DK.ISO8859-1',
'de': 'de_DE.ISO8859-1',
- 'de.iso885915': 'de_DE.ISO8859-15',
'de_at': 'de_AT.ISO8859-1',
- 'de_at.iso88591': 'de_AT.ISO8859-1',
- 'de_at.iso885915': 'de_AT.ISO8859-15',
- 'de_at.iso885915@euro': 'de_AT.ISO8859-15',
- 'de_at.utf8@euro': 'de_AT.UTF-8',
- 'de_at@euro': 'de_AT.ISO8859-15',
'de_be': 'de_BE.ISO8859-1',
- 'de_be.iso88591': 'de_BE.ISO8859-1',
- 'de_be.iso885915': 'de_BE.ISO8859-15',
- 'de_be.iso885915@euro': 'de_BE.ISO8859-15',
- 'de_be.utf8@euro': 'de_BE.UTF-8',
- 'de_be@euro': 'de_BE.ISO8859-15',
'de_ch': 'de_CH.ISO8859-1',
- 'de_ch.iso88591': 'de_CH.ISO8859-1',
- 'de_ch.iso885915': 'de_CH.ISO8859-15',
- 'de_ch@euro': 'de_CH.ISO8859-15',
'de_de': 'de_DE.ISO8859-1',
- 'de_de.88591': 'de_DE.ISO8859-1',
- 'de_de.885915': 'de_DE.ISO8859-15',
- 'de_de.885915@euro': 'de_DE.ISO8859-15',
- 'de_de.iso88591': 'de_DE.ISO8859-1',
- 'de_de.iso885915': 'de_DE.ISO8859-15',
- 'de_de.iso885915@euro': 'de_DE.ISO8859-15',
- 'de_de.utf8@euro': 'de_DE.UTF-8',
- 'de_de@euro': 'de_DE.ISO8859-15',
'de_lu': 'de_LU.ISO8859-1',
- 'de_lu.iso88591': 'de_LU.ISO8859-1',
- 'de_lu.iso885915': 'de_LU.ISO8859-15',
- 'de_lu.iso885915@euro': 'de_LU.ISO8859-15',
- 'de_lu.utf8@euro': 'de_LU.UTF-8',
- 'de_lu@euro': 'de_LU.ISO8859-15',
'deutsch': 'de_DE.ISO8859-1',
'dutch': 'nl_NL.ISO8859-1',
'dutch.iso88591': 'nl_BE.ISO8859-1',
'ee': 'ee_EE.ISO8859-4',
'ee_ee': 'ee_EE.ISO8859-4',
- 'ee_ee.iso88594': 'ee_EE.ISO8859-4',
'eesti': 'et_EE.ISO8859-1',
'el': 'el_GR.ISO8859-7',
'el_gr': 'el_GR.ISO8859-7',
- 'el_gr.iso88597': 'el_GR.ISO8859-7',
'el_gr@euro': 'el_GR.ISO8859-15',
'en': 'en_US.ISO8859-1',
- 'en.iso88591': 'en_US.ISO8859-1',
'en_au': 'en_AU.ISO8859-1',
- 'en_au.iso88591': 'en_AU.ISO8859-1',
'en_be': 'en_BE.ISO8859-1',
- 'en_be@euro': 'en_BE.ISO8859-15',
'en_bw': 'en_BW.ISO8859-1',
- 'en_bw.iso88591': 'en_BW.ISO8859-1',
'en_ca': 'en_CA.ISO8859-1',
- 'en_ca.iso88591': 'en_CA.ISO8859-1',
'en_gb': 'en_GB.ISO8859-1',
- 'en_gb.88591': 'en_GB.ISO8859-1',
- 'en_gb.iso88591': 'en_GB.ISO8859-1',
- 'en_gb.iso885915': 'en_GB.ISO8859-15',
- 'en_gb@euro': 'en_GB.ISO8859-15',
'en_hk': 'en_HK.ISO8859-1',
- 'en_hk.iso88591': 'en_HK.ISO8859-1',
'en_ie': 'en_IE.ISO8859-1',
- 'en_ie.iso88591': 'en_IE.ISO8859-1',
- 'en_ie.iso885915': 'en_IE.ISO8859-15',
- 'en_ie.iso885915@euro': 'en_IE.ISO8859-15',
- 'en_ie.utf8@euro': 'en_IE.UTF-8',
- 'en_ie@euro': 'en_IE.ISO8859-15',
'en_in': 'en_IN.ISO8859-1',
'en_nz': 'en_NZ.ISO8859-1',
- 'en_nz.iso88591': 'en_NZ.ISO8859-1',
'en_ph': 'en_PH.ISO8859-1',
- 'en_ph.iso88591': 'en_PH.ISO8859-1',
'en_sg': 'en_SG.ISO8859-1',
- 'en_sg.iso88591': 'en_SG.ISO8859-1',
'en_uk': 'en_GB.ISO8859-1',
'en_us': 'en_US.ISO8859-1',
- 'en_us.88591': 'en_US.ISO8859-1',
- 'en_us.885915': 'en_US.ISO8859-15',
- 'en_us.iso88591': 'en_US.ISO8859-1',
- 'en_us.iso885915': 'en_US.ISO8859-15',
- 'en_us.iso885915@euro': 'en_US.ISO8859-15',
- 'en_us@euro': 'en_US.ISO8859-15',
'en_us@euro@euro': 'en_US.ISO8859-15',
'en_za': 'en_ZA.ISO8859-1',
- 'en_za.88591': 'en_ZA.ISO8859-1',
- 'en_za.iso88591': 'en_ZA.ISO8859-1',
- 'en_za.iso885915': 'en_ZA.ISO8859-15',
- 'en_za@euro': 'en_ZA.ISO8859-15',
'en_zw': 'en_ZW.ISO8859-1',
- 'en_zw.iso88591': 'en_ZW.ISO8859-1',
'eng_gb': 'en_GB.ISO8859-1',
- 'eng_gb.8859': 'en_GB.ISO8859-1',
'english': 'en_EN.ISO8859-1',
- 'english.iso88591': 'en_EN.ISO8859-1',
'english_uk': 'en_GB.ISO8859-1',
- 'english_uk.8859': 'en_GB.ISO8859-1',
'english_united-states': 'en_US.ISO8859-1',
'english_united-states.437': 'C',
'english_us': 'en_US.ISO8859-1',
- 'english_us.8859': 'en_US.ISO8859-1',
- 'english_us.ascii': 'en_US.ISO8859-1',
'eo': 'eo_XX.ISO8859-3',
'eo_eo': 'eo_EO.ISO8859-3',
- 'eo_eo.iso88593': 'eo_EO.ISO8859-3',
'eo_xx': 'eo_XX.ISO8859-3',
- 'eo_xx.iso88593': 'eo_XX.ISO8859-3',
'es': 'es_ES.ISO8859-1',
'es_ar': 'es_AR.ISO8859-1',
- 'es_ar.iso88591': 'es_AR.ISO8859-1',
'es_bo': 'es_BO.ISO8859-1',
- 'es_bo.iso88591': 'es_BO.ISO8859-1',
'es_cl': 'es_CL.ISO8859-1',
- 'es_cl.iso88591': 'es_CL.ISO8859-1',
'es_co': 'es_CO.ISO8859-1',
- 'es_co.iso88591': 'es_CO.ISO8859-1',
'es_cr': 'es_CR.ISO8859-1',
- 'es_cr.iso88591': 'es_CR.ISO8859-1',
'es_do': 'es_DO.ISO8859-1',
- 'es_do.iso88591': 'es_DO.ISO8859-1',
'es_ec': 'es_EC.ISO8859-1',
- 'es_ec.iso88591': 'es_EC.ISO8859-1',
'es_es': 'es_ES.ISO8859-1',
- 'es_es.88591': 'es_ES.ISO8859-1',
- 'es_es.iso88591': 'es_ES.ISO8859-1',
- 'es_es.iso885915': 'es_ES.ISO8859-15',
- 'es_es.iso885915@euro': 'es_ES.ISO8859-15',
- 'es_es.utf8@euro': 'es_ES.UTF-8',
- 'es_es@euro': 'es_ES.ISO8859-15',
'es_gt': 'es_GT.ISO8859-1',
- 'es_gt.iso88591': 'es_GT.ISO8859-1',
'es_hn': 'es_HN.ISO8859-1',
- 'es_hn.iso88591': 'es_HN.ISO8859-1',
'es_mx': 'es_MX.ISO8859-1',
- 'es_mx.iso88591': 'es_MX.ISO8859-1',
'es_ni': 'es_NI.ISO8859-1',
- 'es_ni.iso88591': 'es_NI.ISO8859-1',
'es_pa': 'es_PA.ISO8859-1',
- 'es_pa.iso88591': 'es_PA.ISO8859-1',
- 'es_pa.iso885915': 'es_PA.ISO8859-15',
- 'es_pa@euro': 'es_PA.ISO8859-15',
'es_pe': 'es_PE.ISO8859-1',
- 'es_pe.iso88591': 'es_PE.ISO8859-1',
- 'es_pe.iso885915': 'es_PE.ISO8859-15',
- 'es_pe@euro': 'es_PE.ISO8859-15',
'es_pr': 'es_PR.ISO8859-1',
- 'es_pr.iso88591': 'es_PR.ISO8859-1',
'es_py': 'es_PY.ISO8859-1',
- 'es_py.iso88591': 'es_PY.ISO8859-1',
- 'es_py.iso885915': 'es_PY.ISO8859-15',
- 'es_py@euro': 'es_PY.ISO8859-15',
'es_sv': 'es_SV.ISO8859-1',
- 'es_sv.iso88591': 'es_SV.ISO8859-1',
- 'es_sv.iso885915': 'es_SV.ISO8859-15',
- 'es_sv@euro': 'es_SV.ISO8859-15',
'es_us': 'es_US.ISO8859-1',
- 'es_us.iso88591': 'es_US.ISO8859-1',
'es_uy': 'es_UY.ISO8859-1',
- 'es_uy.iso88591': 'es_UY.ISO8859-1',
- 'es_uy.iso885915': 'es_UY.ISO8859-15',
- 'es_uy@euro': 'es_UY.ISO8859-15',
'es_ve': 'es_VE.ISO8859-1',
- 'es_ve.iso88591': 'es_VE.ISO8859-1',
- 'es_ve.iso885915': 'es_VE.ISO8859-15',
- 'es_ve@euro': 'es_VE.ISO8859-15',
'estonian': 'et_EE.ISO8859-1',
'et': 'et_EE.ISO8859-15',
'et_ee': 'et_EE.ISO8859-15',
- 'et_ee.iso88591': 'et_EE.ISO8859-1',
- 'et_ee.iso885913': 'et_EE.ISO8859-13',
- 'et_ee.iso885915': 'et_EE.ISO8859-15',
- 'et_ee.iso88594': 'et_EE.ISO8859-4',
- 'et_ee@euro': 'et_EE.ISO8859-15',
'eu': 'eu_ES.ISO8859-1',
'eu_es': 'eu_ES.ISO8859-1',
- 'eu_es.iso88591': 'eu_ES.ISO8859-1',
- 'eu_es.iso885915': 'eu_ES.ISO8859-15',
- 'eu_es.iso885915@euro': 'eu_ES.ISO8859-15',
- 'eu_es.utf8@euro': 'eu_ES.UTF-8',
- 'eu_es@euro': 'eu_ES.ISO8859-15',
'fa': 'fa_IR.UTF-8',
'fa_ir': 'fa_IR.UTF-8',
'fa_ir.isiri3342': 'fa_IR.ISIRI-3342',
'fi': 'fi_FI.ISO8859-15',
- 'fi.iso885915': 'fi_FI.ISO8859-15',
'fi_fi': 'fi_FI.ISO8859-15',
- 'fi_fi.88591': 'fi_FI.ISO8859-1',
- 'fi_fi.iso88591': 'fi_FI.ISO8859-1',
- 'fi_fi.iso885915': 'fi_FI.ISO8859-15',
- 'fi_fi.iso885915@euro': 'fi_FI.ISO8859-15',
- 'fi_fi.utf8@euro': 'fi_FI.UTF-8',
- 'fi_fi@euro': 'fi_FI.ISO8859-15',
'finnish': 'fi_FI.ISO8859-1',
- 'finnish.iso88591': 'fi_FI.ISO8859-1',
'fo': 'fo_FO.ISO8859-1',
'fo_fo': 'fo_FO.ISO8859-1',
- 'fo_fo.iso88591': 'fo_FO.ISO8859-1',
- 'fo_fo.iso885915': 'fo_FO.ISO8859-15',
- 'fo_fo@euro': 'fo_FO.ISO8859-15',
'fr': 'fr_FR.ISO8859-1',
- 'fr.iso885915': 'fr_FR.ISO8859-15',
'fr_be': 'fr_BE.ISO8859-1',
- 'fr_be.88591': 'fr_BE.ISO8859-1',
- 'fr_be.iso88591': 'fr_BE.ISO8859-1',
- 'fr_be.iso885915': 'fr_BE.ISO8859-15',
- 'fr_be.iso885915@euro': 'fr_BE.ISO8859-15',
- 'fr_be.utf8@euro': 'fr_BE.UTF-8',
- 'fr_be@euro': 'fr_BE.ISO8859-15',
'fr_ca': 'fr_CA.ISO8859-1',
- 'fr_ca.88591': 'fr_CA.ISO8859-1',
- 'fr_ca.iso88591': 'fr_CA.ISO8859-1',
- 'fr_ca.iso885915': 'fr_CA.ISO8859-15',
- 'fr_ca@euro': 'fr_CA.ISO8859-15',
'fr_ch': 'fr_CH.ISO8859-1',
- 'fr_ch.88591': 'fr_CH.ISO8859-1',
- 'fr_ch.iso88591': 'fr_CH.ISO8859-1',
- 'fr_ch.iso885915': 'fr_CH.ISO8859-15',
- 'fr_ch@euro': 'fr_CH.ISO8859-15',
'fr_fr': 'fr_FR.ISO8859-1',
- 'fr_fr.88591': 'fr_FR.ISO8859-1',
- 'fr_fr.iso88591': 'fr_FR.ISO8859-1',
- 'fr_fr.iso885915': 'fr_FR.ISO8859-15',
- 'fr_fr.iso885915@euro': 'fr_FR.ISO8859-15',
- 'fr_fr.utf8@euro': 'fr_FR.UTF-8',
- 'fr_fr@euro': 'fr_FR.ISO8859-15',
'fr_lu': 'fr_LU.ISO8859-1',
- 'fr_lu.88591': 'fr_LU.ISO8859-1',
- 'fr_lu.iso88591': 'fr_LU.ISO8859-1',
- 'fr_lu.iso885915': 'fr_LU.ISO8859-15',
- 'fr_lu.iso885915@euro': 'fr_LU.ISO8859-15',
- 'fr_lu.utf8@euro': 'fr_LU.UTF-8',
- 'fr_lu@euro': 'fr_LU.ISO8859-15',
'fran\xe7ais': 'fr_FR.ISO8859-1',
'fre_fr': 'fr_FR.ISO8859-1',
- 'fre_fr.8859': 'fr_FR.ISO8859-1',
'french': 'fr_FR.ISO8859-1',
'french.iso88591': 'fr_CH.ISO8859-1',
'french_france': 'fr_FR.ISO8859-1',
- 'french_france.8859': 'fr_FR.ISO8859-1',
'ga': 'ga_IE.ISO8859-1',
'ga_ie': 'ga_IE.ISO8859-1',
- 'ga_ie.iso88591': 'ga_IE.ISO8859-1',
- 'ga_ie.iso885914': 'ga_IE.ISO8859-14',
- 'ga_ie.iso885915': 'ga_IE.ISO8859-15',
- 'ga_ie.iso885915@euro': 'ga_IE.ISO8859-15',
- 'ga_ie.utf8@euro': 'ga_IE.UTF-8',
- 'ga_ie@euro': 'ga_IE.ISO8859-15',
'galego': 'gl_ES.ISO8859-1',
'galician': 'gl_ES.ISO8859-1',
'gd': 'gd_GB.ISO8859-1',
'gd_gb': 'gd_GB.ISO8859-1',
- 'gd_gb.iso88591': 'gd_GB.ISO8859-1',
- 'gd_gb.iso885914': 'gd_GB.ISO8859-14',
- 'gd_gb.iso885915': 'gd_GB.ISO8859-15',
- 'gd_gb@euro': 'gd_GB.ISO8859-15',
'ger_de': 'de_DE.ISO8859-1',
- 'ger_de.8859': 'de_DE.ISO8859-1',
'german': 'de_DE.ISO8859-1',
'german.iso88591': 'de_CH.ISO8859-1',
'german_germany': 'de_DE.ISO8859-1',
- 'german_germany.8859': 'de_DE.ISO8859-1',
'gl': 'gl_ES.ISO8859-1',
'gl_es': 'gl_ES.ISO8859-1',
- 'gl_es.iso88591': 'gl_ES.ISO8859-1',
- 'gl_es.iso885915': 'gl_ES.ISO8859-15',
- 'gl_es.iso885915@euro': 'gl_ES.ISO8859-15',
- 'gl_es.utf8@euro': 'gl_ES.UTF-8',
- 'gl_es@euro': 'gl_ES.ISO8859-15',
'greek': 'el_GR.ISO8859-7',
- 'greek.iso88597': 'el_GR.ISO8859-7',
'gu_in': 'gu_IN.UTF-8',
'gv': 'gv_GB.ISO8859-1',
'gv_gb': 'gv_GB.ISO8859-1',
- 'gv_gb.iso88591': 'gv_GB.ISO8859-1',
- 'gv_gb.iso885914': 'gv_GB.ISO8859-14',
- 'gv_gb.iso885915': 'gv_GB.ISO8859-15',
- 'gv_gb@euro': 'gv_GB.ISO8859-15',
'he': 'he_IL.ISO8859-8',
'he_il': 'he_IL.ISO8859-8',
- 'he_il.cp1255': 'he_IL.CP1255',
- 'he_il.iso88598': 'he_IL.ISO8859-8',
- 'he_il.microsoftcp1255': 'he_IL.CP1255',
'hebrew': 'he_IL.ISO8859-8',
- 'hebrew.iso88598': 'he_IL.ISO8859-8',
'hi': 'hi_IN.ISCII-DEV',
'hi_in': 'hi_IN.ISCII-DEV',
'hi_in.isciidev': 'hi_IN.ISCII-DEV',
@@ -1234,23 +1009,17 @@ locale_alias = {
'hne_in': 'hne_IN.UTF-8',
'hr': 'hr_HR.ISO8859-2',
'hr_hr': 'hr_HR.ISO8859-2',
- 'hr_hr.iso88592': 'hr_HR.ISO8859-2',
'hrvatski': 'hr_HR.ISO8859-2',
'hu': 'hu_HU.ISO8859-2',
'hu_hu': 'hu_HU.ISO8859-2',
- 'hu_hu.iso88592': 'hu_HU.ISO8859-2',
'hungarian': 'hu_HU.ISO8859-2',
'icelandic': 'is_IS.ISO8859-1',
- 'icelandic.iso88591': 'is_IS.ISO8859-1',
'id': 'id_ID.ISO8859-1',
'id_id': 'id_ID.ISO8859-1',
'in': 'id_ID.ISO8859-1',
'in_id': 'id_ID.ISO8859-1',
'is': 'is_IS.ISO8859-1',
'is_is': 'is_IS.ISO8859-1',
- 'is_is.iso88591': 'is_IS.ISO8859-1',
- 'is_is.iso885915': 'is_IS.ISO8859-15',
- 'is_is@euro': 'is_IS.ISO8859-15',
'iso-8859-1': 'en_US.ISO8859-1',
'iso-8859-15': 'en_US.ISO8859-15',
'iso8859-1': 'en_US.ISO8859-1',
@@ -1258,46 +1027,23 @@ locale_alias = {
'iso_8859_1': 'en_US.ISO8859-1',
'iso_8859_15': 'en_US.ISO8859-15',
'it': 'it_IT.ISO8859-1',
- 'it.iso885915': 'it_IT.ISO8859-15',
'it_ch': 'it_CH.ISO8859-1',
- 'it_ch.iso88591': 'it_CH.ISO8859-1',
- 'it_ch.iso885915': 'it_CH.ISO8859-15',
- 'it_ch@euro': 'it_CH.ISO8859-15',
'it_it': 'it_IT.ISO8859-1',
- 'it_it.88591': 'it_IT.ISO8859-1',
- 'it_it.iso88591': 'it_IT.ISO8859-1',
- 'it_it.iso885915': 'it_IT.ISO8859-15',
- 'it_it.iso885915@euro': 'it_IT.ISO8859-15',
- 'it_it.utf8@euro': 'it_IT.UTF-8',
- 'it_it@euro': 'it_IT.ISO8859-15',
'italian': 'it_IT.ISO8859-1',
- 'italian.iso88591': 'it_IT.ISO8859-1',
'iu': 'iu_CA.NUNACOM-8',
'iu_ca': 'iu_CA.NUNACOM-8',
'iu_ca.nunacom8': 'iu_CA.NUNACOM-8',
'iw': 'he_IL.ISO8859-8',
'iw_il': 'he_IL.ISO8859-8',
- 'iw_il.iso88598': 'he_IL.ISO8859-8',
'ja': 'ja_JP.eucJP',
- 'ja.jis': 'ja_JP.JIS7',
- 'ja.sjis': 'ja_JP.SJIS',
'ja_jp': 'ja_JP.eucJP',
- 'ja_jp.ajec': 'ja_JP.eucJP',
'ja_jp.euc': 'ja_JP.eucJP',
- 'ja_jp.eucjp': 'ja_JP.eucJP',
- 'ja_jp.iso-2022-jp': 'ja_JP.JIS7',
- 'ja_jp.iso2022jp': 'ja_JP.JIS7',
- 'ja_jp.jis': 'ja_JP.JIS7',
- 'ja_jp.jis7': 'ja_JP.JIS7',
'ja_jp.mscode': 'ja_JP.SJIS',
'ja_jp.pck': 'ja_JP.SJIS',
- 'ja_jp.sjis': 'ja_JP.SJIS',
- 'ja_jp.ujis': 'ja_JP.eucJP',
'japan': 'ja_JP.eucJP',
'japanese': 'ja_JP.eucJP',
'japanese-euc': 'ja_JP.eucJP',
'japanese.euc': 'ja_JP.eucJP',
- 'japanese.sjis': 'ja_JP.SJIS',
'jp_jp': 'ja_JP.eucJP',
'ka': 'ka_GE.GEORGIAN-ACADEMY',
'ka_ge': 'ka_GE.GEORGIAN-ACADEMY',
@@ -1306,27 +1052,18 @@ locale_alias = {
'ka_ge.georgianrs': 'ka_GE.GEORGIAN-ACADEMY',
'kl': 'kl_GL.ISO8859-1',
'kl_gl': 'kl_GL.ISO8859-1',
- 'kl_gl.iso88591': 'kl_GL.ISO8859-1',
- 'kl_gl.iso885915': 'kl_GL.ISO8859-15',
- 'kl_gl@euro': 'kl_GL.ISO8859-15',
'km_kh': 'km_KH.UTF-8',
'kn': 'kn_IN.UTF-8',
'kn_in': 'kn_IN.UTF-8',
'ko': 'ko_KR.eucKR',
'ko_kr': 'ko_KR.eucKR',
'ko_kr.euc': 'ko_KR.eucKR',
- 'ko_kr.euckr': 'ko_KR.eucKR',
'korean': 'ko_KR.eucKR',
'korean.euc': 'ko_KR.eucKR',
'ks': 'ks_IN.UTF-8',
'ks_in': 'ks_IN.UTF-8',
- 'ks_in@devanagari': 'ks_IN.UTF-8@devanagari',
'kw': 'kw_GB.ISO8859-1',
'kw_gb': 'kw_GB.ISO8859-1',
- 'kw_gb.iso88591': 'kw_GB.ISO8859-1',
- 'kw_gb.iso885914': 'kw_GB.ISO8859-14',
- 'kw_gb.iso885915': 'kw_GB.ISO8859-15',
- 'kw_gb@euro': 'kw_GB.ISO8859-15',
'ky': 'ky_KG.UTF-8',
'ky_kg': 'ky_KG.UTF-8',
'lithuanian': 'lt_LT.ISO8859-13',
@@ -1337,157 +1074,78 @@ locale_alias = {
'lo_la.mulelao1': 'lo_LA.MULELAO-1',
'lt': 'lt_LT.ISO8859-13',
'lt_lt': 'lt_LT.ISO8859-13',
- 'lt_lt.iso885913': 'lt_LT.ISO8859-13',
- 'lt_lt.iso88594': 'lt_LT.ISO8859-4',
'lv': 'lv_LV.ISO8859-13',
'lv_lv': 'lv_LV.ISO8859-13',
- 'lv_lv.iso885913': 'lv_LV.ISO8859-13',
- 'lv_lv.iso88594': 'lv_LV.ISO8859-4',
'mai': 'mai_IN.UTF-8',
'mai_in': 'mai_IN.UTF-8',
'mi': 'mi_NZ.ISO8859-1',
'mi_nz': 'mi_NZ.ISO8859-1',
- 'mi_nz.iso88591': 'mi_NZ.ISO8859-1',
'mk': 'mk_MK.ISO8859-5',
'mk_mk': 'mk_MK.ISO8859-5',
- 'mk_mk.cp1251': 'mk_MK.CP1251',
- 'mk_mk.iso88595': 'mk_MK.ISO8859-5',
- 'mk_mk.microsoftcp1251': 'mk_MK.CP1251',
'ml': 'ml_IN.UTF-8',
'ml_in': 'ml_IN.UTF-8',
'mr': 'mr_IN.UTF-8',
'mr_in': 'mr_IN.UTF-8',
'ms': 'ms_MY.ISO8859-1',
'ms_my': 'ms_MY.ISO8859-1',
- 'ms_my.iso88591': 'ms_MY.ISO8859-1',
'mt': 'mt_MT.ISO8859-3',
'mt_mt': 'mt_MT.ISO8859-3',
- 'mt_mt.iso88593': 'mt_MT.ISO8859-3',
'nb': 'nb_NO.ISO8859-1',
'nb_no': 'nb_NO.ISO8859-1',
- 'nb_no.88591': 'nb_NO.ISO8859-1',
- 'nb_no.iso88591': 'nb_NO.ISO8859-1',
- 'nb_no.iso885915': 'nb_NO.ISO8859-15',
- 'nb_no@euro': 'nb_NO.ISO8859-15',
'ne_np': 'ne_NP.UTF-8',
'nl': 'nl_NL.ISO8859-1',
- 'nl.iso885915': 'nl_NL.ISO8859-15',
'nl_be': 'nl_BE.ISO8859-1',
- 'nl_be.88591': 'nl_BE.ISO8859-1',
- 'nl_be.iso88591': 'nl_BE.ISO8859-1',
- 'nl_be.iso885915': 'nl_BE.ISO8859-15',
- 'nl_be.iso885915@euro': 'nl_BE.ISO8859-15',
- 'nl_be.utf8@euro': 'nl_BE.UTF-8',
- 'nl_be@euro': 'nl_BE.ISO8859-15',
'nl_nl': 'nl_NL.ISO8859-1',
- 'nl_nl.88591': 'nl_NL.ISO8859-1',
- 'nl_nl.iso88591': 'nl_NL.ISO8859-1',
- 'nl_nl.iso885915': 'nl_NL.ISO8859-15',
- 'nl_nl.iso885915@euro': 'nl_NL.ISO8859-15',
- 'nl_nl.utf8@euro': 'nl_NL.UTF-8',
- 'nl_nl@euro': 'nl_NL.ISO8859-15',
'nn': 'nn_NO.ISO8859-1',
'nn_no': 'nn_NO.ISO8859-1',
- 'nn_no.88591': 'nn_NO.ISO8859-1',
- 'nn_no.iso88591': 'nn_NO.ISO8859-1',
- 'nn_no.iso885915': 'nn_NO.ISO8859-15',
- 'nn_no@euro': 'nn_NO.ISO8859-15',
'no': 'no_NO.ISO8859-1',
'no@nynorsk': 'ny_NO.ISO8859-1',
'no_no': 'no_NO.ISO8859-1',
- 'no_no.88591': 'no_NO.ISO8859-1',
- 'no_no.iso88591': 'no_NO.ISO8859-1',
- 'no_no.iso885915': 'no_NO.ISO8859-15',
'no_no.iso88591@bokmal': 'no_NO.ISO8859-1',
'no_no.iso88591@nynorsk': 'no_NO.ISO8859-1',
- 'no_no@euro': 'no_NO.ISO8859-15',
'norwegian': 'no_NO.ISO8859-1',
- 'norwegian.iso88591': 'no_NO.ISO8859-1',
'nr': 'nr_ZA.ISO8859-1',
'nr_za': 'nr_ZA.ISO8859-1',
- 'nr_za.iso88591': 'nr_ZA.ISO8859-1',
'nso': 'nso_ZA.ISO8859-15',
'nso_za': 'nso_ZA.ISO8859-15',
- 'nso_za.iso885915': 'nso_ZA.ISO8859-15',
'ny': 'ny_NO.ISO8859-1',
'ny_no': 'ny_NO.ISO8859-1',
- 'ny_no.88591': 'ny_NO.ISO8859-1',
- 'ny_no.iso88591': 'ny_NO.ISO8859-1',
- 'ny_no.iso885915': 'ny_NO.ISO8859-15',
- 'ny_no@euro': 'ny_NO.ISO8859-15',
'nynorsk': 'nn_NO.ISO8859-1',
'oc': 'oc_FR.ISO8859-1',
'oc_fr': 'oc_FR.ISO8859-1',
- 'oc_fr.iso88591': 'oc_FR.ISO8859-1',
- 'oc_fr.iso885915': 'oc_FR.ISO8859-15',
- 'oc_fr@euro': 'oc_FR.ISO8859-15',
'or': 'or_IN.UTF-8',
'or_in': 'or_IN.UTF-8',
'pa': 'pa_IN.UTF-8',
'pa_in': 'pa_IN.UTF-8',
'pd': 'pd_US.ISO8859-1',
'pd_de': 'pd_DE.ISO8859-1',
- 'pd_de.iso88591': 'pd_DE.ISO8859-1',
- 'pd_de.iso885915': 'pd_DE.ISO8859-15',
- 'pd_de@euro': 'pd_DE.ISO8859-15',
'pd_us': 'pd_US.ISO8859-1',
- 'pd_us.iso88591': 'pd_US.ISO8859-1',
- 'pd_us.iso885915': 'pd_US.ISO8859-15',
- 'pd_us@euro': 'pd_US.ISO8859-15',
'ph': 'ph_PH.ISO8859-1',
'ph_ph': 'ph_PH.ISO8859-1',
- 'ph_ph.iso88591': 'ph_PH.ISO8859-1',
'pl': 'pl_PL.ISO8859-2',
'pl_pl': 'pl_PL.ISO8859-2',
- 'pl_pl.iso88592': 'pl_PL.ISO8859-2',
'polish': 'pl_PL.ISO8859-2',
'portuguese': 'pt_PT.ISO8859-1',
- 'portuguese.iso88591': 'pt_PT.ISO8859-1',
'portuguese_brazil': 'pt_BR.ISO8859-1',
- 'portuguese_brazil.8859': 'pt_BR.ISO8859-1',
'posix': 'C',
'posix-utf2': 'C',
'pp': 'pp_AN.ISO8859-1',
'pp_an': 'pp_AN.ISO8859-1',
- 'pp_an.iso88591': 'pp_AN.ISO8859-1',
'pt': 'pt_PT.ISO8859-1',
- 'pt.iso885915': 'pt_PT.ISO8859-15',
'pt_br': 'pt_BR.ISO8859-1',
- 'pt_br.88591': 'pt_BR.ISO8859-1',
- 'pt_br.iso88591': 'pt_BR.ISO8859-1',
- 'pt_br.iso885915': 'pt_BR.ISO8859-15',
- 'pt_br@euro': 'pt_BR.ISO8859-15',
'pt_pt': 'pt_PT.ISO8859-1',
- 'pt_pt.88591': 'pt_PT.ISO8859-1',
- 'pt_pt.iso88591': 'pt_PT.ISO8859-1',
- 'pt_pt.iso885915': 'pt_PT.ISO8859-15',
- 'pt_pt.iso885915@euro': 'pt_PT.ISO8859-15',
- 'pt_pt.utf8@euro': 'pt_PT.UTF-8',
- 'pt_pt@euro': 'pt_PT.ISO8859-15',
'ro': 'ro_RO.ISO8859-2',
'ro_ro': 'ro_RO.ISO8859-2',
- 'ro_ro.iso88592': 'ro_RO.ISO8859-2',
'romanian': 'ro_RO.ISO8859-2',
'ru': 'ru_RU.UTF-8',
- 'ru.koi8r': 'ru_RU.KOI8-R',
'ru_ru': 'ru_RU.UTF-8',
- 'ru_ru.cp1251': 'ru_RU.CP1251',
- 'ru_ru.iso88595': 'ru_RU.ISO8859-5',
- 'ru_ru.koi8r': 'ru_RU.KOI8-R',
- 'ru_ru.microsoftcp1251': 'ru_RU.CP1251',
'ru_ua': 'ru_UA.KOI8-U',
- 'ru_ua.cp1251': 'ru_UA.CP1251',
- 'ru_ua.koi8u': 'ru_UA.KOI8-U',
- 'ru_ua.microsoftcp1251': 'ru_UA.CP1251',
'rumanian': 'ro_RO.ISO8859-2',
'russian': 'ru_RU.ISO8859-5',
'rw': 'rw_RW.ISO8859-1',
'rw_rw': 'rw_RW.ISO8859-1',
- 'rw_rw.iso88591': 'rw_RW.ISO8859-1',
'sd': 'sd_IN.UTF-8',
- 'sd@devanagari': 'sd_IN.UTF-8@devanagari',
'sd_in': 'sd_IN.UTF-8',
- 'sd_in@devanagari': 'sd_IN.UTF-8@devanagari',
'se_no': 'se_NO.UTF-8',
'serbocroatian': 'sr_RS.UTF-8@latin',
'sh': 'sr_RS.UTF-8@latin',
@@ -1501,37 +1159,26 @@ locale_alias = {
'sinhala': 'si_LK.UTF-8',
'sk': 'sk_SK.ISO8859-2',
'sk_sk': 'sk_SK.ISO8859-2',
- 'sk_sk.iso88592': 'sk_SK.ISO8859-2',
'sl': 'sl_SI.ISO8859-2',
'sl_cs': 'sl_CS.ISO8859-2',
'sl_si': 'sl_SI.ISO8859-2',
- 'sl_si.iso88592': 'sl_SI.ISO8859-2',
'slovak': 'sk_SK.ISO8859-2',
'slovene': 'sl_SI.ISO8859-2',
'slovenian': 'sl_SI.ISO8859-2',
'sp': 'sr_CS.ISO8859-5',
'sp_yu': 'sr_CS.ISO8859-5',
'spanish': 'es_ES.ISO8859-1',
- 'spanish.iso88591': 'es_ES.ISO8859-1',
'spanish_spain': 'es_ES.ISO8859-1',
- 'spanish_spain.8859': 'es_ES.ISO8859-1',
'sq': 'sq_AL.ISO8859-2',
'sq_al': 'sq_AL.ISO8859-2',
- 'sq_al.iso88592': 'sq_AL.ISO8859-2',
'sr': 'sr_RS.UTF-8',
'sr@cyrillic': 'sr_RS.UTF-8',
- 'sr@latin': 'sr_RS.UTF-8@latin',
'sr@latn': 'sr_CS.UTF-8@latin',
'sr_cs': 'sr_CS.UTF-8',
- 'sr_cs.iso88592': 'sr_CS.ISO8859-2',
'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2',
- 'sr_cs.iso88595': 'sr_CS.ISO8859-5',
- 'sr_cs.utf8@latn': 'sr_CS.UTF-8@latin',
'sr_cs@latn': 'sr_CS.UTF-8@latin',
'sr_me': 'sr_ME.UTF-8',
'sr_rs': 'sr_RS.UTF-8',
- 'sr_rs.utf8@latn': 'sr_RS.UTF-8@latin',
- 'sr_rs@latin': 'sr_RS.UTF-8@latin',
'sr_rs@latn': 'sr_RS.UTF-8@latin',
'sr_sp': 'sr_CS.ISO8859-2',
'sr_yu': 'sr_RS.UTF-8@latin',
@@ -1540,29 +1187,15 @@ locale_alias = {
'sr_yu.iso88595': 'sr_CS.ISO8859-5',
'sr_yu.iso88595@cyrillic': 'sr_CS.ISO8859-5',
'sr_yu.microsoftcp1251@cyrillic': 'sr_CS.CP1251',
- 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8',
'sr_yu@cyrillic': 'sr_RS.UTF-8',
'ss': 'ss_ZA.ISO8859-1',
'ss_za': 'ss_ZA.ISO8859-1',
- 'ss_za.iso88591': 'ss_ZA.ISO8859-1',
'st': 'st_ZA.ISO8859-1',
'st_za': 'st_ZA.ISO8859-1',
- 'st_za.iso88591': 'st_ZA.ISO8859-1',
'sv': 'sv_SE.ISO8859-1',
- 'sv.iso885915': 'sv_SE.ISO8859-15',
'sv_fi': 'sv_FI.ISO8859-1',
- 'sv_fi.iso88591': 'sv_FI.ISO8859-1',
- 'sv_fi.iso885915': 'sv_FI.ISO8859-15',
- 'sv_fi.iso885915@euro': 'sv_FI.ISO8859-15',
- 'sv_fi.utf8@euro': 'sv_FI.UTF-8',
- 'sv_fi@euro': 'sv_FI.ISO8859-15',
'sv_se': 'sv_SE.ISO8859-1',
- 'sv_se.88591': 'sv_SE.ISO8859-1',
- 'sv_se.iso88591': 'sv_SE.ISO8859-1',
- 'sv_se.iso885915': 'sv_SE.ISO8859-15',
- 'sv_se@euro': 'sv_SE.ISO8859-15',
'swedish': 'sv_SE.ISO8859-1',
- 'swedish.iso88591': 'sv_SE.ISO8859-1',
'ta': 'ta_IN.TSCII-0',
'ta_in': 'ta_IN.TSCII-0',
'ta_in.tscii': 'ta_IN.TSCII-0',
@@ -1570,49 +1203,33 @@ locale_alias = {
'te': 'te_IN.UTF-8',
'tg': 'tg_TJ.KOI8-C',
'tg_tj': 'tg_TJ.KOI8-C',
- 'tg_tj.koi8c': 'tg_TJ.KOI8-C',
'th': 'th_TH.ISO8859-11',
'th_th': 'th_TH.ISO8859-11',
- 'th_th.iso885911': 'th_TH.ISO8859-11',
'th_th.tactis': 'th_TH.TIS620',
'th_th.tis620': 'th_TH.TIS620',
'thai': 'th_TH.ISO8859-11',
'tl': 'tl_PH.ISO8859-1',
'tl_ph': 'tl_PH.ISO8859-1',
- 'tl_ph.iso88591': 'tl_PH.ISO8859-1',
'tn': 'tn_ZA.ISO8859-15',
'tn_za': 'tn_ZA.ISO8859-15',
- 'tn_za.iso885915': 'tn_ZA.ISO8859-15',
'tr': 'tr_TR.ISO8859-9',
'tr_tr': 'tr_TR.ISO8859-9',
- 'tr_tr.iso88599': 'tr_TR.ISO8859-9',
'ts': 'ts_ZA.ISO8859-1',
'ts_za': 'ts_ZA.ISO8859-1',
- 'ts_za.iso88591': 'ts_ZA.ISO8859-1',
'tt': 'tt_RU.TATAR-CYR',
'tt_ru': 'tt_RU.TATAR-CYR',
- 'tt_ru.koi8c': 'tt_RU.KOI8-C',
'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR',
'turkish': 'tr_TR.ISO8859-9',
- 'turkish.iso88599': 'tr_TR.ISO8859-9',
'uk': 'uk_UA.KOI8-U',
'uk_ua': 'uk_UA.KOI8-U',
- 'uk_ua.cp1251': 'uk_UA.CP1251',
- 'uk_ua.iso88595': 'uk_UA.ISO8859-5',
- 'uk_ua.koi8u': 'uk_UA.KOI8-U',
- 'uk_ua.microsoftcp1251': 'uk_UA.CP1251',
'univ': 'en_US.utf',
'universal': 'en_US.utf',
'universal.utf8@ucs4': 'en_US.UTF-8',
'ur': 'ur_PK.CP1256',
'ur_in': 'ur_IN.UTF-8',
'ur_pk': 'ur_PK.CP1256',
- 'ur_pk.cp1256': 'ur_PK.CP1256',
- 'ur_pk.microsoftcp1256': 'ur_PK.CP1256',
'uz': 'uz_UZ.UTF-8',
'uz_uz': 'uz_UZ.UTF-8',
- 'uz_uz.iso88591': 'uz_UZ.ISO8859-1',
- 'uz_uz.utf8@cyrillic': 'uz_UZ.UTF-8',
'uz_uz@cyrillic': 'uz_UZ.UTF-8',
've': 've_ZA.UTF-8',
've_za': 've_ZA.UTF-8',
@@ -1624,35 +1241,21 @@ locale_alias = {
'vi_vn.viscii111': 'vi_VN.VISCII',
'wa': 'wa_BE.ISO8859-1',
'wa_be': 'wa_BE.ISO8859-1',
- 'wa_be.iso88591': 'wa_BE.ISO8859-1',
- 'wa_be.iso885915': 'wa_BE.ISO8859-15',
- 'wa_be.iso885915@euro': 'wa_BE.ISO8859-15',
- 'wa_be@euro': 'wa_BE.ISO8859-15',
'xh': 'xh_ZA.ISO8859-1',
'xh_za': 'xh_ZA.ISO8859-1',
- 'xh_za.iso88591': 'xh_ZA.ISO8859-1',
'yi': 'yi_US.CP1255',
'yi_us': 'yi_US.CP1255',
- 'yi_us.cp1255': 'yi_US.CP1255',
- 'yi_us.microsoftcp1255': 'yi_US.CP1255',
'zh': 'zh_CN.eucCN',
'zh_cn': 'zh_CN.gb2312',
'zh_cn.big5': 'zh_TW.big5',
'zh_cn.euc': 'zh_CN.eucCN',
- 'zh_cn.gb18030': 'zh_CN.gb18030',
- 'zh_cn.gb2312': 'zh_CN.gb2312',
- 'zh_cn.gbk': 'zh_CN.gbk',
'zh_hk': 'zh_HK.big5hkscs',
- 'zh_hk.big5': 'zh_HK.big5',
'zh_hk.big5hk': 'zh_HK.big5hkscs',
- 'zh_hk.big5hkscs': 'zh_HK.big5hkscs',
'zh_tw': 'zh_TW.big5',
- 'zh_tw.big5': 'zh_TW.big5',
'zh_tw.euc': 'zh_TW.eucTW',
'zh_tw.euctw': 'zh_TW.eucTW',
'zu': 'zu_ZA.ISO8859-1',
'zu_za': 'zu_ZA.ISO8859-1',
- 'zu_za.iso88591': 'zu_ZA.ISO8859-1',
}
#
diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py
index 9f436f3..478c5af 100644
--- a/Lib/logging/__init__.py
+++ b/Lib/logging/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved.
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted,
@@ -18,7 +18,7 @@
Logging package for Python. Based on PEP 282 and comments thereto in
comp.lang.python.
-Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved.
To use, simply 'import logging' and log away!
"""
@@ -42,6 +42,7 @@ except ImportError: #pragma: no cover
__author__ = "Vinay Sajip <vinay_sajip@red-dove.com>"
__status__ = "production"
+# The following module attributes are no longer updated.
__version__ = "0.5.1.2"
__date__ = "07 February 2010"
@@ -67,7 +68,7 @@ else: #pragma: no cover
"""Return the frame object for the caller's stack frame."""
try:
raise Exception
- except:
+ except Exception:
return sys.exc_info()[2].tb_frame.f_back
# _srcfile is only used in conjunction with sys._getframe().
@@ -123,20 +124,22 @@ INFO = 20
DEBUG = 10
NOTSET = 0
-_levelNames = {
- CRITICAL : 'CRITICAL',
- ERROR : 'ERROR',
- WARNING : 'WARNING',
- INFO : 'INFO',
- DEBUG : 'DEBUG',
- NOTSET : 'NOTSET',
- 'CRITICAL' : CRITICAL,
- 'ERROR' : ERROR,
- 'WARN' : WARNING,
- 'WARNING' : WARNING,
- 'INFO' : INFO,
- 'DEBUG' : DEBUG,
- 'NOTSET' : NOTSET,
+_levelToName = {
+ CRITICAL: 'CRITICAL',
+ ERROR: 'ERROR',
+ WARNING: 'WARNING',
+ INFO: 'INFO',
+ DEBUG: 'DEBUG',
+ NOTSET: 'NOTSET',
+}
+_nameToLevel = {
+ 'CRITICAL': CRITICAL,
+ 'ERROR': ERROR,
+ 'WARN': WARNING,
+ 'WARNING': WARNING,
+ 'INFO': INFO,
+ 'DEBUG': DEBUG,
+ 'NOTSET': NOTSET,
}
def getLevelName(level):
@@ -153,7 +156,7 @@ def getLevelName(level):
Otherwise, the string "Level %s" % level is returned.
"""
- return _levelNames.get(level, ("Level %s" % level))
+ return _levelToName.get(level, ("Level %s" % level))
def addLevelName(level, levelName):
"""
@@ -163,8 +166,8 @@ def addLevelName(level, levelName):
"""
_acquireLock()
try: #unlikely to cause an exception, but you never know...
- _levelNames[level] = levelName
- _levelNames[levelName] = level
+ _levelToName[level] = levelName
+ _nameToLevel[levelName] = level
finally:
_releaseLock()
@@ -172,9 +175,9 @@ def _checkLevel(level):
if isinstance(level, int):
rv = level
elif str(level) == level:
- if level not in _levelNames:
+ if level not in _nameToLevel:
raise ValueError("Unknown level: %r" % level)
- rv = _levelNames[level]
+ rv = _nameToLevel[level]
else:
raise TypeError("Level not an integer or a valid string: %r" % level)
return rv
@@ -882,16 +885,37 @@ class Handler(Filterer):
The record which was being processed is passed in to this method.
"""
if raiseExceptions and sys.stderr: # see issue 13807
- ei = sys.exc_info()
+ t, v, tb = sys.exc_info()
try:
- traceback.print_exception(ei[0], ei[1], ei[2],
- None, sys.stderr)
- sys.stderr.write('Logged from file %s, line %s\n' % (
- record.filename, record.lineno))
- except IOError: #pragma: no cover
+ sys.stderr.write('--- Logging error ---\n')
+ traceback.print_exception(t, v, tb, None, sys.stderr)
+ sys.stderr.write('Call stack:\n')
+ # Walk the stack frame up until we're out of logging,
+ # so as to print the calling context.
+ frame = tb.tb_frame
+ while (frame and os.path.dirname(frame.f_code.co_filename) ==
+ __path__[0]):
+ frame = frame.f_back
+ if frame:
+ traceback.print_stack(frame, file=sys.stderr)
+ else:
+ # couldn't find the right stack frame, for some reason
+ sys.stderr.write('Logged from file %s, line %s\n' % (
+ record.filename, record.lineno))
+ # Issue 18671: output logging message and arguments
+ try:
+ sys.stderr.write('Message: %r\n'
+ 'Arguments: %s\n' % (record.msg,
+ record.args))
+ except Exception:
+ sys.stderr.write('Unable to print the message and arguments'
+ ' - possible formatting error.\nUse the'
+ ' traceback above to help find the error.\n'
+ )
+ except OSError: #pragma: no cover
pass # see issue 5971
finally:
- del ei
+ del t, v, tb
class StreamHandler(Handler):
"""
@@ -941,9 +965,7 @@ class StreamHandler(Handler):
stream.write(msg)
stream.write(self.terminator)
self.flush()
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
class FileHandler(StreamHandler):
@@ -1836,7 +1858,7 @@ def shutdown(handlerList=_handlerList):
h.acquire()
h.flush()
h.close()
- except (IOError, ValueError):
+ except (OSError, ValueError):
# Ignore errors which might be caused
# because handlers have been closed but
# references to them are still around at
@@ -1844,7 +1866,7 @@ def shutdown(handlerList=_handlerList):
pass
finally:
h.release()
- except:
+ except: # ignore everything, as we're shutting down
if raiseExceptions:
raise
#else, swallow
diff --git a/Lib/logging/config.py b/Lib/logging/config.py
index 1880614..895fb26 100644
--- a/Lib/logging/config.py
+++ b/Lib/logging/config.py
@@ -1,4 +1,4 @@
-# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved.
+# Copyright 2001-2014 by Vinay Sajip. All Rights Reserved.
#
# Permission to use, copy, modify, and distribute this software and its
# documentation for any purpose and without fee is hereby granted,
@@ -19,13 +19,19 @@ Configuration functions for the logging package for Python. The core package
is based on PEP 282 and comments thereto in comp.lang.python, and influenced
by Apache's log4j system.
-Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved.
+Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved.
To use, simply 'import logging' and log away!
"""
-import sys, logging, logging.handlers, socket, struct, traceback, re
+import errno
import io
+import logging
+import logging.handlers
+import re
+import struct
+import sys
+import traceback
try:
import _thread as thread
@@ -38,10 +44,7 @@ from socketserver import ThreadingTCPServer, StreamRequestHandler
DEFAULT_LOGGING_CONFIG_PORT = 9030
-if sys.platform == "win32":
- RESET_ERROR = 10054 #WSAECONNRESET
-else:
- RESET_ERROR = 104 #ECONNRESET
+RESET_ERROR = errno.ECONNRESET
#
# The following code implements a socket listener for on-the-fly
@@ -61,11 +64,14 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True):
"""
import configparser
- cp = configparser.ConfigParser(defaults)
- if hasattr(fname, 'readline'):
- cp.read_file(fname)
+ if isinstance(fname, configparser.RawConfigParser):
+ cp = fname
else:
- cp.read(fname)
+ cp = configparser.ConfigParser(defaults)
+ if hasattr(fname, 'readline'):
+ cp.read_file(fname)
+ else:
+ cp.read(fname)
formatters = _create_formatters(cp)
@@ -141,7 +147,7 @@ def _install_handlers(cp, formatters):
h = klass(*args)
if "level" in section:
level = section["level"]
- h.setLevel(logging._levelNames[level])
+ h.setLevel(level)
if len(fmt):
h.setFormatter(formatters[fmt])
if issubclass(klass, logging.handlers.MemoryHandler):
@@ -188,7 +194,7 @@ def _install_loggers(cp, handlers, disable_existing):
log = root
if "level" in section:
level = section["level"]
- log.setLevel(logging._levelNames[level])
+ log.setLevel(level)
for h in root.handlers[:]:
root.removeHandler(h)
hlist = section["handlers"]
@@ -234,7 +240,7 @@ def _install_loggers(cp, handlers, disable_existing):
existing.remove(qn)
if "level" in section:
level = section["level"]
- logger.setLevel(logging._levelNames[level])
+ logger.setLevel(level)
for h in logger.handlers[:]:
logger.removeHandler(h)
logger.propagate = propagate
@@ -271,6 +277,30 @@ def valid_ident(s):
return True
+class ConvertingMixin(object):
+ """For ConvertingXXX's, this mixin class provides common functions"""
+
+ def convert_with_key(self, key, value, replace=True):
+ result = self.configurator.convert(value)
+ #If the converted value is different, save for next time
+ if value is not result:
+ if replace:
+ self[key] = result
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+ def convert(self, value):
+ result = self.configurator.convert(value)
+ if value is not result:
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ return result
+
+
# The ConvertingXXX classes are wrappers around standard Python containers,
# and they serve to convert any suitable values in the container. The
# conversion converts base dicts, lists and tuples to their wrapped
@@ -280,77 +310,37 @@ def valid_ident(s):
# Each wrapper should have a configurator attribute holding the actual
# configurator to use for conversion.
-class ConvertingDict(dict):
+class ConvertingDict(dict, ConvertingMixin):
"""A converting dictionary wrapper."""
def __getitem__(self, key):
value = dict.__getitem__(self, key)
- result = self.configurator.convert(value)
- #If the converted value is different, save for next time
- if value is not result:
- self[key] = result
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- result.key = key
- return result
+ return self.convert_with_key(key, value)
def get(self, key, default=None):
value = dict.get(self, key, default)
- result = self.configurator.convert(value)
- #If the converted value is different, save for next time
- if value is not result:
- self[key] = result
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- result.key = key
- return result
+ return self.convert_with_key(key, value)
def pop(self, key, default=None):
value = dict.pop(self, key, default)
- result = self.configurator.convert(value)
- if value is not result:
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- result.key = key
- return result
+ return self.convert_with_key(key, value, replace=False)
-class ConvertingList(list):
+class ConvertingList(list, ConvertingMixin):
"""A converting list wrapper."""
def __getitem__(self, key):
value = list.__getitem__(self, key)
- result = self.configurator.convert(value)
- #If the converted value is different, save for next time
- if value is not result:
- self[key] = result
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- result.key = key
- return result
+ return self.convert_with_key(key, value)
def pop(self, idx=-1):
value = list.pop(self, idx)
- result = self.configurator.convert(value)
- if value is not result:
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- return result
+ return self.convert(value)
-class ConvertingTuple(tuple):
+class ConvertingTuple(tuple, ConvertingMixin):
"""A converting tuple wrapper."""
def __getitem__(self, key):
value = tuple.__getitem__(self, key)
- result = self.configurator.convert(value)
- if value is not result:
- if type(result) in (ConvertingDict, ConvertingList,
- ConvertingTuple):
- result.parent = self
- result.key = key
- return result
+ # Can't replace a tuple entry.
+ return self.convert_with_key(key, value, replace=False)
class BaseConfigurator(object):
"""
@@ -729,6 +719,7 @@ class DictConfigurator(BaseConfigurator):
'address' in config:
config['address'] = self.as_tuple(config['address'])
factory = klass
+ props = config.pop('.', None)
kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
try:
result = factory(**kwargs)
@@ -747,6 +738,9 @@ class DictConfigurator(BaseConfigurator):
result.setLevel(logging._checkLevel(level))
if filters:
self.add_filters(result, filters)
+ if props:
+ for name, value in props.items():
+ setattr(result, name, value)
return result
def add_handlers(self, logger, handlers):
@@ -795,7 +789,7 @@ def dictConfig(config):
dictConfigClass(config).configure()
-def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
+def listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None):
"""
Start up a socket server on the specified port, and listen for new
configurations.
@@ -804,6 +798,15 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
Returns a Thread object on which you can call start() to start the server,
and which you can join() when appropriate. To stop the server, call
stopListening().
+
+ Use the ``verify`` argument to verify any bytes received across the wire
+ from a client. If specified, it should be a callable which receives a
+ single argument - the bytes of configuration data received across the
+ network - and it should return either ``None``, to indicate that the
+ passed in bytes could not be verified and should be discarded, or a
+ byte string which is then passed to the configuration machinery as
+ normal. Note that you can return transformed bytes, e.g. by decrypting
+ the bytes passed in.
"""
if not thread: #pragma: no cover
raise NotImplementedError("listen() needs threading to work")
@@ -831,31 +834,28 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
chunk = self.connection.recv(slen)
while len(chunk) < slen:
chunk = chunk + conn.recv(slen - len(chunk))
- chunk = chunk.decode("utf-8")
- try:
- import json
- d =json.loads(chunk)
- assert isinstance(d, dict)
- dictConfig(d)
- except:
- #Apply new configuration.
-
- file = io.StringIO(chunk)
+ if self.server.verify is not None:
+ chunk = self.server.verify(chunk)
+ if chunk is not None: # verified, can process
+ chunk = chunk.decode("utf-8")
try:
- fileConfig(file)
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
- traceback.print_exc()
+ import json
+ d =json.loads(chunk)
+ assert isinstance(d, dict)
+ dictConfig(d)
+ except Exception:
+ #Apply new configuration.
+
+ file = io.StringIO(chunk)
+ try:
+ fileConfig(file)
+ except Exception:
+ traceback.print_exc()
if self.server.ready:
self.server.ready.set()
- except socket.error as e:
- if not isinstance(e.args, tuple):
+ except OSError as e:
+ if e.errno != RESET_ERROR:
raise
- else:
- errcode = e.args[0]
- if errcode != RESET_ERROR:
- raise
class ConfigSocketReceiver(ThreadingTCPServer):
"""
@@ -865,13 +865,14 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
allow_reuse_address = 1
def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
- handler=None, ready=None):
+ handler=None, ready=None, verify=None):
ThreadingTCPServer.__init__(self, (host, port), handler)
logging._acquireLock()
self.abort = 0
logging._releaseLock()
self.timeout = 1
self.ready = ready
+ self.verify = verify
def serve_until_stopped(self):
import select
@@ -889,16 +890,18 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
class Server(threading.Thread):
- def __init__(self, rcvr, hdlr, port):
+ def __init__(self, rcvr, hdlr, port, verify):
super(Server, self).__init__()
self.rcvr = rcvr
self.hdlr = hdlr
self.port = port
+ self.verify = verify
self.ready = threading.Event()
def run(self):
server = self.rcvr(port=self.port, handler=self.hdlr,
- ready=self.ready)
+ ready=self.ready,
+ verify=self.verify)
if self.port == 0:
self.port = server.server_address[1]
self.ready.set()
@@ -908,7 +911,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT):
logging._releaseLock()
server.serve_until_stopped()
- return Server(ConfigSocketReceiver, ConfigStreamHandler, port)
+ return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify)
def stopListening():
"""
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
index ddec7dd..f547d17 100644
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -23,8 +23,7 @@ Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved.
To use, simply 'import logging.handlers' and log away!
"""
-import errno, logging, socket, os, pickle, struct, time, re
-from codecs import BOM_UTF8
+import logging, socket, os, pickle, struct, time, re
from stat import ST_DEV, ST_INO, ST_MTIME
import queue
try:
@@ -72,9 +71,7 @@ class BaseRotatingHandler(logging.FileHandler):
if self.shouldRollover(record):
self.doRollover()
logging.FileHandler.emit(self, record)
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
def rotation_filename(self, default_name):
@@ -201,11 +198,12 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
If backupCount is > 0, when rollover is done, no more than backupCount
files are kept - the oldest ones are deleted.
"""
- def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False):
+ def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None):
BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay)
self.when = when.upper()
self.backupCount = backupCount
self.utc = utc
+ self.atTime = atTime
# Calculate the real rollover interval, which is just the number of
# seconds between rollovers. Also set the filename suffix used when
# a rollover occurs. Current 'when' events supported:
@@ -275,9 +273,22 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
currentHour = t[3]
currentMinute = t[4]
currentSecond = t[5]
- # r is the number of seconds left between now and midnight
- r = _MIDNIGHT - ((currentHour * 60 + currentMinute) * 60 +
- currentSecond)
+ currentDay = t[6]
+ # r is the number of seconds left between now and the next rotation
+ if self.atTime is None:
+ rotate_ts = _MIDNIGHT
+ else:
+ rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 +
+ self.atTime.second)
+
+ r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 +
+ currentSecond)
+ if r < 0:
+ # Rotate time is before the current time (for example when
+ # self.rotateAt is 13:45 and it now 14:15), rotation is
+ # tomorrow.
+ r += _MIDNIGHT
+ currentDay = (currentDay + 1) % 7
result = currentTime + r
# If we are rolling over on a certain day, add in the number of days until
# the next rollover, but offset by 1 since we just calculated the time
@@ -295,7 +306,7 @@ class TimedRotatingFileHandler(BaseRotatingHandler):
# This is because the above time calculation takes us to midnight on this
# day, i.e. the start of the next day.
if self.when.startswith('W'):
- day = t[6] # 0 is Monday
+ day = currentDay # 0 is Monday
if day != self.dayOfWeek:
if day < self.dayOfWeek:
daysToWait = self.dayOfWeek - day
@@ -444,11 +455,8 @@ class WatchedFileHandler(logging.FileHandler):
try:
# stat the file by path, checking for existence
sres = os.stat(self.baseFilename)
- except OSError as err:
- if err.errno == errno.ENOENT:
- sres = None
- else:
- raise
+ except FileNotFoundError:
+ sres = None
# compare file system stat with that of our stream file handle
if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino:
if self.stream is not None:
@@ -485,6 +493,10 @@ class SocketHandler(logging.Handler):
logging.Handler.__init__(self)
self.host = host
self.port = port
+ if port is None:
+ self.address = host
+ else:
+ self.address = (host, port)
self.sock = None
self.closeOnError = False
self.retryTime = None
@@ -500,15 +512,17 @@ class SocketHandler(logging.Handler):
A factory method which allows subclasses to define the precise
type of socket they want.
"""
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- if hasattr(s, 'settimeout'):
- s.settimeout(timeout)
- try:
- s.connect((self.host, self.port))
- return s
- except socket.error:
- s.close()
- raise
+ if self.port is not None:
+ result = socket.create_connection(self.address, timeout=timeout)
+ else:
+ result = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ result.settimeout(timeout)
+ try:
+ result.connect(self.address)
+ except OSError:
+ result.close() # Issue 19182
+ raise
+ return result
def createSocket(self):
"""
@@ -528,7 +542,7 @@ class SocketHandler(logging.Handler):
try:
self.sock = self.makeSocket()
self.retryTime = None # next time, no delay before trying
- except socket.error:
+ except OSError:
#Creation failed, so set the retry time and return.
if self.retryTime is None:
self.retryPeriod = self.retryStart
@@ -552,16 +566,8 @@ class SocketHandler(logging.Handler):
#but are still unable to connect.
if self.sock:
try:
- if hasattr(self.sock, "sendall"):
- self.sock.sendall(s)
- else: #pragma: no cover
- sentsofar = 0
- left = len(s)
- while left > 0:
- sent = self.sock.send(s[sentsofar:])
- sentsofar = sentsofar + sent
- left = left - sent
- except socket.error: #pragma: no cover
+ self.sock.sendall(s)
+ except OSError: #pragma: no cover
self.sock.close()
self.sock = None # so we can call createSocket next time
@@ -611,9 +617,7 @@ class SocketHandler(logging.Handler):
try:
s = self.makePickle(record)
self.send(s)
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
def close(self):
@@ -652,7 +656,11 @@ class DatagramHandler(SocketHandler):
The factory method of SocketHandler is here overridden to create
a UDP socket (SOCK_DGRAM).
"""
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ if self.port is None:
+ family = socket.AF_UNIX
+ else:
+ family = socket.AF_INET
+ s = socket.socket(family, socket.SOCK_DGRAM)
return s
def send(self, s):
@@ -665,7 +673,7 @@ class DatagramHandler(SocketHandler):
"""
if self.sock is None:
self.createSocket()
- self.sock.sendto(s, (self.host, self.port))
+ self.sock.sendto(s, self.address)
class SysLogHandler(logging.Handler):
"""
@@ -777,7 +785,11 @@ class SysLogHandler(logging.Handler):
If address is specified as a string, a UNIX socket is used. To log to a
local syslogd, "SysLogHandler(address="/dev/log")" can be used.
- If facility is not specified, LOG_USER is used.
+ If facility is not specified, LOG_USER is used. If socktype is
+ specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific
+ socket type will be used. For Unix sockets, you can also specify a
+ socktype of None, in which case socket.SOCK_DGRAM will be used, falling
+ back to socket.SOCK_STREAM.
"""
logging.Handler.__init__(self)
@@ -807,7 +819,7 @@ class SysLogHandler(logging.Handler):
self.socket.connect(address)
# it worked, so set self.socktype to the used type
self.socktype = use_socktype
- except socket.error:
+ except OSError:
self.socket.close()
if self.socktype is not None:
# user didn't specify falling back, so fail
@@ -818,7 +830,7 @@ class SysLogHandler(logging.Handler):
self.socket.connect(address)
# it worked, so set self.socktype to the used type
self.socktype = use_socktype
- except socket.error:
+ except OSError:
self.socket.close()
raise
@@ -871,10 +883,9 @@ class SysLogHandler(logging.Handler):
msg = self.ident + msg
if self.append_nul:
msg += '\000'
- """
- We need to convert record level to lowercase, maybe this will
- change in the future.
- """
+
+ # We need to convert record level to lowercase, maybe this will
+ # change in the future.
prio = '<%d>' % self.encodePriority(self.facility,
self.mapPriority(record.levelname))
prio = prio.encode('utf-8')
@@ -885,7 +896,7 @@ class SysLogHandler(logging.Handler):
if self.unixsocket:
try:
self.socket.send(msg)
- except socket.error:
+ except OSError:
self.socket.close()
self._connect_unixsocket(self.address)
self.socket.send(msg)
@@ -893,9 +904,7 @@ class SysLogHandler(logging.Handler):
self.socket.sendto(msg, self.address)
else:
self.socket.sendall(msg)
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
class SMTPHandler(logging.Handler):
@@ -973,9 +982,7 @@ class SMTPHandler(logging.Handler):
smtp.login(self.username, self.password)
smtp.sendmail(self.fromaddr, self.toaddrs, msg)
smtp.quit()
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
class NTEventLogHandler(logging.Handler):
@@ -1060,9 +1067,7 @@ class NTEventLogHandler(logging.Handler):
type = self.getEventType(record)
msg = self.format(record)
self._welu.ReportEvent(self.appname, id, cat, type, [msg])
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
def close(self):
@@ -1147,9 +1152,7 @@ class HTTPHandler(logging.Handler):
if self.method == "POST":
h.send(data.encode('utf-8'))
h.getresponse() #can't do anything with the result
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
class BufferingHandler(logging.Handler):
@@ -1329,9 +1332,7 @@ class QueueHandler(logging.Handler):
"""
try:
self.enqueue(self.prepare(record))
- except (KeyboardInterrupt, SystemExit): #pragma: no cover
- raise
- except:
+ except Exception:
self.handleError(record)
if threading:
diff --git a/Lib/lzma.py b/Lib/lzma.py
index 1a89887..f1d3958 100644
--- a/Lib/lzma.py
+++ b/Lib/lzma.py
@@ -54,9 +54,9 @@ class LZMAFile(io.BufferedIOBase):
bytes object), in which case the named file is opened, or it can
be an existing file object to read from or write to.
- mode can be "r" for reading (default), "w" for (over)writing, or
- "a" for appending. These can equivalently be given as "rb", "wb",
- and "ab" respectively.
+ mode can be "r" for reading (default), "w" for (over)writing,
+ "x" for creating exclusively, or "a" for appending. These can
+ equivalently be given as "rb", "wb", "xb" and "ab" respectively.
format specifies the container format to use for the file.
If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the
@@ -110,8 +110,9 @@ class LZMAFile(io.BufferedIOBase):
# stream will need a separate decompressor object.
self._init_args = {"format":format, "filters":filters}
self._decompressor = LZMADecompressor(**self._init_args)
- self._buffer = None
- elif mode in ("w", "wb", "a", "ab"):
+ self._buffer = b""
+ self._buffer_offset = 0
+ elif mode in ("w", "wb", "a", "ab", "x", "xb"):
if format is None:
format = FORMAT_XZ
mode_code = _MODE_WRITE
@@ -143,7 +144,7 @@ class LZMAFile(io.BufferedIOBase):
try:
if self._mode in (_MODE_READ, _MODE_READ_EOF):
self._decompressor = None
- self._buffer = None
+ self._buffer = b""
elif self._mode == _MODE_WRITE:
self._fp.write(self._compressor.flush())
self._compressor = None
@@ -187,15 +188,18 @@ class LZMAFile(io.BufferedIOBase):
raise ValueError("I/O operation on closed file")
def _check_can_read(self):
- if not self.readable():
+ if self._mode not in (_MODE_READ, _MODE_READ_EOF):
+ self._check_not_closed()
raise io.UnsupportedOperation("File not open for reading")
def _check_can_write(self):
- if not self.writable():
+ if self._mode != _MODE_WRITE:
+ self._check_not_closed()
raise io.UnsupportedOperation("File not open for writing")
def _check_can_seek(self):
- if not self.readable():
+ if self._mode not in (_MODE_READ, _MODE_READ_EOF):
+ self._check_not_closed()
raise io.UnsupportedOperation("Seeking is only supported "
"on files open for reading")
if not self._fp.seekable():
@@ -204,16 +208,13 @@ class LZMAFile(io.BufferedIOBase):
# Fill the readahead buffer if it is empty. Returns False on EOF.
def _fill_buffer(self):
+ if self._mode == _MODE_READ_EOF:
+ return False
# Depending on the input data, our call to the decompressor may not
# return any data. In this case, try again after reading another block.
- while True:
- if self._buffer:
- return True
-
- if self._decompressor.unused_data:
- rawblock = self._decompressor.unused_data
- else:
- rawblock = self._fp.read(_BUFFER_SIZE)
+ while self._buffer_offset == len(self._buffer):
+ rawblock = (self._decompressor.unused_data or
+ self._fp.read(_BUFFER_SIZE))
if not rawblock:
if self._decompressor.eof:
@@ -236,30 +237,48 @@ class LZMAFile(io.BufferedIOBase):
return False
else:
self._buffer = self._decompressor.decompress(rawblock)
+ self._buffer_offset = 0
+ return True
# Read data until EOF.
# If return_data is false, consume the data without returning it.
def _read_all(self, return_data=True):
+ # The loop assumes that _buffer_offset is 0. Ensure that this is true.
+ self._buffer = self._buffer[self._buffer_offset:]
+ self._buffer_offset = 0
+
blocks = []
while self._fill_buffer():
if return_data:
blocks.append(self._buffer)
self._pos += len(self._buffer)
- self._buffer = None
+ self._buffer = b""
if return_data:
return b"".join(blocks)
# Read a block of up to n bytes.
# If return_data is false, consume the data without returning it.
def _read_block(self, n, return_data=True):
+ # If we have enough data buffered, return immediately.
+ end = self._buffer_offset + n
+ if end <= len(self._buffer):
+ data = self._buffer[self._buffer_offset : end]
+ self._buffer_offset = end
+ self._pos += len(data)
+ return data if return_data else None
+
+ # The loop assumes that _buffer_offset is 0. Ensure that this is true.
+ self._buffer = self._buffer[self._buffer_offset:]
+ self._buffer_offset = 0
+
blocks = []
while n > 0 and self._fill_buffer():
if n < len(self._buffer):
data = self._buffer[:n]
- self._buffer = self._buffer[n:]
+ self._buffer_offset = n
else:
data = self._buffer
- self._buffer = None
+ self._buffer = b""
if return_data:
blocks.append(data)
self._pos += len(data)
@@ -274,9 +293,9 @@ class LZMAFile(io.BufferedIOBase):
The exact number of bytes returned is unspecified.
"""
self._check_can_read()
- if self._mode == _MODE_READ_EOF or not self._fill_buffer():
+ if not self._fill_buffer():
return b""
- return self._buffer
+ return self._buffer[self._buffer_offset:]
def read(self, size=-1):
"""Read up to size uncompressed bytes from the file.
@@ -285,7 +304,7 @@ class LZMAFile(io.BufferedIOBase):
Returns b"" if the file is already at EOF.
"""
self._check_can_read()
- if self._mode == _MODE_READ_EOF or size == 0:
+ if size == 0:
return b""
elif size < 0:
return self._read_all()
@@ -302,18 +321,40 @@ class LZMAFile(io.BufferedIOBase):
# this does not give enough data for the decompressor to make progress.
# In this case we make multiple reads, to avoid returning b"".
self._check_can_read()
- if (size == 0 or self._mode == _MODE_READ_EOF or
- not self._fill_buffer()):
+ if (size == 0 or
+ # Only call _fill_buffer() if the buffer is actually empty.
+ # This gives a significant speedup if *size* is small.
+ (self._buffer_offset == len(self._buffer) and not self._fill_buffer())):
return b""
- if 0 < size < len(self._buffer):
- data = self._buffer[:size]
- self._buffer = self._buffer[size:]
+ if size > 0:
+ data = self._buffer[self._buffer_offset :
+ self._buffer_offset + size]
+ self._buffer_offset += len(data)
else:
- data = self._buffer
- self._buffer = None
+ data = self._buffer[self._buffer_offset:]
+ self._buffer = b""
+ self._buffer_offset = 0
self._pos += len(data)
return data
+ def readline(self, size=-1):
+ """Read a line of uncompressed bytes from the file.
+
+ The terminating newline (if present) is retained. If size is
+ non-negative, no more than size bytes will be read (in which
+ case the line may be incomplete). Returns b'' if already at EOF.
+ """
+ self._check_can_read()
+ # Shortcut for the common case - the whole line is in the buffer.
+ if size < 0:
+ end = self._buffer.find(b"\n", self._buffer_offset) + 1
+ if end > 0:
+ line = self._buffer[self._buffer_offset : end]
+ self._buffer_offset = end
+ self._pos += len(line)
+ return line
+ return io.BufferedIOBase.readline(self, size)
+
def write(self, data):
"""Write a bytes object to the file.
@@ -333,7 +374,8 @@ class LZMAFile(io.BufferedIOBase):
self._mode = _MODE_READ
self._pos = 0
self._decompressor = LZMADecompressor(**self._init_args)
- self._buffer = None
+ self._buffer = b""
+ self._buffer_offset = 0
def seek(self, offset, whence=0):
"""Change the file position.
@@ -372,8 +414,7 @@ class LZMAFile(io.BufferedIOBase):
offset -= self._pos
# Read and discard data until we reach the desired position.
- if self._mode != _MODE_READ_EOF:
- self._read_block(offset, return_data=False)
+ self._read_block(offset, return_data=False)
return self._pos
@@ -388,23 +429,25 @@ def open(filename, mode="rb", *,
encoding=None, errors=None, newline=None):
"""Open an LZMA-compressed file in binary or text mode.
- filename can be either an actual file name (given as a str or bytes object),
- in which case the named file is opened, or it can be an existing file object
- to read from or write to.
+ filename can be either an actual file name (given as a str or bytes
+ object), in which case the named file is opened, or it can be an
+ existing file object to read from or write to.
- The mode argument can be "r", "rb" (default), "w", "wb", "a", or "ab" for
- binary mode, or "rt", "wt" or "at" for text mode.
+ The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb",
+ "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text
+ mode.
- The format, check, preset and filters arguments specify the compression
- settings, as for LZMACompressor, LZMADecompressor and LZMAFile.
+ The format, check, preset and filters arguments specify the
+ compression settings, as for LZMACompressor, LZMADecompressor and
+ LZMAFile.
- For binary mode, this function is equivalent to the LZMAFile constructor:
- LZMAFile(filename, mode, ...). In this case, the encoding, errors and
- newline arguments must not be provided.
+ For binary mode, this function is equivalent to the LZMAFile
+ constructor: LZMAFile(filename, mode, ...). In this case, the
+ encoding, errors and newline arguments must not be provided.
For text mode, a LZMAFile object is created, and wrapped in an
- io.TextIOWrapper instance with the specified encoding, error handling
- behavior, and line ending(s).
+ io.TextIOWrapper instance with the specified encoding, error
+ handling behavior, and line ending(s).
"""
if "t" in mode:
@@ -434,7 +477,7 @@ def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None):
Refer to LZMACompressor's docstring for a description of the
optional arguments *format*, *check*, *preset* and *filters*.
- For incremental compression, use an LZMACompressor object instead.
+ For incremental compression, use an LZMACompressor instead.
"""
comp = LZMACompressor(format, check, preset, filters)
return comp.compress(data) + comp.flush()
@@ -446,7 +489,7 @@ def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None):
Refer to LZMADecompressor's docstring for a description of the
optional arguments *format*, *check* and *filters*.
- For incremental decompression, use a LZMADecompressor object instead.
+ For incremental decompression, use an LZMADecompressor instead.
"""
results = []
while True:
diff --git a/Lib/macpath.py b/Lib/macpath.py
index 1615d91..d34f9e94 100644
--- a/Lib/macpath.py
+++ b/Lib/macpath.py
@@ -127,7 +127,7 @@ def lexists(path):
try:
st = os.lstat(path)
- except os.error:
+ except OSError:
return False
return True
diff --git a/Lib/mailbox.py b/Lib/mailbox.py
index 3b64c2e..8a25a19 100644
--- a/Lib/mailbox.py
+++ b/Lib/mailbox.py
@@ -6,7 +6,6 @@
# or returning from a flush() method. See functions _sync_flush() and
# _sync_close().
-import sys
import os
import time
import calendar
@@ -20,9 +19,6 @@ import email.generator
import io
import contextlib
try:
- if sys.platform == 'os2emx':
- # OS/2 EMX fcntl() not adequate
- raise ImportError
import fcntl
except ImportError:
fcntl = None
@@ -339,11 +335,8 @@ class Maildir(Mailbox):
# This overrides an inapplicable implementation in the superclass.
try:
self.remove(key)
- except KeyError:
+ except (KeyError, FileNotFoundError):
pass
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
def __setitem__(self, key, message):
"""Replace the keyed message; raise KeyError if it doesn't exist."""
@@ -375,14 +368,11 @@ class Maildir(Mailbox):
def get_message(self, key):
"""Return a Message representation or raise a KeyError."""
subpath = self._lookup(key)
- f = open(os.path.join(self._path, subpath), 'rb')
- try:
+ with open(os.path.join(self._path, subpath), 'rb') as f:
if self._factory:
msg = self._factory(f)
else:
msg = MaildirMessage(f)
- finally:
- f.close()
subdir, name = os.path.split(subpath)
msg.set_subdir(subdir)
if self.colon in name:
@@ -392,11 +382,8 @@ class Maildir(Mailbox):
def get_bytes(self, key):
"""Return a bytes representation or raise a KeyError."""
- f = open(os.path.join(self._path, self._lookup(key)), 'rb')
- try:
+ with open(os.path.join(self._path, self._lookup(key)), 'rb') as f:
return f.read().replace(linesep, b'\n')
- finally:
- f.close()
def get_file(self, key):
"""Return a file-like representation or raise a KeyError."""
@@ -508,16 +495,12 @@ class Maildir(Mailbox):
path = os.path.join(self._path, 'tmp', uniq)
try:
os.stat(path)
- except OSError as e:
- if e.errno == errno.ENOENT:
- Maildir._count += 1
- try:
- return _create_carefully(path)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- else:
- raise
+ except FileNotFoundError:
+ Maildir._count += 1
+ try:
+ return _create_carefully(path)
+ except FileExistsError:
+ pass
# Fall through to here if stat succeeded or open raised EEXIST.
raise ExternalClashError('Name clash prevented file creation: %s' %
@@ -594,7 +577,7 @@ class _singlefileMailbox(Mailbox):
Mailbox.__init__(self, path, factory, create)
try:
f = open(self._path, 'rb+')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
if create:
f = open(self._path, 'wb+')
@@ -637,8 +620,7 @@ class _singlefileMailbox(Mailbox):
def iterkeys(self):
"""Return an iterator over keys."""
self._lookup()
- for key in self._toc.keys():
- yield key
+ yield from self._toc.keys()
def __contains__(self, key):
"""Return True if the keyed message exists, False otherwise."""
@@ -716,13 +698,9 @@ class _singlefileMailbox(Mailbox):
os.chmod(new_file.name, mode)
try:
os.rename(new_file.name, self._path)
- except OSError as e:
- if e.errno == errno.EEXIST or \
- (os.name == 'os2' and e.errno == errno.EACCES):
- os.remove(self._path)
- os.rename(new_file.name, self._path)
- else:
- raise
+ except FileExistsError:
+ os.remove(self._path)
+ os.rename(new_file.name, self._path)
self._file = open(self._path, 'rb+')
self._toc = new_toc
self._pending = False
@@ -999,7 +977,7 @@ class MH(Mailbox):
path = os.path.join(self._path, str(key))
try:
f = open(path, 'rb+')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
raise KeyError('No message with key: %s' % key)
else:
@@ -1013,7 +991,7 @@ class MH(Mailbox):
path = os.path.join(self._path, str(key))
try:
f = open(path, 'rb+')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
raise KeyError('No message with key: %s' % key)
else:
@@ -1039,12 +1017,12 @@ class MH(Mailbox):
f = open(os.path.join(self._path, str(key)), 'rb+')
else:
f = open(os.path.join(self._path, str(key)), 'rb')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
raise KeyError('No message with key: %s' % key)
else:
raise
- try:
+ with f:
if self._locked:
_lock_file(f)
try:
@@ -1052,8 +1030,6 @@ class MH(Mailbox):
finally:
if self._locked:
_unlock_file(f)
- finally:
- f.close()
for name, key_list in self.get_sequences().items():
if key in key_list:
msg.add_sequence(name)
@@ -1066,12 +1042,12 @@ class MH(Mailbox):
f = open(os.path.join(self._path, str(key)), 'rb+')
else:
f = open(os.path.join(self._path, str(key)), 'rb')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
raise KeyError('No message with key: %s' % key)
else:
raise
- try:
+ with f:
if self._locked:
_lock_file(f)
try:
@@ -1079,14 +1055,12 @@ class MH(Mailbox):
finally:
if self._locked:
_unlock_file(f)
- finally:
- f.close()
def get_file(self, key):
"""Return a file-like representation or raise a KeyError."""
try:
f = open(os.path.join(self._path, str(key)), 'rb')
- except IOError as e:
+ except OSError as e:
if e.errno == errno.ENOENT:
raise KeyError('No message with key: %s' % key)
else:
@@ -2079,7 +2053,7 @@ def _lock_file(f, dotlock=True):
if fcntl:
try:
fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except IOError as e:
+ except OSError as e:
if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS):
raise ExternalClashError('lockf: lock unavailable: %s' %
f.name)
@@ -2089,7 +2063,7 @@ def _lock_file(f, dotlock=True):
try:
pre_lock = _create_temporary(f.name + '.lock')
pre_lock.close()
- except IOError as e:
+ except OSError as e:
if e.errno in (errno.EACCES, errno.EROFS):
return # Without write access, just skip dotlocking.
else:
@@ -2102,14 +2076,10 @@ def _lock_file(f, dotlock=True):
else:
os.rename(pre_lock.name, f.name + '.lock')
dotlock_done = True
- except OSError as e:
- if e.errno == errno.EEXIST or \
- (os.name == 'os2' and e.errno == errno.EACCES):
- os.remove(pre_lock.name)
- raise ExternalClashError('dot lock unavailable: %s' %
- f.name)
- else:
- raise
+ except FileExistsError:
+ os.remove(pre_lock.name)
+ raise ExternalClashError('dot lock unavailable: %s' %
+ f.name)
except:
if fcntl:
fcntl.lockf(f, fcntl.LOCK_UN)
diff --git a/Lib/mailcap.py b/Lib/mailcap.py
index 0c0b19c..97e3035 100644
--- a/Lib/mailcap.py
+++ b/Lib/mailcap.py
@@ -20,7 +20,7 @@ def getcaps():
for mailcap in listmailcapfiles():
try:
fp = open(mailcap, 'r')
- except IOError:
+ except OSError:
continue
with fp:
morecaps = readmailcapfile(fp)
diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index cdebf7a..b98c874 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -361,7 +361,7 @@ def init(files=None):
def read_mime_types(file):
try:
f = open(file)
- except IOError:
+ except OSError:
return None
with f:
db = MimeTypes()
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index 264b0f0..cc5b8cc 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -1,13 +1,17 @@
"""Find modules used by a script, using introspection."""
import dis
-import imp
+import importlib._bootstrap
import importlib.machinery
import marshal
import os
import sys
import types
import struct
+import warnings
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore', PendingDeprecationWarning)
+ import imp
# XXX Clean up once str8's cstor matches bytes.
LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')])
@@ -229,7 +233,7 @@ class ModuleFinder:
for dir in m.__path__:
try:
names = os.listdir(dir)
- except os.error:
+ except OSError:
self.msg(2, "can't list directory", dir)
continue
for name in names:
@@ -284,11 +288,12 @@ class ModuleFinder:
if type == imp.PY_SOURCE:
co = compile(fp.read()+'\n', pathname, 'exec')
elif type == imp.PY_COMPILED:
- if fp.read(4) != imp.get_magic():
- self.msgout(2, "raise ImportError: Bad magic number", pathname)
- raise ImportError("Bad magic number in %s" % pathname)
- fp.read(8) # Skip mtime and size.
- co = marshal.load(fp)
+ try:
+ marshal_data = importlib._bootstrap._validate_bytecode_header(fp.read())
+ except ImportError as exc:
+ self.msgout(2, "raise ImportError: " + str(exc), pathname)
+ raise
+ co = marshal.loads(marshal_data)
else:
co = None
m = self.add_module(fqname)
diff --git a/Lib/multiprocessing/__init__.py b/Lib/multiprocessing/__init__.py
index 1f3e67c..86df638 100644
--- a/Lib/multiprocessing/__init__.py
+++ b/Lib/multiprocessing/__init__.py
@@ -8,260 +8,31 @@
# subpackage 'multiprocessing.dummy' has the same API but is a simple
# wrapper for 'threading'.
#
-# Try calling `multiprocessing.doc.main()` to read the html
-# documentation in a webbrowser.
-#
-#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
-__version__ = '0.70a1'
-
-__all__ = [
- 'Process', 'current_process', 'active_children', 'freeze_support',
- 'Manager', 'Pipe', 'cpu_count', 'log_to_stderr', 'get_logger',
- 'allow_connection_pickling', 'BufferTooShort', 'TimeoutError',
- 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition',
- 'Event', 'Barrier', 'Queue', 'SimpleQueue', 'JoinableQueue', 'Pool',
- 'Value', 'Array', 'RawValue', 'RawArray', 'SUBDEBUG', 'SUBWARNING',
- ]
-
-__author__ = 'R. Oudkerk (r.m.oudkerk@gmail.com)'
-
-#
-# Imports
-#
-
-import os
import sys
-
-from multiprocessing.process import Process, current_process, active_children
-from multiprocessing.util import SUBDEBUG, SUBWARNING
+from . import context
#
-# Exceptions
+# Copy stuff from default context
#
-class ProcessError(Exception):
- pass
-
-class BufferTooShort(ProcessError):
- pass
-
-class TimeoutError(ProcessError):
- pass
-
-class AuthenticationError(ProcessError):
- pass
-
-import _multiprocessing
-
-#
-# Definitions not depending on native semaphores
-#
-
-def Manager():
- '''
- Returns a manager associated with a running server process
-
- The managers methods such as `Lock()`, `Condition()` and `Queue()`
- can be used to create shared objects.
- '''
- from multiprocessing.managers import SyncManager
- m = SyncManager()
- m.start()
- return m
-
-def Pipe(duplex=True):
- '''
- Returns two connection object connected by a pipe
- '''
- from multiprocessing.connection import Pipe
- return Pipe(duplex)
-
-def cpu_count():
- '''
- Returns the number of CPUs in the system
- '''
- if sys.platform == 'win32':
- try:
- num = int(os.environ['NUMBER_OF_PROCESSORS'])
- except (ValueError, KeyError):
- num = 0
- elif 'bsd' in sys.platform or sys.platform == 'darwin':
- comm = '/sbin/sysctl -n hw.ncpu'
- if sys.platform == 'darwin':
- comm = '/usr' + comm
- try:
- with os.popen(comm) as p:
- num = int(p.read())
- except ValueError:
- num = 0
- else:
- try:
- num = os.sysconf('SC_NPROCESSORS_ONLN')
- except (ValueError, OSError, AttributeError):
- num = 0
-
- if num >= 1:
- return num
- else:
- raise NotImplementedError('cannot determine number of cpus')
-
-def freeze_support():
- '''
- Check whether this is a fake forked process in a frozen executable.
- If so then run code specified by commandline and exit.
- '''
- if sys.platform == 'win32' and getattr(sys, 'frozen', False):
- from multiprocessing.forking import freeze_support
- freeze_support()
-
-def get_logger():
- '''
- Return package logger -- if it does not already exist then it is created
- '''
- from multiprocessing.util import get_logger
- return get_logger()
-
-def log_to_stderr(level=None):
- '''
- Turn on logging and add a handler which prints to stderr
- '''
- from multiprocessing.util import log_to_stderr
- return log_to_stderr(level)
-
-def allow_connection_pickling():
- '''
- Install support for sending connections and sockets between processes
- '''
- # This is undocumented. In previous versions of multiprocessing
- # its only effect was to make socket objects inheritable on Windows.
- import multiprocessing.connection
+globals().update((name, getattr(context._default_context, name))
+ for name in context._default_context.__all__)
+__all__ = context._default_context.__all__
#
-# Definitions depending on native semaphores
+# XXX These should not really be documented or public.
#
-def Lock():
- '''
- Returns a non-recursive lock object
- '''
- from multiprocessing.synchronize import Lock
- return Lock()
-
-def RLock():
- '''
- Returns a recursive lock object
- '''
- from multiprocessing.synchronize import RLock
- return RLock()
-
-def Condition(lock=None):
- '''
- Returns a condition object
- '''
- from multiprocessing.synchronize import Condition
- return Condition(lock)
-
-def Semaphore(value=1):
- '''
- Returns a semaphore object
- '''
- from multiprocessing.synchronize import Semaphore
- return Semaphore(value)
-
-def BoundedSemaphore(value=1):
- '''
- Returns a bounded semaphore object
- '''
- from multiprocessing.synchronize import BoundedSemaphore
- return BoundedSemaphore(value)
-
-def Event():
- '''
- Returns an event object
- '''
- from multiprocessing.synchronize import Event
- return Event()
-
-def Barrier(parties, action=None, timeout=None):
- '''
- Returns a barrier object
- '''
- from multiprocessing.synchronize import Barrier
- return Barrier(parties, action, timeout)
-
-def Queue(maxsize=0):
- '''
- Returns a queue object
- '''
- from multiprocessing.queues import Queue
- return Queue(maxsize)
+SUBDEBUG = 5
+SUBWARNING = 25
-def JoinableQueue(maxsize=0):
- '''
- Returns a queue object
- '''
- from multiprocessing.queues import JoinableQueue
- return JoinableQueue(maxsize)
-
-def SimpleQueue():
- '''
- Returns a queue object
- '''
- from multiprocessing.queues import SimpleQueue
- return SimpleQueue()
-
-def Pool(processes=None, initializer=None, initargs=(), maxtasksperchild=None):
- '''
- Returns a process pool object
- '''
- from multiprocessing.pool import Pool
- return Pool(processes, initializer, initargs, maxtasksperchild)
-
-def RawValue(typecode_or_type, *args):
- '''
- Returns a shared object
- '''
- from multiprocessing.sharedctypes import RawValue
- return RawValue(typecode_or_type, *args)
-
-def RawArray(typecode_or_type, size_or_initializer):
- '''
- Returns a shared array
- '''
- from multiprocessing.sharedctypes import RawArray
- return RawArray(typecode_or_type, size_or_initializer)
-
-def Value(typecode_or_type, *args, lock=True):
- '''
- Returns a synchronized shared object
- '''
- from multiprocessing.sharedctypes import Value
- return Value(typecode_or_type, *args, lock=lock)
-
-def Array(typecode_or_type, size_or_initializer, *, lock=True):
- '''
- Returns a synchronized shared array
- '''
- from multiprocessing.sharedctypes import Array
- return Array(typecode_or_type, size_or_initializer, lock=lock)
-
-#
#
+# Alias for main module -- will be reset by bootstrapping child processes
#
-if sys.platform == 'win32':
-
- def set_executable(executable):
- '''
- Sets the path to a python.exe or pythonw.exe binary used to run
- child processes on Windows instead of sys.executable.
- Useful for people embedding Python.
- '''
- from multiprocessing.forking import set_executable
- set_executable(executable)
-
- __all__ += ['set_executable']
+if '__main__' in sys.modules:
+ sys.modules['__mp_main__'] = sys.modules['__main__']
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 22589d0..b4f81ba 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -12,22 +12,23 @@ __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
import io
import os
import sys
-import pickle
-import select
import socket
import struct
-import errno
import time
import tempfile
import itertools
import _multiprocessing
-from multiprocessing import current_process, AuthenticationError, BufferTooShort
-from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
-from multiprocessing.forking import ForkingPickler
+
+from . import reduction
+from . import util
+
+from . import AuthenticationError, BufferTooShort
+from .reduction import ForkingPickler
+
try:
import _winapi
- from _winapi import WAIT_OBJECT_0, WAIT_TIMEOUT, INFINITE
+ from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
except ImportError:
if sys.platform == 'win32':
raise
@@ -72,7 +73,7 @@ def arbitrary_address(family):
if family == 'AF_INET':
return ('localhost', 0)
elif family == 'AF_UNIX':
- return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
+ return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
elif family == 'AF_PIPE':
return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
(os.getpid(), next(_mmap_counter)))
@@ -132,22 +133,22 @@ class _ConnectionBase:
def _check_closed(self):
if self._handle is None:
- raise IOError("handle is closed")
+ raise OSError("handle is closed")
def _check_readable(self):
if not self._readable:
- raise IOError("connection is write-only")
+ raise OSError("connection is write-only")
def _check_writable(self):
if not self._writable:
- raise IOError("connection is read-only")
+ raise OSError("connection is read-only")
def _bad_message_length(self):
if self._writable:
self._readable = False
else:
self.close()
- raise IOError("bad message length")
+ raise OSError("bad message length")
@property
def closed(self):
@@ -202,9 +203,7 @@ class _ConnectionBase:
"""Send a (picklable) object"""
self._check_closed()
self._check_writable()
- buf = io.BytesIO()
- ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
- self._send_bytes(buf.getbuffer())
+ self._send_bytes(ForkingPickler.dumps(obj))
def recv_bytes(self, maxlength=None):
"""
@@ -249,7 +248,7 @@ class _ConnectionBase:
self._check_closed()
self._check_readable()
buf = self._recv_bytes()
- return pickle.loads(buf.getbuffer())
+ return ForkingPickler.loads(buf.getbuffer())
def poll(self, timeout=0.0):
"""Whether there is any input available to be read"""
@@ -317,7 +316,7 @@ if _winapi:
return f
elif err == _winapi.ERROR_MORE_DATA:
return self._get_more_data(ov, maxsize)
- except IOError as e:
+ except OSError as e:
if e.winerror == _winapi.ERROR_BROKEN_PIPE:
raise EOFError
else:
@@ -389,7 +388,7 @@ class Connection(_ConnectionBase):
if remaining == size:
raise EOFError
else:
- raise IOError("got end of file during message")
+ raise OSError("got end of file during message")
buf.write(chunk)
remaining -= n
return buf
@@ -459,7 +458,7 @@ class Listener(object):
Returns a `Connection` object.
'''
if self._listener is None:
- raise IOError('listener is closed')
+ raise OSError('listener is closed')
c = self._listener.accept()
if self._authkey:
deliver_challenge(c, self._authkey)
@@ -545,7 +544,9 @@ else:
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
_winapi.PIPE_WAIT,
- 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
+ 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
+ # default security descriptor: the handle cannot be inherited
+ _winapi.NULL
)
h2 = _winapi.CreateFile(
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
@@ -590,7 +591,7 @@ class SocketListener(object):
self._last_accepted = None
if family == 'AF_UNIX':
- self._unlink = Finalize(
+ self._unlink = util.Finalize(
self, os.unlink, args=(address,), exitpriority=0
)
else:
@@ -638,8 +639,8 @@ if sys.platform == 'win32':
self._handle_queue = [self._new_handle(first=True)]
self._last_accepted = None
- sub_debug('listener created with address=%r', self._address)
- self.close = Finalize(
+ util.sub_debug('listener created with address=%r', self._address)
+ self.close = util.Finalize(
self, PipeListener._finalize_pipe_listener,
args=(self._handle_queue, self._address), exitpriority=0
)
@@ -681,7 +682,7 @@ if sys.platform == 'win32':
@staticmethod
def _finalize_pipe_listener(queue, address):
- sub_debug('closing listener with address=%r', address)
+ util.sub_debug('closing listener with address=%r', address)
for handle in queue:
_winapi.CloseHandle(handle)
@@ -698,7 +699,7 @@ if sys.platform == 'win32':
0, _winapi.NULL, _winapi.OPEN_EXISTING,
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
)
- except WindowsError as e:
+ except OSError as e:
if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
_winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
raise
@@ -727,7 +728,7 @@ def deliver_challenge(connection, authkey):
assert isinstance(authkey, bytes)
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
- digest = hmac.new(authkey, message).digest()
+ digest = hmac.new(authkey, message, 'md5').digest()
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
@@ -741,7 +742,7 @@ def answer_challenge(connection, authkey):
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
- digest = hmac.new(authkey, message).digest()
+ digest = hmac.new(authkey, message, 'md5').digest()
connection.send_bytes(digest)
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
@@ -884,28 +885,15 @@ if sys.platform == 'win32':
else:
- if hasattr(select, 'poll'):
- def _poll(fds, timeout):
- if timeout is not None:
- timeout = int(timeout * 1000) # timeout is in milliseconds
- fd_map = {}
- pollster = select.poll()
- for fd in fds:
- pollster.register(fd, select.POLLIN)
- if hasattr(fd, 'fileno'):
- fd_map[fd.fileno()] = fd
- else:
- fd_map[fd] = fd
- ls = []
- for fd, event in pollster.poll(timeout):
- if event & select.POLLNVAL:
- raise ValueError('invalid file descriptor %i' % fd)
- ls.append(fd_map[fd])
- return ls
- else:
- def _poll(fds, timeout):
- return select.select(fds, [], [], timeout)[0]
+ import selectors
+ # poll/select have the advantage of not requiring any extra file
+ # descriptor, contrarily to epoll/kqueue (also, they require a single
+ # syscall).
+ if hasattr(selectors, 'PollSelector'):
+ _WaitSelector = selectors.PollSelector
+ else:
+ _WaitSelector = selectors.SelectSelector
def wait(object_list, timeout=None):
'''
@@ -913,34 +901,54 @@ else:
Returns list of those objects in object_list which are ready/readable.
'''
- if timeout is not None:
- if timeout <= 0:
- return _poll(object_list, 0)
- else:
- deadline = time.time() + timeout
- while True:
- try:
- return _poll(object_list, timeout)
- except OSError as e:
- if e.errno != errno.EINTR:
- raise
+ with _WaitSelector() as selector:
+ for obj in object_list:
+ selector.register(obj, selectors.EVENT_READ)
+
if timeout is not None:
- timeout = deadline - time.time()
+ deadline = time.time() + timeout
+
+ while True:
+ ready = selector.select(timeout)
+ if ready:
+ return [key.fileobj for (key, events) in ready]
+ else:
+ if timeout is not None:
+ timeout = deadline - time.time()
+ if timeout < 0:
+ return ready
#
# Make connection and socket objects sharable if possible
#
if sys.platform == 'win32':
- from . import reduction
- ForkingPickler.register(socket.socket, reduction.reduce_socket)
- ForkingPickler.register(Connection, reduction.reduce_connection)
- ForkingPickler.register(PipeConnection, reduction.reduce_pipe_connection)
+ def reduce_connection(conn):
+ handle = conn.fileno()
+ with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
+ from . import resource_sharer
+ ds = resource_sharer.DupSocket(s)
+ return rebuild_connection, (ds, conn.readable, conn.writable)
+ def rebuild_connection(ds, readable, writable):
+ sock = ds.detach()
+ return Connection(sock.detach(), readable, writable)
+ reduction.register(Connection, reduce_connection)
+
+ def reduce_pipe_connection(conn):
+ access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
+ (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
+ dh = reduction.DupHandle(conn.fileno(), access)
+ return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
+ def rebuild_pipe_connection(dh, readable, writable):
+ handle = dh.detach()
+ return PipeConnection(handle, readable, writable)
+ reduction.register(PipeConnection, reduce_pipe_connection)
+
else:
- try:
- from . import reduction
- except ImportError:
- pass
- else:
- ForkingPickler.register(socket.socket, reduction.reduce_socket)
- ForkingPickler.register(Connection, reduction.reduce_connection)
+ def reduce_connection(conn):
+ df = reduction.DupFd(conn.fileno())
+ return rebuild_connection, (df, conn.readable, conn.writable)
+ def rebuild_connection(df, readable, writable):
+ fd = df.detach()
+ return Connection(fd, readable, writable)
+ reduction.register(Connection, reduce_connection)
diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py
new file mode 100644
index 0000000..63849f9
--- /dev/null
+++ b/Lib/multiprocessing/context.py
@@ -0,0 +1,348 @@
+import os
+import sys
+import threading
+
+from . import process
+
+__all__ = [] # things are copied from here to __init__.py
+
+#
+# Exceptions
+#
+
+class ProcessError(Exception):
+ pass
+
+class BufferTooShort(ProcessError):
+ pass
+
+class TimeoutError(ProcessError):
+ pass
+
+class AuthenticationError(ProcessError):
+ pass
+
+#
+# Base type for contexts
+#
+
+class BaseContext(object):
+
+ ProcessError = ProcessError
+ BufferTooShort = BufferTooShort
+ TimeoutError = TimeoutError
+ AuthenticationError = AuthenticationError
+
+ current_process = staticmethod(process.current_process)
+ active_children = staticmethod(process.active_children)
+
+ def cpu_count(self):
+ '''Returns the number of CPUs in the system'''
+ num = os.cpu_count()
+ if num is None:
+ raise NotImplementedError('cannot determine number of cpus')
+ else:
+ return num
+
+ def Manager(self):
+ '''Returns a manager associated with a running server process
+
+ The managers methods such as `Lock()`, `Condition()` and `Queue()`
+ can be used to create shared objects.
+ '''
+ from .managers import SyncManager
+ m = SyncManager(ctx=self.get_context())
+ m.start()
+ return m
+
+ def Pipe(self, duplex=True):
+ '''Returns two connection object connected by a pipe'''
+ from .connection import Pipe
+ return Pipe(duplex)
+
+ def Lock(self):
+ '''Returns a non-recursive lock object'''
+ from .synchronize import Lock
+ return Lock(ctx=self.get_context())
+
+ def RLock(self):
+ '''Returns a recursive lock object'''
+ from .synchronize import RLock
+ return RLock(ctx=self.get_context())
+
+ def Condition(self, lock=None):
+ '''Returns a condition object'''
+ from .synchronize import Condition
+ return Condition(lock, ctx=self.get_context())
+
+ def Semaphore(self, value=1):
+ '''Returns a semaphore object'''
+ from .synchronize import Semaphore
+ return Semaphore(value, ctx=self.get_context())
+
+ def BoundedSemaphore(self, value=1):
+ '''Returns a bounded semaphore object'''
+ from .synchronize import BoundedSemaphore
+ return BoundedSemaphore(value, ctx=self.get_context())
+
+ def Event(self):
+ '''Returns an event object'''
+ from .synchronize import Event
+ return Event(ctx=self.get_context())
+
+ def Barrier(self, parties, action=None, timeout=None):
+ '''Returns a barrier object'''
+ from .synchronize import Barrier
+ return Barrier(parties, action, timeout, ctx=self.get_context())
+
+ def Queue(self, maxsize=0):
+ '''Returns a queue object'''
+ from .queues import Queue
+ return Queue(maxsize, ctx=self.get_context())
+
+ def JoinableQueue(self, maxsize=0):
+ '''Returns a queue object'''
+ from .queues import JoinableQueue
+ return JoinableQueue(maxsize, ctx=self.get_context())
+
+ def SimpleQueue(self):
+ '''Returns a queue object'''
+ from .queues import SimpleQueue
+ return SimpleQueue(ctx=self.get_context())
+
+ def Pool(self, processes=None, initializer=None, initargs=(),
+ maxtasksperchild=None):
+ '''Returns a process pool object'''
+ from .pool import Pool
+ return Pool(processes, initializer, initargs, maxtasksperchild,
+ context=self.get_context())
+
+ def RawValue(self, typecode_or_type, *args):
+ '''Returns a shared object'''
+ from .sharedctypes import RawValue
+ return RawValue(typecode_or_type, *args)
+
+ def RawArray(self, typecode_or_type, size_or_initializer):
+ '''Returns a shared array'''
+ from .sharedctypes import RawArray
+ return RawArray(typecode_or_type, size_or_initializer)
+
+ def Value(self, typecode_or_type, *args, lock=True):
+ '''Returns a synchronized shared object'''
+ from .sharedctypes import Value
+ return Value(typecode_or_type, *args, lock=lock,
+ ctx=self.get_context())
+
+ def Array(self, typecode_or_type, size_or_initializer, *, lock=True):
+ '''Returns a synchronized shared array'''
+ from .sharedctypes import Array
+ return Array(typecode_or_type, size_or_initializer, lock=lock,
+ ctx=self.get_context())
+
+ def freeze_support(self):
+ '''Check whether this is a fake forked process in a frozen executable.
+ If so then run code specified by commandline and exit.
+ '''
+ if sys.platform == 'win32' and getattr(sys, 'frozen', False):
+ from .spawn import freeze_support
+ freeze_support()
+
+ def get_logger(self):
+ '''Return package logger -- if it does not already exist then
+ it is created.
+ '''
+ from .util import get_logger
+ return get_logger()
+
+ def log_to_stderr(self, level=None):
+ '''Turn on logging and add a handler which prints to stderr'''
+ from .util import log_to_stderr
+ return log_to_stderr(level)
+
+ def allow_connection_pickling(self):
+ '''Install support for sending connections and sockets
+ between processes
+ '''
+ # This is undocumented. In previous versions of multiprocessing
+ # its only effect was to make socket objects inheritable on Windows.
+ from . import connection
+
+ def set_executable(self, executable):
+ '''Sets the path to a python.exe or pythonw.exe binary used to run
+ child processes instead of sys.executable when using the 'spawn'
+ start method. Useful for people embedding Python.
+ '''
+ from .spawn import set_executable
+ set_executable(executable)
+
+ def set_forkserver_preload(self, module_names):
+ '''Set list of module names to try to load in forkserver process.
+ This is really just a hint.
+ '''
+ from .forkserver import set_forkserver_preload
+ set_forkserver_preload(module_names)
+
+ def get_context(self, method=None):
+ if method is None:
+ return self
+ try:
+ ctx = _concrete_contexts[method]
+ except KeyError:
+ raise ValueError('cannot find context for %r' % method)
+ ctx._check_available()
+ return ctx
+
+ def get_start_method(self, allow_none=False):
+ return self._name
+
+ def set_start_method(self, method=None):
+ raise ValueError('cannot set start method of concrete context')
+
+ def _check_available(self):
+ pass
+
+#
+# Type of default context -- underlying context can be set at most once
+#
+
+class Process(process.BaseProcess):
+ _start_method = None
+ @staticmethod
+ def _Popen(process_obj):
+ return _default_context.get_context().Process._Popen(process_obj)
+
+class DefaultContext(BaseContext):
+ Process = Process
+
+ def __init__(self, context):
+ self._default_context = context
+ self._actual_context = None
+
+ def get_context(self, method=None):
+ if method is None:
+ if self._actual_context is None:
+ self._actual_context = self._default_context
+ return self._actual_context
+ else:
+ return super().get_context(method)
+
+ def set_start_method(self, method, force=False):
+ if self._actual_context is not None and not force:
+ raise RuntimeError('context has already been set')
+ if method is None and force:
+ self._actual_context = None
+ return
+ self._actual_context = self.get_context(method)
+
+ def get_start_method(self, allow_none=False):
+ if self._actual_context is None:
+ if allow_none:
+ return None
+ self._actual_context = self._default_context
+ return self._actual_context._name
+
+ def get_all_start_methods(self):
+ if sys.platform == 'win32':
+ return ['spawn']
+ else:
+ from . import reduction
+ if reduction.HAVE_SEND_HANDLE:
+ return ['fork', 'spawn', 'forkserver']
+ else:
+ return ['fork', 'spawn']
+
+DefaultContext.__all__ = list(x for x in dir(DefaultContext) if x[0] != '_')
+
+#
+# Context types for fixed start method
+#
+
+if sys.platform != 'win32':
+
+ class ForkProcess(process.BaseProcess):
+ _start_method = 'fork'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_fork import Popen
+ return Popen(process_obj)
+
+ class SpawnProcess(process.BaseProcess):
+ _start_method = 'spawn'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_spawn_posix import Popen
+ return Popen(process_obj)
+
+ class ForkServerProcess(process.BaseProcess):
+ _start_method = 'forkserver'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_forkserver import Popen
+ return Popen(process_obj)
+
+ class ForkContext(BaseContext):
+ _name = 'fork'
+ Process = ForkProcess
+
+ class SpawnContext(BaseContext):
+ _name = 'spawn'
+ Process = SpawnProcess
+
+ class ForkServerContext(BaseContext):
+ _name = 'forkserver'
+ Process = ForkServerProcess
+ def _check_available(self):
+ from . import reduction
+ if not reduction.HAVE_SEND_HANDLE:
+ raise ValueError('forkserver start method not available')
+
+ _concrete_contexts = {
+ 'fork': ForkContext(),
+ 'spawn': SpawnContext(),
+ 'forkserver': ForkServerContext(),
+ }
+ _default_context = DefaultContext(_concrete_contexts['fork'])
+
+else:
+
+ class SpawnProcess(process.BaseProcess):
+ _start_method = 'spawn'
+ @staticmethod
+ def _Popen(process_obj):
+ from .popen_spawn_win32 import Popen
+ return Popen(process_obj)
+
+ class SpawnContext(BaseContext):
+ _name = 'spawn'
+ Process = SpawnProcess
+
+ _concrete_contexts = {
+ 'spawn': SpawnContext(),
+ }
+ _default_context = DefaultContext(_concrete_contexts['spawn'])
+
+#
+# Force the start method
+#
+
+def _force_start_method(method):
+ _default_context._actual_context = _concrete_contexts[method]
+
+#
+# Check that the current thread is spawning a child process
+#
+
+_tls = threading.local()
+
+def get_spawning_popen():
+ return getattr(_tls, 'spawning_popen', None)
+
+def set_spawning_popen(popen):
+ _tls.spawning_popen = popen
+
+def assert_spawning(obj):
+ if get_spawning_popen() is None:
+ raise RuntimeError(
+ '%s objects should only be shared between processes'
+ ' through inheritance' % type(obj).__name__
+ )
diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py
index e31fc61..97f7af7 100644
--- a/Lib/multiprocessing/dummy/__init__.py
+++ b/Lib/multiprocessing/dummy/__init__.py
@@ -4,32 +4,7 @@
# multiprocessing/dummy/__init__.py
#
# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-# used to endorse or promote products derived from this software
-# without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
#
__all__ = [
@@ -47,7 +22,7 @@ import sys
import weakref
import array
-from multiprocessing.dummy.connection import Pipe
+from .connection import Pipe
from threading import Lock, RLock, Semaphore, BoundedSemaphore
from threading import Event, Condition, Barrier
from queue import Queue
@@ -138,7 +113,7 @@ def shutdown():
pass
def Pool(processes=None, initializer=None, initargs=()):
- from multiprocessing.pool import ThreadPool
+ from ..pool import ThreadPool
return ThreadPool(processes, initializer, initargs)
JoinableQueue = Queue
diff --git a/Lib/multiprocessing/dummy/connection.py b/Lib/multiprocessing/dummy/connection.py
index 874ec8e..694ef96 100644
--- a/Lib/multiprocessing/dummy/connection.py
+++ b/Lib/multiprocessing/dummy/connection.py
@@ -4,32 +4,7 @@
# multiprocessing/dummy/connection.py
#
# Copyright (c) 2006-2008, R Oudkerk
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-# 3. Neither the name of author nor the names of any contributors may be
-# used to endorse or promote products derived from this software
-# without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
+# Licensed to PSF under a Contributor Agreement.
#
__all__ = [ 'Client', 'Listener', 'Pipe' ]
diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py
deleted file mode 100644
index 449978a..0000000
--- a/Lib/multiprocessing/forking.py
+++ /dev/null
@@ -1,474 +0,0 @@
-#
-# Module for starting a process object using os.fork() or CreateProcess()
-#
-# multiprocessing/forking.py
-#
-# Copyright (c) 2006-2008, R Oudkerk
-# Licensed to PSF under a Contributor Agreement.
-#
-
-import os
-import sys
-import signal
-import errno
-
-from multiprocessing import util, process
-
-__all__ = ['Popen', 'assert_spawning', 'duplicate', 'close', 'ForkingPickler']
-
-#
-# Check that the current thread is spawning a child process
-#
-
-def assert_spawning(self):
- if not Popen.thread_is_spawning():
- raise RuntimeError(
- '%s objects should only be shared between processes'
- ' through inheritance' % type(self).__name__
- )
-
-#
-# Try making some callable types picklable
-#
-
-from pickle import Pickler
-from copyreg import dispatch_table
-
-class ForkingPickler(Pickler):
- _extra_reducers = {}
- def __init__(self, *args):
- Pickler.__init__(self, *args)
- self.dispatch_table = dispatch_table.copy()
- self.dispatch_table.update(self._extra_reducers)
- @classmethod
- def register(cls, type, reduce):
- cls._extra_reducers[type] = reduce
-
-def _reduce_method(m):
- if m.__self__ is None:
- return getattr, (m.__class__, m.__func__.__name__)
- else:
- return getattr, (m.__self__, m.__func__.__name__)
-class _C:
- def f(self):
- pass
-ForkingPickler.register(type(_C().f), _reduce_method)
-
-
-def _reduce_method_descriptor(m):
- return getattr, (m.__objclass__, m.__name__)
-ForkingPickler.register(type(list.append), _reduce_method_descriptor)
-ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
-
-try:
- from functools import partial
-except ImportError:
- pass
-else:
- def _reduce_partial(p):
- return _rebuild_partial, (p.func, p.args, p.keywords or {})
- def _rebuild_partial(func, args, keywords):
- return partial(func, *args, **keywords)
- ForkingPickler.register(partial, _reduce_partial)
-
-#
-# Unix
-#
-
-if sys.platform != 'win32':
- duplicate = os.dup
- close = os.close
-
- #
- # We define a Popen class similar to the one from subprocess, but
- # whose constructor takes a process object as its argument.
- #
-
- class Popen(object):
-
- def __init__(self, process_obj):
- sys.stdout.flush()
- sys.stderr.flush()
- self.returncode = None
-
- r, w = os.pipe()
- self.sentinel = r
-
- self.pid = os.fork()
- if self.pid == 0:
- os.close(r)
- if 'random' in sys.modules:
- import random
- random.seed()
- code = process_obj._bootstrap()
- os._exit(code)
-
- # `w` will be closed when the child exits, at which point `r`
- # will become ready for reading (using e.g. select()).
- os.close(w)
- util.Finalize(self, os.close, (r,))
-
- def poll(self, flag=os.WNOHANG):
- if self.returncode is None:
- while True:
- try:
- pid, sts = os.waitpid(self.pid, flag)
- except os.error as e:
- if e.errno == errno.EINTR:
- continue
- # Child process not yet created. See #1731717
- # e.errno == errno.ECHILD == 10
- return None
- else:
- break
- if pid == self.pid:
- if os.WIFSIGNALED(sts):
- self.returncode = -os.WTERMSIG(sts)
- else:
- assert os.WIFEXITED(sts)
- self.returncode = os.WEXITSTATUS(sts)
- return self.returncode
-
- def wait(self, timeout=None):
- if self.returncode is None:
- if timeout is not None:
- from multiprocessing.connection import wait
- if not wait([self.sentinel], timeout):
- return None
- # This shouldn't block if wait() returned successfully.
- return self.poll(os.WNOHANG if timeout == 0.0 else 0)
- return self.returncode
-
- def terminate(self):
- if self.returncode is None:
- try:
- os.kill(self.pid, signal.SIGTERM)
- except OSError:
- if self.wait(timeout=0.1) is None:
- raise
-
- @staticmethod
- def thread_is_spawning():
- return False
-
-#
-# Windows
-#
-
-else:
- import _thread
- import msvcrt
- import _winapi
-
- from pickle import load, HIGHEST_PROTOCOL
-
- def dump(obj, file, protocol=None):
- ForkingPickler(file, protocol).dump(obj)
-
- #
- #
- #
-
- TERMINATE = 0x10000
- WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
- WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
-
- close = _winapi.CloseHandle
-
- #
- # _python_exe is the assumed path to the python executable.
- # People embedding Python want to modify it.
- #
-
- if WINSERVICE:
- _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
- else:
- _python_exe = sys.executable
-
- def set_executable(exe):
- global _python_exe
- _python_exe = exe
-
- #
- #
- #
-
- def duplicate(handle, target_process=None, inheritable=False):
- if target_process is None:
- target_process = _winapi.GetCurrentProcess()
- return _winapi.DuplicateHandle(
- _winapi.GetCurrentProcess(), handle, target_process,
- 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS
- )
-
- #
- # We define a Popen class similar to the one from subprocess, but
- # whose constructor takes a process object as its argument.
- #
-
- class Popen(object):
- '''
- Start a subprocess to run the code of a process object
- '''
- _tls = _thread._local()
-
- def __init__(self, process_obj):
- cmd = ' '.join('"%s"' % x for x in get_command_line())
- prep_data = get_preparation_data(process_obj._name)
-
- # create pipe for communication with child
- rfd, wfd = os.pipe()
-
- # get handle for read end of the pipe and make it inheritable
- rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
- os.close(rfd)
-
- with open(wfd, 'wb', closefd=True) as to_child:
- # start process
- try:
- hp, ht, pid, tid = _winapi.CreateProcess(
- _python_exe, cmd + (' %s' % rhandle),
- None, None, 1, 0, None, None, None
- )
- _winapi.CloseHandle(ht)
- finally:
- close(rhandle)
-
- # set attributes of self
- self.pid = pid
- self.returncode = None
- self._handle = hp
- self.sentinel = int(hp)
- util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
-
- # send information to child
- Popen._tls.process_handle = int(hp)
- try:
- dump(prep_data, to_child, HIGHEST_PROTOCOL)
- dump(process_obj, to_child, HIGHEST_PROTOCOL)
- finally:
- del Popen._tls.process_handle
-
- @staticmethod
- def thread_is_spawning():
- return getattr(Popen._tls, 'process_handle', None) is not None
-
- @staticmethod
- def duplicate_for_child(handle):
- return duplicate(handle, Popen._tls.process_handle)
-
- def wait(self, timeout=None):
- if self.returncode is None:
- if timeout is None:
- msecs = _winapi.INFINITE
- else:
- msecs = max(0, int(timeout * 1000 + 0.5))
-
- res = _winapi.WaitForSingleObject(int(self._handle), msecs)
- if res == _winapi.WAIT_OBJECT_0:
- code = _winapi.GetExitCodeProcess(self._handle)
- if code == TERMINATE:
- code = -signal.SIGTERM
- self.returncode = code
-
- return self.returncode
-
- def poll(self):
- return self.wait(timeout=0)
-
- def terminate(self):
- if self.returncode is None:
- try:
- _winapi.TerminateProcess(int(self._handle), TERMINATE)
- except OSError:
- if self.wait(timeout=1.0) is None:
- raise
-
- #
- #
- #
-
- def is_forking(argv):
- '''
- Return whether commandline indicates we are forking
- '''
- if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
- assert len(argv) == 3
- return True
- else:
- return False
-
-
- def freeze_support():
- '''
- Run code for process object if this in not the main process
- '''
- if is_forking(sys.argv):
- main()
- sys.exit()
-
-
- def get_command_line():
- '''
- Returns prefix of command line used for spawning a child process
- '''
- if getattr(process.current_process(), '_inheriting', False):
- raise RuntimeError('''
- Attempt to start a new process before the current process
- has finished its bootstrapping phase.
-
- This probably means that you are on Windows and you have
- forgotten to use the proper idiom in the main module:
-
- if __name__ == '__main__':
- freeze_support()
- ...
-
- The "freeze_support()" line can be omitted if the program
- is not going to be frozen to produce a Windows executable.''')
-
- if getattr(sys, 'frozen', False):
- return [sys.executable, '--multiprocessing-fork']
- else:
- prog = 'from multiprocessing.forking import main; main()'
- opts = util._args_from_interpreter_flags()
- return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
-
-
- def main():
- '''
- Run code specified by data received over pipe
- '''
- assert is_forking(sys.argv)
-
- handle = int(sys.argv[-1])
- fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
- from_parent = os.fdopen(fd, 'rb')
-
- process.current_process()._inheriting = True
- preparation_data = load(from_parent)
- prepare(preparation_data)
- self = load(from_parent)
- process.current_process()._inheriting = False
-
- from_parent.close()
-
- exitcode = self._bootstrap()
- sys.exit(exitcode)
-
-
- def get_preparation_data(name):
- '''
- Return info about parent needed by child to unpickle process object
- '''
- from .util import _logger, _log_to_stderr
-
- d = dict(
- name=name,
- sys_path=sys.path,
- sys_argv=sys.argv,
- log_to_stderr=_log_to_stderr,
- orig_dir=process.ORIGINAL_DIR,
- authkey=process.current_process().authkey,
- )
-
- if _logger is not None:
- d['log_level'] = _logger.getEffectiveLevel()
-
- if not WINEXE and not WINSERVICE:
- main_path = getattr(sys.modules['__main__'], '__file__', None)
- if not main_path and sys.argv[0] not in ('', '-c'):
- main_path = sys.argv[0]
- if main_path is not None:
- if not os.path.isabs(main_path) and \
- process.ORIGINAL_DIR is not None:
- main_path = os.path.join(process.ORIGINAL_DIR, main_path)
- d['main_path'] = os.path.normpath(main_path)
-
- return d
-
-#
-# Prepare current process
-#
-
-old_main_modules = []
-
-def prepare(data):
- '''
- Try to get current process ready to unpickle process object
- '''
- old_main_modules.append(sys.modules['__main__'])
-
- if 'name' in data:
- process.current_process().name = data['name']
-
- if 'authkey' in data:
- process.current_process()._authkey = data['authkey']
-
- if 'log_to_stderr' in data and data['log_to_stderr']:
- util.log_to_stderr()
-
- if 'log_level' in data:
- util.get_logger().setLevel(data['log_level'])
-
- if 'sys_path' in data:
- sys.path = data['sys_path']
-
- if 'sys_argv' in data:
- sys.argv = data['sys_argv']
-
- if 'dir' in data:
- os.chdir(data['dir'])
-
- if 'orig_dir' in data:
- process.ORIGINAL_DIR = data['orig_dir']
-
- if 'main_path' in data:
- # XXX (ncoghlan): The following code makes several bogus
- # assumptions regarding the relationship between __file__
- # and a module's real name. See PEP 302 and issue #10845
- main_path = data['main_path']
- main_name = os.path.splitext(os.path.basename(main_path))[0]
- if main_name == '__init__':
- main_name = os.path.basename(os.path.dirname(main_path))
-
- if main_name == '__main__':
- main_module = sys.modules['__main__']
- main_module.__file__ = main_path
- elif main_name != 'ipython':
- # Main modules not actually called __main__.py may
- # contain additional code that should still be executed
- import imp
-
- if main_path is None:
- dirs = None
- elif os.path.basename(main_path).startswith('__init__.py'):
- dirs = [os.path.dirname(os.path.dirname(main_path))]
- else:
- dirs = [os.path.dirname(main_path)]
-
- assert main_name not in sys.modules, main_name
- file, path_name, etc = imp.find_module(main_name, dirs)
- try:
- # We would like to do "imp.load_module('__main__', ...)"
- # here. However, that would cause 'if __name__ ==
- # "__main__"' clauses to be executed.
- main_module = imp.load_module(
- '__parents_main__', file, path_name, etc
- )
- finally:
- if file:
- file.close()
-
- sys.modules['__main__'] = main_module
- main_module.__name__ = '__main__'
-
- # Try to make the potentially picklable objects in
- # sys.modules['__main__'] realize they are in the main
- # module -- somewhat ugly.
- for obj in list(main_module.__dict__.values()):
- try:
- if obj.__module__ == '__parents_main__':
- obj.__module__ = '__main__'
- except Exception:
- pass
diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py
new file mode 100644
index 0000000..387517e
--- /dev/null
+++ b/Lib/multiprocessing/forkserver.py
@@ -0,0 +1,267 @@
+import errno
+import os
+import selectors
+import signal
+import socket
+import struct
+import sys
+import threading
+
+from . import connection
+from . import process
+from . import reduction
+from . import semaphore_tracker
+from . import spawn
+from . import util
+
+__all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
+ 'set_forkserver_preload']
+
+#
+#
+#
+
+MAXFDS_TO_SEND = 256
+UNSIGNED_STRUCT = struct.Struct('Q') # large enough for pid_t
+
+#
+# Forkserver class
+#
+
+class ForkServer(object):
+
+ def __init__(self):
+ self._forkserver_address = None
+ self._forkserver_alive_fd = None
+ self._inherited_fds = None
+ self._lock = threading.Lock()
+ self._preload_modules = ['__main__']
+
+ def set_forkserver_preload(self, modules_names):
+ '''Set list of module names to try to load in forkserver process.'''
+ if not all(type(mod) is str for mod in self._preload_modules):
+ raise TypeError('module_names must be a list of strings')
+ self._preload_modules = modules_names
+
+ def get_inherited_fds(self):
+ '''Return list of fds inherited from parent process.
+
+ This returns None if the current process was not started by fork
+ server.
+ '''
+ return self._inherited_fds
+
+ def connect_to_new_process(self, fds):
+ '''Request forkserver to create a child process.
+
+ Returns a pair of fds (status_r, data_w). The calling process can read
+ the child process's pid and (eventually) its returncode from status_r.
+ The calling process should write to data_w the pickled preparation and
+ process data.
+ '''
+ self.ensure_running()
+ if len(fds) + 4 >= MAXFDS_TO_SEND:
+ raise ValueError('too many fds')
+ with socket.socket(socket.AF_UNIX) as client:
+ client.connect(self._forkserver_address)
+ parent_r, child_w = os.pipe()
+ child_r, parent_w = os.pipe()
+ allfds = [child_r, child_w, self._forkserver_alive_fd,
+ semaphore_tracker.getfd()]
+ allfds += fds
+ try:
+ reduction.sendfds(client, allfds)
+ return parent_r, parent_w
+ except:
+ os.close(parent_r)
+ os.close(parent_w)
+ raise
+ finally:
+ os.close(child_r)
+ os.close(child_w)
+
+ def ensure_running(self):
+ '''Make sure that a fork server is running.
+
+ This can be called from any process. Note that usually a child
+ process will just reuse the forkserver started by its parent, so
+ ensure_running() will do nothing.
+ '''
+ with self._lock:
+ semaphore_tracker.ensure_running()
+ if self._forkserver_alive_fd is not None:
+ return
+
+ cmd = ('from multiprocessing.forkserver import main; ' +
+ 'main(%d, %d, %r, **%r)')
+
+ if self._preload_modules:
+ desired_keys = {'main_path', 'sys_path'}
+ data = spawn.get_preparation_data('ignore')
+ data = dict((x,y) for (x,y) in data.items()
+ if x in desired_keys)
+ else:
+ data = {}
+
+ with socket.socket(socket.AF_UNIX) as listener:
+ address = connection.arbitrary_address('AF_UNIX')
+ listener.bind(address)
+ os.chmod(address, 0o600)
+ listener.listen(100)
+
+ # all client processes own the write end of the "alive" pipe;
+ # when they all terminate the read end becomes ready.
+ alive_r, alive_w = os.pipe()
+ try:
+ fds_to_pass = [listener.fileno(), alive_r]
+ cmd %= (listener.fileno(), alive_r, self._preload_modules,
+ data)
+ exe = spawn.get_executable()
+ args = [exe] + util._args_from_interpreter_flags()
+ args += ['-c', cmd]
+ pid = util.spawnv_passfds(exe, args, fds_to_pass)
+ except:
+ os.close(alive_w)
+ raise
+ finally:
+ os.close(alive_r)
+ self._forkserver_address = address
+ self._forkserver_alive_fd = alive_w
+
+#
+#
+#
+
+def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
+ '''Run forkserver.'''
+ if preload:
+ if '__main__' in preload and main_path is not None:
+ process.current_process()._inheriting = True
+ try:
+ spawn.import_main_path(main_path)
+ finally:
+ del process.current_process()._inheriting
+ for modname in preload:
+ try:
+ __import__(modname)
+ except ImportError:
+ pass
+
+ # close sys.stdin
+ if sys.stdin is not None:
+ try:
+ sys.stdin.close()
+ sys.stdin = open(os.devnull)
+ except (OSError, ValueError):
+ pass
+
+ # ignoring SIGCHLD means no need to reap zombie processes
+ handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN)
+ with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
+ selectors.DefaultSelector() as selector:
+ _forkserver._forkserver_address = listener.getsockname()
+
+ selector.register(listener, selectors.EVENT_READ)
+ selector.register(alive_r, selectors.EVENT_READ)
+
+ while True:
+ try:
+ while True:
+ rfds = [key.fileobj for (key, events) in selector.select()]
+ if rfds:
+ break
+
+ if alive_r in rfds:
+ # EOF because no more client processes left
+ assert os.read(alive_r, 1) == b''
+ raise SystemExit
+
+ assert listener in rfds
+ with listener.accept()[0] as s:
+ code = 1
+ if os.fork() == 0:
+ try:
+ _serve_one(s, listener, alive_r, handler)
+ except Exception:
+ sys.excepthook(*sys.exc_info())
+ sys.stderr.flush()
+ finally:
+ os._exit(code)
+
+ except InterruptedError:
+ pass
+ except OSError as e:
+ if e.errno != errno.ECONNABORTED:
+ raise
+
+def _serve_one(s, listener, alive_r, handler):
+ # close unnecessary stuff and reset SIGCHLD handler
+ listener.close()
+ os.close(alive_r)
+ signal.signal(signal.SIGCHLD, handler)
+
+ # receive fds from parent process
+ fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
+ s.close()
+ assert len(fds) <= MAXFDS_TO_SEND
+ (child_r, child_w, _forkserver._forkserver_alive_fd,
+ stfd, *_forkserver._inherited_fds) = fds
+ semaphore_tracker._semaphore_tracker._fd = stfd
+
+ # send pid to client processes
+ write_unsigned(child_w, os.getpid())
+
+ # reseed random number generator
+ if 'random' in sys.modules:
+ import random
+ random.seed()
+
+ # run process object received over pipe
+ code = spawn._main(child_r)
+
+ # write the exit code to the pipe
+ write_unsigned(child_w, code)
+
+#
+# Read and write unsigned numbers
+#
+
+def read_unsigned(fd):
+ data = b''
+ length = UNSIGNED_STRUCT.size
+ while len(data) < length:
+ while True:
+ try:
+ s = os.read(fd, length - len(data))
+ except InterruptedError:
+ pass
+ else:
+ break
+ if not s:
+ raise EOFError('unexpected EOF')
+ data += s
+ return UNSIGNED_STRUCT.unpack(data)[0]
+
+def write_unsigned(fd, n):
+ msg = UNSIGNED_STRUCT.pack(n)
+ while msg:
+ while True:
+ try:
+ nbytes = os.write(fd, msg)
+ except InterruptedError:
+ pass
+ else:
+ break
+ if nbytes == 0:
+ raise RuntimeError('should not get here')
+ msg = msg[nbytes:]
+
+#
+#
+#
+
+_forkserver = ForkServer()
+ensure_running = _forkserver.ensure_running
+get_inherited_fds = _forkserver.get_inherited_fds
+connect_to_new_process = _forkserver.connect_to_new_process
+set_forkserver_preload = _forkserver.set_forkserver_preload
diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py
index e63fdb8..344a45f 100644
--- a/Lib/multiprocessing/heap.py
+++ b/Lib/multiprocessing/heap.py
@@ -11,12 +11,12 @@ import bisect
import mmap
import os
import sys
+import tempfile
import threading
-import itertools
-import _multiprocessing
-from multiprocessing.util import Finalize, info
-from multiprocessing.forking import assert_spawning
+from . import context
+from . import reduction
+from . import util
__all__ = ['BufferWrapper']
@@ -30,17 +30,25 @@ if sys.platform == 'win32':
class Arena(object):
- _counter = itertools.count()
+ _rand = tempfile._RandomNameSequence()
def __init__(self, size):
self.size = size
- self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
- self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
- assert _winapi.GetLastError() == 0, 'tagname already in use'
+ for i in range(100):
+ name = 'pym-%d-%s' % (os.getpid(), next(self._rand))
+ buf = mmap.mmap(-1, size, tagname=name)
+ if _winapi.GetLastError() == 0:
+ break
+ # We have reopened a preexisting mmap.
+ buf.close()
+ else:
+ raise FileExistsError('Cannot find name for new mmap')
+ self.name = name
+ self.buffer = buf
self._state = (self.size, self.name)
def __getstate__(self):
- assert_spawning(self)
+ context.assert_spawning(self)
return self._state
def __setstate__(self, state):
@@ -52,10 +60,28 @@ else:
class Arena(object):
- def __init__(self, size):
- self.buffer = mmap.mmap(-1, size)
+ def __init__(self, size, fd=-1):
self.size = size
- self.name = None
+ self.fd = fd
+ if fd == -1:
+ self.fd, name = tempfile.mkstemp(
+ prefix='pym-%d-'%os.getpid(), dir=util.get_temp_dir())
+ os.unlink(name)
+ util.Finalize(self, os.close, (self.fd,))
+ with open(self.fd, 'wb', closefd=False) as f:
+ f.write(b'\0'*size)
+ self.buffer = mmap.mmap(self.fd, self.size)
+
+ def reduce_arena(a):
+ if a.fd == -1:
+ raise ValueError('Arena is unpicklable because '
+ 'forking was enabled when it was created')
+ return rebuild_arena, (a.size, reduction.DupFd(a.fd))
+
+ def rebuild_arena(size, dupfd):
+ return Arena(size, dupfd.detach())
+
+ reduction.register(Arena, reduce_arena)
#
# Class allowing allocation of chunks of memory from arenas
@@ -90,7 +116,7 @@ class Heap(object):
if i == len(self._lengths):
length = self._roundup(max(self._size, size), mmap.PAGESIZE)
self._size *= 2
- info('allocating a new mmap of length %d', length)
+ util.info('allocating a new mmap of length %d', length)
arena = Arena(length)
self._arenas.append(arena)
return (arena, 0, length)
@@ -216,7 +242,7 @@ class BufferWrapper(object):
assert 0 <= size < sys.maxsize
block = BufferWrapper._heap.malloc(size)
self._state = (block, size)
- Finalize(self, BufferWrapper._heap.free, args=(block,))
+ util.Finalize(self, BufferWrapper._heap.free, args=(block,))
def create_memoryview(self):
(arena, start, stop), size = self._state
diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py
index 96056b0..66d46fc 100644
--- a/Lib/multiprocessing/managers.py
+++ b/Lib/multiprocessing/managers.py
@@ -19,11 +19,16 @@ import threading
import array
import queue
-from traceback import format_exc
-from multiprocessing import Process, current_process, active_children, Pool, util, connection
-from multiprocessing.process import AuthenticationString
-from multiprocessing.forking import Popen, ForkingPickler
from time import time as _time
+from traceback import format_exc
+
+from . import connection
+from . import context
+from . import pool
+from . import process
+from . import reduction
+from . import util
+from . import get_context
#
# Register some things for pickling
@@ -31,16 +36,14 @@ from time import time as _time
def reduce_array(a):
return array.array, (a.typecode, a.tobytes())
-ForkingPickler.register(array.array, reduce_array)
+reduction.register(array.array, reduce_array)
view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
if view_types[0] is not list: # only needed in Py3.0
def rebuild_as_list(obj):
return list, (list(obj),)
for view_type in view_types:
- ForkingPickler.register(view_type, rebuild_as_list)
- import copyreg
- copyreg.pickle(view_type, rebuild_as_list)
+ reduction.register(view_type, rebuild_as_list)
#
# Type for identifying shared objects
@@ -130,7 +133,7 @@ class Server(object):
def __init__(self, registry, address, authkey, serializer):
assert isinstance(authkey, bytes)
self.registry = registry
- self.authkey = AuthenticationString(authkey)
+ self.authkey = process.AuthenticationString(authkey)
Listener, Client = listener_client[serializer]
# do authentication later
@@ -146,7 +149,7 @@ class Server(object):
Run the server forever
'''
self.stop_event = threading.Event()
- current_process()._manager_server = self
+ process.current_process()._manager_server = self
try:
accepter = threading.Thread(target=self.accepter)
accepter.daemon = True
@@ -167,7 +170,7 @@ class Server(object):
while True:
try:
c = self.listener.accept()
- except (OSError, IOError):
+ except OSError:
continue
t = threading.Thread(target=self.handle_request, args=(c,))
t.daemon = True
@@ -436,15 +439,17 @@ class BaseManager(object):
_registry = {}
_Server = Server
- def __init__(self, address=None, authkey=None, serializer='pickle'):
+ def __init__(self, address=None, authkey=None, serializer='pickle',
+ ctx=None):
if authkey is None:
- authkey = current_process().authkey
+ authkey = process.current_process().authkey
self._address = address # XXX not final address if eg ('', 0)
- self._authkey = AuthenticationString(authkey)
+ self._authkey = process.AuthenticationString(authkey)
self._state = State()
self._state.value = State.INITIAL
self._serializer = serializer
self._Listener, self._Client = listener_client[serializer]
+ self._ctx = ctx or get_context()
def get_server(self):
'''
@@ -476,7 +481,7 @@ class BaseManager(object):
reader, writer = connection.Pipe(duplex=False)
# spawn process which runs a server
- self._process = Process(
+ self._process = self._ctx.Process(
target=type(self)._run_server,
args=(self._registry, self._address, self._authkey,
self._serializer, writer, initializer, initargs),
@@ -691,11 +696,11 @@ class BaseProxy(object):
self._Client = listener_client[serializer][1]
if authkey is not None:
- self._authkey = AuthenticationString(authkey)
+ self._authkey = process.AuthenticationString(authkey)
elif self._manager is not None:
self._authkey = self._manager._authkey
else:
- self._authkey = current_process().authkey
+ self._authkey = process.current_process().authkey
if incref:
self._incref()
@@ -704,7 +709,7 @@ class BaseProxy(object):
def _connect(self):
util.debug('making connection to manager')
- name = current_process().name
+ name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
conn = self._Client(self._token.address, authkey=self._authkey)
@@ -798,7 +803,7 @@ class BaseProxy(object):
def __reduce__(self):
kwds = {}
- if Popen.thread_is_spawning():
+ if context.get_spawning_popen() is not None:
kwds['authkey'] = self._authkey
if getattr(self, '_isauto', False):
@@ -835,14 +840,14 @@ def RebuildProxy(func, token, serializer, kwds):
If possible the shared object is returned, or otherwise a proxy for it.
'''
- server = getattr(current_process(), '_manager_server', None)
+ server = getattr(process.current_process(), '_manager_server', None)
if server and server.address == token.address:
return server.id_to_obj[token.id][0]
else:
incref = (
kwds.pop('incref', True) and
- not getattr(current_process(), '_inheriting', False)
+ not getattr(process.current_process(), '_inheriting', False)
)
return func(token, serializer, incref=incref, **kwds)
@@ -889,7 +894,7 @@ def AutoProxy(token, serializer, manager=None, authkey=None,
if authkey is None and manager is not None:
authkey = manager._authkey
if authkey is None:
- authkey = current_process().authkey
+ authkey = process.current_process().authkey
ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
@@ -1072,17 +1077,22 @@ ArrayProxy = MakeProxyType('ArrayProxy', (
))
-PoolProxy = MakeProxyType('PoolProxy', (
+BasePoolProxy = MakeProxyType('PoolProxy', (
'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
- 'map', 'map_async', 'starmap', 'starmap_async', 'terminate'
+ 'map', 'map_async', 'starmap', 'starmap_async', 'terminate',
))
-PoolProxy._method_to_typeid_ = {
+BasePoolProxy._method_to_typeid_ = {
'apply_async': 'AsyncResult',
'map_async': 'AsyncResult',
'starmap_async': 'AsyncResult',
'imap': 'Iterator',
'imap_unordered': 'Iterator'
}
+class PoolProxy(BasePoolProxy):
+ def __enter__(self):
+ return self
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.terminate()
#
# Definition of SyncManager
@@ -1109,7 +1119,7 @@ SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
AcquirerProxy)
SyncManager.register('Condition', threading.Condition, ConditionProxy)
SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
-SyncManager.register('Pool', Pool, PoolProxy)
+SyncManager.register('Pool', pool.Pool, PoolProxy)
SyncManager.register('list', list, ListProxy)
SyncManager.register('dict', dict, DictProxy)
SyncManager.register('Value', Value, ValueProxy)
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
index 0f2dab4..8832a5c 100644
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -7,7 +7,7 @@
# Licensed to PSF under a Contributor Agreement.
#
-__all__ = ['Pool']
+__all__ = ['Pool', 'ThreadPool']
#
# Imports
@@ -17,10 +17,14 @@ import threading
import queue
import itertools
import collections
+import os
import time
+import traceback
-from multiprocessing import Process, cpu_count, TimeoutError
-from multiprocessing.util import Finalize, debug
+# If threading is available then ThreadPool should be provided. Therefore
+# we avoid top-level imports which are liable to fail on some systems.
+from . import util
+from . import get_context, TimeoutError
#
# Constants representing the state of a pool
@@ -43,6 +47,29 @@ def starmapstar(args):
return list(itertools.starmap(args[0], args[1]))
#
+# Hack to embed stringification of remote traceback in local traceback
+#
+
+class RemoteTraceback(Exception):
+ def __init__(self, tb):
+ self.tb = tb
+ def __str__(self):
+ return self.tb
+
+class ExceptionWithTraceback:
+ def __init__(self, exc, tb):
+ tb = traceback.format_exception(type(exc), exc, tb)
+ tb = ''.join(tb)
+ self.exc = exc
+ self.tb = '\n"""\n%s"""' % tb
+ def __reduce__(self):
+ return rebuild_exc, (self.exc, self.tb)
+
+def rebuild_exc(exc, tb):
+ exc.__cause__ = RemoteTraceback(tb)
+ return exc
+
+#
# Code run by worker processes
#
@@ -63,7 +90,8 @@ class MaybeEncodingError(Exception):
return "<MaybeEncodingError: %s>" % str(self)
-def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None):
+def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None,
+ wrap_exception=False):
assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
put = outqueue.put
get = inqueue.get
@@ -78,28 +106,30 @@ def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None):
while maxtasks is None or (maxtasks and completed < maxtasks):
try:
task = get()
- except (EOFError, IOError):
- debug('worker got EOFError or IOError -- exiting')
+ except (EOFError, OSError):
+ util.debug('worker got EOFError or OSError -- exiting')
break
if task is None:
- debug('worker got sentinel -- exiting')
+ util.debug('worker got sentinel -- exiting')
break
job, i, func, args, kwds = task
try:
result = (True, func(*args, **kwds))
except Exception as e:
+ if wrap_exception:
+ e = ExceptionWithTraceback(e, e.__traceback__)
result = (False, e)
try:
put((job, i, result))
except Exception as e:
wrapped = MaybeEncodingError(e, result[1])
- debug("Possible encoding error while sending result: %s" % (
+ util.debug("Possible encoding error while sending result: %s" % (
wrapped))
put((job, i, (False, wrapped)))
completed += 1
- debug('worker exiting after %d tasks' % completed)
+ util.debug('worker exiting after %d tasks' % completed)
#
# Class representing a process pool
@@ -109,10 +139,14 @@ class Pool(object):
'''
Class which supports an async version of applying functions to arguments.
'''
- Process = Process
+ _wrap_exception = True
+
+ def Process(self, *args, **kwds):
+ return self._ctx.Process(*args, **kwds)
def __init__(self, processes=None, initializer=None, initargs=(),
- maxtasksperchild=None):
+ maxtasksperchild=None, context=None):
+ self._ctx = context or get_context()
self._setup_queues()
self._taskqueue = queue.Queue()
self._cache = {}
@@ -122,10 +156,7 @@ class Pool(object):
self._initargs = initargs
if processes is None:
- try:
- processes = cpu_count()
- except NotImplementedError:
- processes = 1
+ processes = os.cpu_count() or 1
if processes < 1:
raise ValueError("Number of processes must be at least 1")
@@ -162,7 +193,7 @@ class Pool(object):
self._result_handler._state = RUN
self._result_handler.start()
- self._terminate = Finalize(
+ self._terminate = util.Finalize(
self, self._terminate_pool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._worker_handler, self._task_handler,
@@ -179,7 +210,7 @@ class Pool(object):
worker = self._pool[i]
if worker.exitcode is not None:
# worker exited
- debug('cleaning up worker %d' % i)
+ util.debug('cleaning up worker %d' % i)
worker.join()
cleaned = True
del self._pool[i]
@@ -193,13 +224,14 @@ class Pool(object):
w = self.Process(target=worker,
args=(self._inqueue, self._outqueue,
self._initializer,
- self._initargs, self._maxtasksperchild)
+ self._initargs, self._maxtasksperchild,
+ self._wrap_exception)
)
self._pool.append(w)
w.name = w.name.replace('Process', 'PoolWorker')
w.daemon = True
w.start()
- debug('added worker')
+ util.debug('added worker')
def _maintain_pool(self):
"""Clean up any exited workers and start replacements for them.
@@ -208,9 +240,8 @@ class Pool(object):
self._repopulate_pool()
def _setup_queues(self):
- from .queues import SimpleQueue
- self._inqueue = SimpleQueue()
- self._outqueue = SimpleQueue()
+ self._inqueue = self._ctx.SimpleQueue()
+ self._outqueue = self._ctx.SimpleQueue()
self._quick_put = self._inqueue._writer.send
self._quick_get = self._outqueue._reader.recv
@@ -336,7 +367,7 @@ class Pool(object):
time.sleep(0.1)
# send sentinel to stop workers
pool._taskqueue.put(None)
- debug('worker handler exiting')
+ util.debug('worker handler exiting')
@staticmethod
def _handle_tasks(taskqueue, put, outqueue, pool, cache):
@@ -346,7 +377,7 @@ class Pool(object):
i = -1
for i, task in enumerate(taskseq):
if thread._state:
- debug('task handler found thread._state != RUN')
+ util.debug('task handler found thread._state != RUN')
break
try:
put(task)
@@ -358,27 +389,27 @@ class Pool(object):
pass
else:
if set_length:
- debug('doing set_length()')
+ util.debug('doing set_length()')
set_length(i+1)
continue
break
else:
- debug('task handler got sentinel')
+ util.debug('task handler got sentinel')
try:
# tell result handler to finish when cache is empty
- debug('task handler sending sentinel to result handler')
+ util.debug('task handler sending sentinel to result handler')
outqueue.put(None)
# tell workers there is no more work
- debug('task handler sending sentinel to workers')
+ util.debug('task handler sending sentinel to workers')
for p in pool:
put(None)
- except IOError:
- debug('task handler got IOError when sending sentinels')
+ except OSError:
+ util.debug('task handler got OSError when sending sentinels')
- debug('task handler exiting')
+ util.debug('task handler exiting')
@staticmethod
def _handle_results(outqueue, get, cache):
@@ -387,17 +418,17 @@ class Pool(object):
while 1:
try:
task = get()
- except (IOError, EOFError):
- debug('result handler got EOFError/IOError -- exiting')
+ except (OSError, EOFError):
+ util.debug('result handler got EOFError/OSError -- exiting')
return
if thread._state:
assert thread._state == TERMINATE
- debug('result handler found thread._state=TERMINATE')
+ util.debug('result handler found thread._state=TERMINATE')
break
if task is None:
- debug('result handler got sentinel')
+ util.debug('result handler got sentinel')
break
job, i, obj = task
@@ -409,12 +440,12 @@ class Pool(object):
while cache and thread._state != TERMINATE:
try:
task = get()
- except (IOError, EOFError):
- debug('result handler got EOFError/IOError -- exiting')
+ except (OSError, EOFError):
+ util.debug('result handler got EOFError/OSError -- exiting')
return
if task is None:
- debug('result handler ignoring extra sentinel')
+ util.debug('result handler ignoring extra sentinel')
continue
job, i, obj = task
try:
@@ -423,7 +454,7 @@ class Pool(object):
pass
if hasattr(outqueue, '_reader'):
- debug('ensuring that outqueue is not full')
+ util.debug('ensuring that outqueue is not full')
# If we don't make room available in outqueue then
# attempts to add the sentinel (None) to outqueue may
# block. There is guaranteed to be no more than 2 sentinels.
@@ -432,10 +463,10 @@ class Pool(object):
if not outqueue._reader.poll():
break
get()
- except (IOError, EOFError):
+ except (OSError, EOFError):
pass
- debug('result handler exiting: len(cache)=%s, thread._state=%s',
+ util.debug('result handler exiting: len(cache)=%s, thread._state=%s',
len(cache), thread._state)
@staticmethod
@@ -453,19 +484,19 @@ class Pool(object):
)
def close(self):
- debug('closing pool')
+ util.debug('closing pool')
if self._state == RUN:
self._state = CLOSE
self._worker_handler._state = CLOSE
def terminate(self):
- debug('terminating pool')
+ util.debug('terminating pool')
self._state = TERMINATE
self._worker_handler._state = TERMINATE
self._terminate()
def join(self):
- debug('joining pool')
+ util.debug('joining pool')
assert self._state in (CLOSE, TERMINATE)
self._worker_handler.join()
self._task_handler.join()
@@ -476,7 +507,7 @@ class Pool(object):
@staticmethod
def _help_stuff_finish(inqueue, task_handler, size):
# task_handler may be blocked trying to put items on inqueue
- debug('removing tasks from inqueue until task handler finished')
+ util.debug('removing tasks from inqueue until task handler finished')
inqueue._rlock.acquire()
while task_handler.is_alive() and inqueue._reader.poll():
inqueue._reader.recv()
@@ -486,12 +517,12 @@ class Pool(object):
def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool,
worker_handler, task_handler, result_handler, cache):
# this is guaranteed to only be called once
- debug('finalizing pool')
+ util.debug('finalizing pool')
worker_handler._state = TERMINATE
task_handler._state = TERMINATE
- debug('helping task handler/workers to finish')
+ util.debug('helping task handler/workers to finish')
cls._help_stuff_finish(inqueue, task_handler, len(pool))
assert result_handler.is_alive() or len(cache) == 0
@@ -501,31 +532,31 @@ class Pool(object):
# We must wait for the worker handler to exit before terminating
# workers because we don't want workers to be restarted behind our back.
- debug('joining worker handler')
+ util.debug('joining worker handler')
if threading.current_thread() is not worker_handler:
worker_handler.join()
# Terminate workers which haven't already finished.
if pool and hasattr(pool[0], 'terminate'):
- debug('terminating workers')
+ util.debug('terminating workers')
for p in pool:
if p.exitcode is None:
p.terminate()
- debug('joining task handler')
+ util.debug('joining task handler')
if threading.current_thread() is not task_handler:
task_handler.join()
- debug('joining result handler')
+ util.debug('joining result handler')
if threading.current_thread() is not result_handler:
result_handler.join()
if pool and hasattr(pool[0], 'terminate'):
- debug('joining pool workers')
+ util.debug('joining pool workers')
for p in pool:
if p.is_alive():
# worker has not yet exited
- debug('cleaning up worker %d' % p.pid)
+ util.debug('cleaning up worker %d' % p.pid)
p.join()
def __enter__(self):
@@ -710,8 +741,12 @@ class IMapUnorderedIterator(IMapIterator):
#
class ThreadPool(Pool):
+ _wrap_exception = False
- from .dummy import Process
+ @staticmethod
+ def Process(*args, **kwds):
+ from .dummy import Process
+ return Process(*args, **kwds)
def __init__(self, processes=None, initializer=None, initargs=()):
Pool.__init__(self, processes, initializer, initargs)
diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py
new file mode 100644
index 0000000..367e72e
--- /dev/null
+++ b/Lib/multiprocessing/popen_fork.py
@@ -0,0 +1,83 @@
+import os
+import sys
+import signal
+import errno
+
+from . import util
+
+__all__ = ['Popen']
+
+#
+# Start child process using fork
+#
+
+class Popen(object):
+ method = 'fork'
+
+ def __init__(self, process_obj):
+ sys.stdout.flush()
+ sys.stderr.flush()
+ self.returncode = None
+ self._launch(process_obj)
+
+ def duplicate_for_child(self, fd):
+ return fd
+
+ def poll(self, flag=os.WNOHANG):
+ if self.returncode is None:
+ while True:
+ try:
+ pid, sts = os.waitpid(self.pid, flag)
+ except OSError as e:
+ if e.errno == errno.EINTR:
+ continue
+ # Child process not yet created. See #1731717
+ # e.errno == errno.ECHILD == 10
+ return None
+ else:
+ break
+ if pid == self.pid:
+ if os.WIFSIGNALED(sts):
+ self.returncode = -os.WTERMSIG(sts)
+ else:
+ assert os.WIFEXITED(sts)
+ self.returncode = os.WEXITSTATUS(sts)
+ return self.returncode
+
+ def wait(self, timeout=None):
+ if self.returncode is None:
+ if timeout is not None:
+ from multiprocessing.connection import wait
+ if not wait([self.sentinel], timeout):
+ return None
+ # This shouldn't block if wait() returned successfully.
+ return self.poll(os.WNOHANG if timeout == 0.0 else 0)
+ return self.returncode
+
+ def terminate(self):
+ if self.returncode is None:
+ try:
+ os.kill(self.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+ except OSError:
+ if self.wait(timeout=0.1) is None:
+ raise
+
+ def _launch(self, process_obj):
+ code = 1
+ parent_r, child_w = os.pipe()
+ self.pid = os.fork()
+ if self.pid == 0:
+ try:
+ os.close(parent_r)
+ if 'random' in sys.modules:
+ import random
+ random.seed()
+ code = process_obj._bootstrap()
+ finally:
+ os._exit(code)
+ else:
+ os.close(child_w)
+ util.Finalize(self, os.close, (parent_r,))
+ self.sentinel = parent_r
diff --git a/Lib/multiprocessing/popen_forkserver.py b/Lib/multiprocessing/popen_forkserver.py
new file mode 100644
index 0000000..e792194
--- /dev/null
+++ b/Lib/multiprocessing/popen_forkserver.py
@@ -0,0 +1,69 @@
+import io
+import os
+
+from . import reduction
+if not reduction.HAVE_SEND_HANDLE:
+ raise ImportError('No support for sending fds between processes')
+from . import context
+from . import forkserver
+from . import popen_fork
+from . import spawn
+from . import util
+
+
+__all__ = ['Popen']
+
+#
+# Wrapper for an fd used while launching a process
+#
+
+class _DupFd(object):
+ def __init__(self, ind):
+ self.ind = ind
+ def detach(self):
+ return forkserver.get_inherited_fds()[self.ind]
+
+#
+# Start child process using a server process
+#
+
+class Popen(popen_fork.Popen):
+ method = 'forkserver'
+ DupFd = _DupFd
+
+ def __init__(self, process_obj):
+ self._fds = []
+ super().__init__(process_obj)
+
+ def duplicate_for_child(self, fd):
+ self._fds.append(fd)
+ return len(self._fds) - 1
+
+ def _launch(self, process_obj):
+ prep_data = spawn.get_preparation_data(process_obj._name)
+ buf = io.BytesIO()
+ context.set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, buf)
+ reduction.dump(process_obj, buf)
+ finally:
+ context.set_spawning_popen(None)
+
+ self.sentinel, w = forkserver.connect_to_new_process(self._fds)
+ util.Finalize(self, os.close, (self.sentinel,))
+ with open(w, 'wb', closefd=True) as f:
+ f.write(buf.getbuffer())
+ self.pid = forkserver.read_unsigned(self.sentinel)
+
+ def poll(self, flag=os.WNOHANG):
+ if self.returncode is None:
+ from multiprocessing.connection import wait
+ timeout = 0 if flag == os.WNOHANG else None
+ if not wait([self.sentinel], timeout):
+ return None
+ try:
+ self.returncode = forkserver.read_unsigned(self.sentinel)
+ except (OSError, EOFError):
+ # The process ended abnormally perhaps because of a signal
+ self.returncode = 255
+ return self.returncode
diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py
new file mode 100644
index 0000000..6b0a8d6
--- /dev/null
+++ b/Lib/multiprocessing/popen_spawn_posix.py
@@ -0,0 +1,69 @@
+import io
+import os
+
+from . import context
+from . import popen_fork
+from . import reduction
+from . import spawn
+from . import util
+
+__all__ = ['Popen']
+
+
+#
+# Wrapper for an fd used while launching a process
+#
+
+class _DupFd(object):
+ def __init__(self, fd):
+ self.fd = fd
+ def detach(self):
+ return self.fd
+
+#
+# Start child process using a fresh interpreter
+#
+
+class Popen(popen_fork.Popen):
+ method = 'spawn'
+ DupFd = _DupFd
+
+ def __init__(self, process_obj):
+ self._fds = []
+ super().__init__(process_obj)
+
+ def duplicate_for_child(self, fd):
+ self._fds.append(fd)
+ return fd
+
+ def _launch(self, process_obj):
+ from . import semaphore_tracker
+ tracker_fd = semaphore_tracker.getfd()
+ self._fds.append(tracker_fd)
+ prep_data = spawn.get_preparation_data(process_obj._name)
+ fp = io.BytesIO()
+ context.set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, fp)
+ reduction.dump(process_obj, fp)
+ finally:
+ context.set_spawning_popen(None)
+
+ parent_r = child_w = child_r = parent_w = None
+ try:
+ parent_r, child_w = os.pipe()
+ child_r, parent_w = os.pipe()
+ cmd = spawn.get_command_line(tracker_fd=tracker_fd,
+ pipe_handle=child_r)
+ self._fds.extend([child_r, child_w])
+ self.pid = util.spawnv_passfds(spawn.get_executable(),
+ cmd, self._fds)
+ self.sentinel = parent_r
+ with open(parent_w, 'wb', closefd=False) as f:
+ f.write(fp.getbuffer())
+ finally:
+ if parent_r is not None:
+ util.Finalize(self, os.close, (parent_r,))
+ for fd in (child_r, child_w, parent_w):
+ if fd is not None:
+ os.close(fd)
diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
new file mode 100644
index 0000000..3b53068
--- /dev/null
+++ b/Lib/multiprocessing/popen_spawn_win32.py
@@ -0,0 +1,99 @@
+import os
+import msvcrt
+import signal
+import sys
+import _winapi
+
+from . import context
+from . import spawn
+from . import reduction
+from . import util
+
+__all__ = ['Popen']
+
+#
+#
+#
+
+TERMINATE = 0x10000
+WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+
+#
+# We define a Popen class similar to the one from subprocess, but
+# whose constructor takes a process object as its argument.
+#
+
+class Popen(object):
+ '''
+ Start a subprocess to run the code of a process object
+ '''
+ method = 'spawn'
+
+ def __init__(self, process_obj):
+ prep_data = spawn.get_preparation_data(process_obj._name)
+
+ # read end of pipe will be "stolen" by the child process
+ # -- see spawn_main() in spawn.py.
+ rhandle, whandle = _winapi.CreatePipe(None, 0)
+ wfd = msvcrt.open_osfhandle(whandle, 0)
+ cmd = spawn.get_command_line(parent_pid=os.getpid(),
+ pipe_handle=rhandle)
+ cmd = ' '.join('"%s"' % x for x in cmd)
+
+ with open(wfd, 'wb', closefd=True) as to_child:
+ # start process
+ try:
+ hp, ht, pid, tid = _winapi.CreateProcess(
+ spawn.get_executable(), cmd,
+ None, None, False, 0, None, None, None)
+ _winapi.CloseHandle(ht)
+ except:
+ _winapi.CloseHandle(rhandle)
+ raise
+
+ # set attributes of self
+ self.pid = pid
+ self.returncode = None
+ self._handle = hp
+ self.sentinel = int(hp)
+ util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
+
+ # send information to child
+ context.set_spawning_popen(self)
+ try:
+ reduction.dump(prep_data, to_child)
+ reduction.dump(process_obj, to_child)
+ finally:
+ context.set_spawning_popen(None)
+
+ def duplicate_for_child(self, handle):
+ assert self is context.get_spawning_popen()
+ return reduction.duplicate(handle, self.sentinel)
+
+ def wait(self, timeout=None):
+ if self.returncode is None:
+ if timeout is None:
+ msecs = _winapi.INFINITE
+ else:
+ msecs = max(0, int(timeout * 1000 + 0.5))
+
+ res = _winapi.WaitForSingleObject(int(self._handle), msecs)
+ if res == _winapi.WAIT_OBJECT_0:
+ code = _winapi.GetExitCodeProcess(self._handle)
+ if code == TERMINATE:
+ code = -signal.SIGTERM
+ self.returncode = code
+
+ return self.returncode
+
+ def poll(self):
+ return self.wait(timeout=0)
+
+ def terminate(self):
+ if self.returncode is None:
+ try:
+ _winapi.TerminateProcess(int(self._handle), TERMINATE)
+ except OSError:
+ if self.wait(timeout=1.0) is None:
+ raise
diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py
index 3d32add..68959bf 100644
--- a/Lib/multiprocessing/process.py
+++ b/Lib/multiprocessing/process.py
@@ -7,7 +7,7 @@
# Licensed to PSF under a Contributor Agreement.
#
-__all__ = ['Process', 'current_process', 'active_children']
+__all__ = ['BaseProcess', 'current_process', 'active_children']
#
# Imports
@@ -43,7 +43,7 @@ def active_children():
Return list of process objects corresponding to live child processes
'''
_cleanup()
- return list(_current_process._children)
+ return list(_children)
#
#
@@ -51,33 +51,29 @@ def active_children():
def _cleanup():
# check for processes which have finished
- for p in list(_current_process._children):
+ for p in list(_children):
if p._popen.poll() is not None:
- _current_process._children.discard(p)
+ _children.discard(p)
#
# The `Process` class
#
-class Process(object):
+class BaseProcess(object):
'''
Process objects represent activity that is run in a separate process
- The class is analagous to `threading.Thread`
+ The class is analogous to `threading.Thread`
'''
- _Popen = None
+ def _Popen(self):
+ raise NotImplementedError
def __init__(self, group=None, target=None, name=None, args=(), kwargs={},
*, daemon=None):
assert group is None, 'group argument must be None for now'
- count = next(_current_process._counter)
+ count = next(_process_counter)
self._identity = _current_process._identity + (count,)
- self._authkey = _current_process._authkey
- if daemon is not None:
- self._daemonic = daemon
- else:
- self._daemonic = _current_process._daemonic
- self._tempdir = _current_process._tempdir
+ self._config = _current_process._config.copy()
self._parent_pid = os.getpid()
self._popen = None
self._target = target
@@ -85,6 +81,8 @@ class Process(object):
self._kwargs = dict(kwargs)
self._name = name or type(self).__name__ + '-' + \
':'.join(str(i) for i in self._identity)
+ if daemon is not None:
+ self.daemon = daemon
_dangling.add(self)
def run(self):
@@ -101,16 +99,12 @@ class Process(object):
assert self._popen is None, 'cannot start a process twice'
assert self._parent_pid == os.getpid(), \
'can only start a process object created by current process'
- assert not _current_process._daemonic, \
+ assert not _current_process._config.get('daemon'), \
'daemonic processes are not allowed to have children'
_cleanup()
- if self._Popen is not None:
- Popen = self._Popen
- else:
- from .forking import Popen
- self._popen = Popen(self)
+ self._popen = self._Popen(self)
self._sentinel = self._popen.sentinel
- _current_process._children.add(self)
+ _children.add(self)
def terminate(self):
'''
@@ -126,7 +120,7 @@ class Process(object):
assert self._popen is not None, 'can only join a started process'
res = self._popen.wait(timeout)
if res is not None:
- _current_process._children.discard(self)
+ _children.discard(self)
def is_alive(self):
'''
@@ -154,7 +148,7 @@ class Process(object):
'''
Return whether process is a daemon
'''
- return self._daemonic
+ return self._config.get('daemon', False)
@daemon.setter
def daemon(self, daemonic):
@@ -162,18 +156,18 @@ class Process(object):
Set whether process is a daemon
'''
assert self._popen is None, 'process has already started'
- self._daemonic = daemonic
+ self._config['daemon'] = daemonic
@property
def authkey(self):
- return self._authkey
+ return self._config['authkey']
@authkey.setter
def authkey(self, authkey):
'''
Set authorization key of process
'''
- self._authkey = AuthenticationString(authkey)
+ self._config['authkey'] = AuthenticationString(authkey)
@property
def exitcode(self):
@@ -227,17 +221,19 @@ class Process(object):
status = 'stopped[%s]' % _exitcode_to_name.get(status, status)
return '<%s(%s, %s%s)>' % (type(self).__name__, self._name,
- status, self._daemonic and ' daemon' or '')
+ status, self.daemon and ' daemon' or '')
##
def _bootstrap(self):
- from . import util
- global _current_process
+ from . import util, context
+ global _current_process, _process_counter, _children
try:
- self._children = set()
- self._counter = itertools.count(1)
+ if self._start_method is not None:
+ context._force_start_method(self._start_method)
+ _process_counter = itertools.count(1)
+ _children = set()
if sys.stdin is not None:
try:
sys.stdin.close()
@@ -285,8 +281,8 @@ class Process(object):
class AuthenticationString(bytes):
def __reduce__(self):
- from .forking import Popen
- if not Popen.thread_is_spawning():
+ from .context import get_spawning_popen
+ if get_spawning_popen() is None:
raise TypeError(
'Pickling an AuthenticationString object is '
'disallowed for security reasons'
@@ -297,20 +293,29 @@ class AuthenticationString(bytes):
# Create object representing the main process
#
-class _MainProcess(Process):
+class _MainProcess(BaseProcess):
def __init__(self):
self._identity = ()
- self._daemonic = False
self._name = 'MainProcess'
self._parent_pid = None
self._popen = None
- self._counter = itertools.count(1)
- self._children = set()
- self._authkey = AuthenticationString(os.urandom(32))
- self._tempdir = None
+ self._config = {'authkey': AuthenticationString(os.urandom(32)),
+ 'semprefix': '/mp'}
+ # Note that some versions of FreeBSD only allow named
+ # semaphores to have names of up to 14 characters. Therefore
+ # we choose a short prefix.
+ #
+ # On MacOSX in a sandbox it may be necessary to use a
+ # different prefix -- see #19478.
+ #
+ # Everything in self._config will be inherited by descendant
+ # processes.
+
_current_process = _MainProcess()
+_process_counter = itertools.count(1)
+_children = set()
del _MainProcess
#
diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py
index 37271fb..f650771 100644
--- a/Lib/multiprocessing/queues.py
+++ b/Lib/multiprocessing/queues.py
@@ -18,11 +18,14 @@ import weakref
import errno
from queue import Empty, Full
+
import _multiprocessing
-from multiprocessing.connection import Pipe
-from multiprocessing.synchronize import Lock, BoundedSemaphore, Semaphore, Condition
-from multiprocessing.util import debug, info, Finalize, register_after_fork
-from multiprocessing.forking import assert_spawning
+
+from . import connection
+from . import context
+
+from .util import debug, info, Finalize, register_after_fork, is_exiting
+from .reduction import ForkingPickler
#
# Queue type using a pipe, buffer and thread
@@ -30,18 +33,18 @@ from multiprocessing.forking import assert_spawning
class Queue(object):
- def __init__(self, maxsize=0):
+ def __init__(self, maxsize=0, *, ctx):
if maxsize <= 0:
maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
self._maxsize = maxsize
- self._reader, self._writer = Pipe(duplex=False)
- self._rlock = Lock()
+ self._reader, self._writer = connection.Pipe(duplex=False)
+ self._rlock = ctx.Lock()
self._opid = os.getpid()
if sys.platform == 'win32':
self._wlock = None
else:
- self._wlock = Lock()
- self._sem = BoundedSemaphore(maxsize)
+ self._wlock = ctx.Lock()
+ self._sem = ctx.BoundedSemaphore(maxsize)
# For use by concurrent.futures
self._ignore_epipe = False
@@ -51,7 +54,7 @@ class Queue(object):
register_after_fork(self, Queue._after_fork)
def __getstate__(self):
- assert_spawning(self)
+ context.assert_spawning(self)
return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
self._rlock, self._wlock, self._sem, self._opid)
@@ -69,8 +72,8 @@ class Queue(object):
self._joincancelled = False
self._closed = False
self._close = None
- self._send = self._writer.send
- self._recv = self._reader.recv
+ self._send_bytes = self._writer.send_bytes
+ self._recv_bytes = self._reader.recv_bytes
self._poll = self._reader.poll
def put(self, obj, block=True, timeout=None):
@@ -89,14 +92,9 @@ class Queue(object):
def get(self, block=True, timeout=None):
if block and timeout is None:
- self._rlock.acquire()
- try:
- res = self._recv()
- self._sem.release()
- return res
- finally:
- self._rlock.release()
-
+ with self._rlock:
+ res = self._recv_bytes()
+ self._sem.release()
else:
if block:
deadline = time.time() + timeout
@@ -109,11 +107,12 @@ class Queue(object):
raise Empty
elif not self._poll():
raise Empty
- res = self._recv()
+ res = self._recv_bytes()
self._sem.release()
- return res
finally:
self._rlock.release()
+ # unserialize the data after having released the lock
+ return ForkingPickler.loads(res)
def qsize(self):
# Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
@@ -158,7 +157,7 @@ class Queue(object):
self._buffer.clear()
self._thread = threading.Thread(
target=Queue._feed,
- args=(self._buffer, self._notempty, self._send,
+ args=(self._buffer, self._notempty, self._send_bytes,
self._wlock, self._writer.close, self._ignore_epipe),
name='QueueFeederThread'
)
@@ -210,10 +209,8 @@ class Queue(object):
notempty.release()
@staticmethod
- def _feed(buffer, notempty, send, writelock, close, ignore_epipe):
+ def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe):
debug('starting thread to feed data to pipe')
- from .util import is_exiting
-
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
@@ -241,12 +238,14 @@ class Queue(object):
close()
return
+ # serialize the data before acquiring the lock
+ obj = ForkingPickler.dumps(obj)
if wacquire is None:
- send(obj)
+ send_bytes(obj)
else:
wacquire()
try:
- send(obj)
+ send_bytes(obj)
finally:
wrelease()
except IndexError:
@@ -279,10 +278,10 @@ _sentinel = object()
class JoinableQueue(Queue):
- def __init__(self, maxsize=0):
- Queue.__init__(self, maxsize)
- self._unfinished_tasks = Semaphore(0)
- self._cond = Condition()
+ def __init__(self, maxsize=0, *, ctx):
+ Queue.__init__(self, maxsize, ctx=ctx)
+ self._unfinished_tasks = ctx.Semaphore(0)
+ self._cond = ctx.Condition()
def __getstate__(self):
return Queue.__getstate__(self) + (self._cond, self._unfinished_tasks)
@@ -332,48 +331,37 @@ class JoinableQueue(Queue):
class SimpleQueue(object):
- def __init__(self):
- self._reader, self._writer = Pipe(duplex=False)
- self._rlock = Lock()
+ def __init__(self, *, ctx):
+ self._reader, self._writer = connection.Pipe(duplex=False)
+ self._rlock = ctx.Lock()
self._poll = self._reader.poll
if sys.platform == 'win32':
self._wlock = None
else:
- self._wlock = Lock()
- self._make_methods()
+ self._wlock = ctx.Lock()
def empty(self):
return not self._poll()
def __getstate__(self):
- assert_spawning(self)
+ context.assert_spawning(self)
return (self._reader, self._writer, self._rlock, self._wlock)
def __setstate__(self, state):
(self._reader, self._writer, self._rlock, self._wlock) = state
- self._make_methods()
- def _make_methods(self):
- recv = self._reader.recv
- racquire, rrelease = self._rlock.acquire, self._rlock.release
- def get():
- racquire()
- try:
- return recv()
- finally:
- rrelease()
- self.get = get
+ def get(self):
+ with self._rlock:
+ res = self._reader.recv_bytes()
+ # unserialize the data after having released the lock
+ return ForkingPickler.loads(res)
+ def put(self, obj):
+ # serialize the data before acquiring the lock
+ obj = ForkingPickler.dumps(obj)
if self._wlock is None:
# writes to a message oriented win32 pipe are atomic
- self.put = self._writer.send
+ self._writer.send_bytes(obj)
else:
- send = self._writer.send
- wacquire, wrelease = self._wlock.acquire, self._wlock.release
- def put(obj):
- wacquire()
- try:
- return send(obj)
- finally:
- wrelease()
- self.put = put
+ with self._wlock:
+ self._writer.send_bytes(obj)
diff --git a/Lib/multiprocessing/reduction.py b/Lib/multiprocessing/reduction.py
index 656fa8f..8f209b4 100644
--- a/Lib/multiprocessing/reduction.py
+++ b/Lib/multiprocessing/reduction.py
@@ -1,6 +1,5 @@
#
-# Module to allow connection and socket objects to be transferred
-# between processes
+# Module which deals with pickling of objects.
#
# multiprocessing/reduction.py
#
@@ -8,27 +7,56 @@
# Licensed to PSF under a Contributor Agreement.
#
-__all__ = ['reduce_socket', 'reduce_connection', 'send_handle', 'recv_handle']
-
+import copyreg
+import functools
+import io
import os
-import sys
+import pickle
import socket
-import threading
-import struct
-import signal
+import sys
-from multiprocessing import current_process
-from multiprocessing.util import register_after_fork, debug, sub_debug
-from multiprocessing.util import is_exiting, sub_warning
+from . import context
+__all__ = ['send_handle', 'recv_handle', 'ForkingPickler', 'register', 'dump']
+
+
+HAVE_SEND_HANDLE = (sys.platform == 'win32' or
+ (hasattr(socket, 'CMSG_LEN') and
+ hasattr(socket, 'SCM_RIGHTS') and
+ hasattr(socket.socket, 'sendmsg')))
#
+# Pickler subclass
#
-#
-if not(sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and
- hasattr(socket, 'SCM_RIGHTS'))):
- raise ImportError('pickling of connections not supported')
+class ForkingPickler(pickle.Pickler):
+ '''Pickler subclass used by multiprocessing.'''
+ _extra_reducers = {}
+ _copyreg_dispatch_table = copyreg.dispatch_table
+
+ def __init__(self, *args):
+ super().__init__(*args)
+ self.dispatch_table = self._copyreg_dispatch_table.copy()
+ self.dispatch_table.update(self._extra_reducers)
+
+ @classmethod
+ def register(cls, type, reduce):
+ '''Register a reduce function for a type.'''
+ cls._extra_reducers[type] = reduce
+
+ @classmethod
+ def dumps(cls, obj, protocol=None):
+ buf = io.BytesIO()
+ cls(buf, protocol).dump(obj)
+ return buf.getbuffer()
+
+ loads = pickle.loads
+
+register = ForkingPickler.register
+
+def dump(obj, file, protocol=None):
+ '''Replacement for pickle.dump() using ForkingPickler.'''
+ ForkingPickler(file, protocol).dump(obj)
#
# Platform specific definitions
@@ -36,20 +64,44 @@ if not(sys.platform == 'win32' or (hasattr(socket, 'CMSG_LEN') and
if sys.platform == 'win32':
# Windows
- __all__ += ['reduce_pipe_connection']
+ __all__ += ['DupHandle', 'duplicate', 'steal_handle']
import _winapi
+ def duplicate(handle, target_process=None, inheritable=False):
+ '''Duplicate a handle. (target_process is a handle not a pid!)'''
+ if target_process is None:
+ target_process = _winapi.GetCurrentProcess()
+ return _winapi.DuplicateHandle(
+ _winapi.GetCurrentProcess(), handle, target_process,
+ 0, inheritable, _winapi.DUPLICATE_SAME_ACCESS)
+
+ def steal_handle(source_pid, handle):
+ '''Steal a handle from process identified by source_pid.'''
+ source_process_handle = _winapi.OpenProcess(
+ _winapi.PROCESS_DUP_HANDLE, False, source_pid)
+ try:
+ return _winapi.DuplicateHandle(
+ source_process_handle, handle,
+ _winapi.GetCurrentProcess(), 0, False,
+ _winapi.DUPLICATE_SAME_ACCESS | _winapi.DUPLICATE_CLOSE_SOURCE)
+ finally:
+ _winapi.CloseHandle(source_process_handle)
+
def send_handle(conn, handle, destination_pid):
+ '''Send a handle over a local connection.'''
dh = DupHandle(handle, _winapi.DUPLICATE_SAME_ACCESS, destination_pid)
conn.send(dh)
def recv_handle(conn):
+ '''Receive a handle over a local connection.'''
return conn.recv().detach()
class DupHandle(object):
+ '''Picklable wrapper for a handle.'''
def __init__(self, handle, access, pid=None):
- # duplicate handle for process with given pid
if pid is None:
+ # We just duplicate the handle in the current process and
+ # let the receiving process steal the handle.
pid = os.getpid()
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid)
try:
@@ -62,9 +114,12 @@ if sys.platform == 'win32':
self._pid = pid
def detach(self):
+ '''Get the handle. This should only be called once.'''
# retrieve handle from process which currently owns it
if self._pid == os.getpid():
+ # The handle has already been duplicated for this process.
return self._handle
+ # We must steal the handle from the process whose pid is self._pid.
proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False,
self._pid)
try:
@@ -74,207 +129,112 @@ if sys.platform == 'win32':
finally:
_winapi.CloseHandle(proc)
- class DupSocket(object):
- def __init__(self, sock):
- new_sock = sock.dup()
- def send(conn, pid):
- share = new_sock.share(pid)
- conn.send_bytes(share)
- self._id = resource_sharer.register(send, new_sock.close)
-
- def detach(self):
- conn = resource_sharer.get_connection(self._id)
- try:
- share = conn.recv_bytes()
- return socket.fromshare(share)
- finally:
- conn.close()
-
- def reduce_socket(s):
- return rebuild_socket, (DupSocket(s),)
-
- def rebuild_socket(ds):
- return ds.detach()
-
- def reduce_connection(conn):
- handle = conn.fileno()
- with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
- ds = DupSocket(s)
- return rebuild_connection, (ds, conn.readable, conn.writable)
-
- def rebuild_connection(ds, readable, writable):
- from .connection import Connection
- sock = ds.detach()
- return Connection(sock.detach(), readable, writable)
-
- def reduce_pipe_connection(conn):
- access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
- (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
- dh = DupHandle(conn.fileno(), access)
- return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
-
- def rebuild_pipe_connection(dh, readable, writable):
- from .connection import PipeConnection
- handle = dh.detach()
- return PipeConnection(handle, readable, writable)
-
else:
# Unix
+ __all__ += ['DupFd', 'sendfds', 'recvfds']
+ import array
# On MacOSX we should acknowledge receipt of fds -- see Issue14669
ACKNOWLEDGE = sys.platform == 'darwin'
+ def sendfds(sock, fds):
+ '''Send an array of fds over an AF_UNIX socket.'''
+ fds = array.array('i', fds)
+ msg = bytes([len(fds) % 256])
+ sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
+ if ACKNOWLEDGE and sock.recv(1) != b'A':
+ raise RuntimeError('did not receive acknowledgement of fd')
+
+ def recvfds(sock, size):
+ '''Receive an array of fds over an AF_UNIX socket.'''
+ a = array.array('i')
+ bytes_size = a.itemsize * size
+ msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
+ if not msg and not ancdata:
+ raise EOFError
+ try:
+ if ACKNOWLEDGE:
+ sock.send(b'A')
+ if len(ancdata) != 1:
+ raise RuntimeError('received %d items of ancdata' %
+ len(ancdata))
+ cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+ if (cmsg_level == socket.SOL_SOCKET and
+ cmsg_type == socket.SCM_RIGHTS):
+ if len(cmsg_data) % a.itemsize != 0:
+ raise ValueError
+ a.frombytes(cmsg_data)
+ assert len(a) % 256 == msg[0]
+ return list(a)
+ except (ValueError, IndexError):
+ pass
+ raise RuntimeError('Invalid data received')
+
def send_handle(conn, handle, destination_pid):
+ '''Send a handle over a local connection.'''
with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
- s.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS,
- struct.pack("@i", handle))])
- if ACKNOWLEDGE and conn.recv_bytes() != b'ACK':
- raise RuntimeError('did not receive acknowledgement of fd')
+ sendfds(s, [handle])
def recv_handle(conn):
- size = struct.calcsize("@i")
+ '''Receive a handle over a local connection.'''
with socket.fromfd(conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM) as s:
- msg, ancdata, flags, addr = s.recvmsg(1, socket.CMSG_LEN(size))
- try:
- if ACKNOWLEDGE:
- conn.send_bytes(b'ACK')
- cmsg_level, cmsg_type, cmsg_data = ancdata[0]
- if (cmsg_level == socket.SOL_SOCKET and
- cmsg_type == socket.SCM_RIGHTS):
- return struct.unpack("@i", cmsg_data[:size])[0]
- except (ValueError, IndexError, struct.error):
- pass
- raise RuntimeError('Invalid data received')
-
- class DupFd(object):
- def __init__(self, fd):
- new_fd = os.dup(fd)
- def send(conn, pid):
- send_handle(conn, new_fd, pid)
- def close():
- os.close(new_fd)
- self._id = resource_sharer.register(send, close)
+ return recvfds(s, 1)[0]
+
+ def DupFd(fd):
+ '''Return a wrapper for an fd.'''
+ popen_obj = context.get_spawning_popen()
+ if popen_obj is not None:
+ return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
+ elif HAVE_SEND_HANDLE:
+ from . import resource_sharer
+ return resource_sharer.DupFd(fd)
+ else:
+ raise ValueError('SCM_RIGHTS appears not to be available')
- def detach(self):
- conn = resource_sharer.get_connection(self._id)
- try:
- return recv_handle(conn)
- finally:
- conn.close()
+#
+# Try making some callable types picklable
+#
- def reduce_socket(s):
- df = DupFd(s.fileno())
- return rebuild_socket, (df, s.family, s.type, s.proto)
+def _reduce_method(m):
+ if m.__self__ is None:
+ return getattr, (m.__class__, m.__func__.__name__)
+ else:
+ return getattr, (m.__self__, m.__func__.__name__)
+class _C:
+ def f(self):
+ pass
+register(type(_C().f), _reduce_method)
- def rebuild_socket(df, family, type, proto):
- fd = df.detach()
- s = socket.fromfd(fd, family, type, proto)
- os.close(fd)
- return s
- def reduce_connection(conn):
- df = DupFd(conn.fileno())
- return rebuild_connection, (df, conn.readable, conn.writable)
+def _reduce_method_descriptor(m):
+ return getattr, (m.__objclass__, m.__name__)
+register(type(list.append), _reduce_method_descriptor)
+register(type(int.__add__), _reduce_method_descriptor)
- def rebuild_connection(df, readable, writable):
- from .connection import Connection
- fd = df.detach()
- return Connection(fd, readable, writable)
+
+def _reduce_partial(p):
+ return _rebuild_partial, (p.func, p.args, p.keywords or {})
+def _rebuild_partial(func, args, keywords):
+ return functools.partial(func, *args, **keywords)
+register(functools.partial, _reduce_partial)
#
-# Server which shares registered resources with clients
+# Make sockets picklable
#
-class ResourceSharer(object):
- def __init__(self):
- self._key = 0
- self._cache = {}
- self._old_locks = []
- self._lock = threading.Lock()
- self._listener = None
- self._address = None
- self._thread = None
- register_after_fork(self, ResourceSharer._afterfork)
-
- def register(self, send, close):
- with self._lock:
- if self._address is None:
- self._start()
- self._key += 1
- self._cache[self._key] = (send, close)
- return (self._address, self._key)
-
- @staticmethod
- def get_connection(ident):
- from .connection import Client
- address, key = ident
- c = Client(address, authkey=current_process().authkey)
- c.send((key, os.getpid()))
- return c
-
- def stop(self, timeout=None):
- from .connection import Client
- with self._lock:
- if self._address is not None:
- c = Client(self._address, authkey=current_process().authkey)
- c.send(None)
- c.close()
- self._thread.join(timeout)
- if self._thread.is_alive():
- sub_warn('ResourceSharer thread did not stop when asked')
- self._listener.close()
- self._thread = None
- self._address = None
- self._listener = None
- for key, (send, close) in self._cache.items():
- close()
- self._cache.clear()
-
- def _afterfork(self):
- for key, (send, close) in self._cache.items():
- close()
- self._cache.clear()
- # If self._lock was locked at the time of the fork, it may be broken
- # -- see issue 6721. Replace it without letting it be gc'ed.
- self._old_locks.append(self._lock)
- self._lock = threading.Lock()
- if self._listener is not None:
- self._listener.close()
- self._listener = None
- self._address = None
- self._thread = None
-
- def _start(self):
- from .connection import Listener
- assert self._listener is None
- debug('starting listener and thread for sending handles')
- self._listener = Listener(authkey=current_process().authkey)
- self._address = self._listener.address
- t = threading.Thread(target=self._serve)
- t.daemon = True
- t.start()
- self._thread = t
-
- def _serve(self):
- if hasattr(signal, 'pthread_sigmask'):
- signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
- while 1:
- try:
- conn = self._listener.accept()
- msg = conn.recv()
- if msg is None:
- break
- key, destination_pid = msg
- send, close = self._cache.pop(key)
- send(conn, destination_pid)
- close()
- conn.close()
- except:
- if not is_exiting():
- import traceback
- sub_warning(
- 'thread for sharing handles raised exception :\n' +
- '-'*79 + '\n' + traceback.format_exc() + '-'*79
- )
-
-resource_sharer = ResourceSharer()
+if sys.platform == 'win32':
+ def _reduce_socket(s):
+ from .resource_sharer import DupSocket
+ return _rebuild_socket, (DupSocket(s),)
+ def _rebuild_socket(ds):
+ return ds.detach()
+ register(socket.socket, _reduce_socket)
+
+else:
+ def _reduce_socket(s):
+ df = DupFd(s.fileno())
+ return _rebuild_socket, (df, s.family, s.type, s.proto)
+ def _rebuild_socket(df, family, type, proto):
+ fd = df.detach()
+ return socket.socket(family, type, proto, fileno=fd)
+ register(socket.socket, _reduce_socket)
diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py
new file mode 100644
index 0000000..5e46fc6
--- /dev/null
+++ b/Lib/multiprocessing/resource_sharer.py
@@ -0,0 +1,158 @@
+#
+# We use a background thread for sharing fds on Unix, and for sharing sockets on
+# Windows.
+#
+# A client which wants to pickle a resource registers it with the resource
+# sharer and gets an identifier in return. The unpickling process will connect
+# to the resource sharer, sends the identifier and its pid, and then receives
+# the resource.
+#
+
+import os
+import signal
+import socket
+import sys
+import threading
+
+from . import process
+from . import reduction
+from . import util
+
+__all__ = ['stop']
+
+
+if sys.platform == 'win32':
+ __all__ += ['DupSocket']
+
+ class DupSocket(object):
+ '''Picklable wrapper for a socket.'''
+ def __init__(self, sock):
+ new_sock = sock.dup()
+ def send(conn, pid):
+ share = new_sock.share(pid)
+ conn.send_bytes(share)
+ self._id = _resource_sharer.register(send, new_sock.close)
+
+ def detach(self):
+ '''Get the socket. This should only be called once.'''
+ with _resource_sharer.get_connection(self._id) as conn:
+ share = conn.recv_bytes()
+ return socket.fromshare(share)
+
+else:
+ __all__ += ['DupFd']
+
+ class DupFd(object):
+ '''Wrapper for fd which can be used at any time.'''
+ def __init__(self, fd):
+ new_fd = os.dup(fd)
+ def send(conn, pid):
+ reduction.send_handle(conn, new_fd, pid)
+ def close():
+ os.close(new_fd)
+ self._id = _resource_sharer.register(send, close)
+
+ def detach(self):
+ '''Get the fd. This should only be called once.'''
+ with _resource_sharer.get_connection(self._id) as conn:
+ return reduction.recv_handle(conn)
+
+
+class _ResourceSharer(object):
+ '''Manager for resouces using background thread.'''
+ def __init__(self):
+ self._key = 0
+ self._cache = {}
+ self._old_locks = []
+ self._lock = threading.Lock()
+ self._listener = None
+ self._address = None
+ self._thread = None
+ util.register_after_fork(self, _ResourceSharer._afterfork)
+
+ def register(self, send, close):
+ '''Register resource, returning an identifier.'''
+ with self._lock:
+ if self._address is None:
+ self._start()
+ self._key += 1
+ self._cache[self._key] = (send, close)
+ return (self._address, self._key)
+
+ @staticmethod
+ def get_connection(ident):
+ '''Return connection from which to receive identified resource.'''
+ from .connection import Client
+ address, key = ident
+ c = Client(address, authkey=process.current_process().authkey)
+ c.send((key, os.getpid()))
+ return c
+
+ def stop(self, timeout=None):
+ '''Stop the background thread and clear registered resources.'''
+ from .connection import Client
+ with self._lock:
+ if self._address is not None:
+ c = Client(self._address,
+ authkey=process.current_process().authkey)
+ c.send(None)
+ c.close()
+ self._thread.join(timeout)
+ if self._thread.is_alive():
+ util.sub_warning('_ResourceSharer thread did '
+ 'not stop when asked')
+ self._listener.close()
+ self._thread = None
+ self._address = None
+ self._listener = None
+ for key, (send, close) in self._cache.items():
+ close()
+ self._cache.clear()
+
+ def _afterfork(self):
+ for key, (send, close) in self._cache.items():
+ close()
+ self._cache.clear()
+ # If self._lock was locked at the time of the fork, it may be broken
+ # -- see issue 6721. Replace it without letting it be gc'ed.
+ self._old_locks.append(self._lock)
+ self._lock = threading.Lock()
+ if self._listener is not None:
+ self._listener.close()
+ self._listener = None
+ self._address = None
+ self._thread = None
+
+ def _start(self):
+ from .connection import Listener
+ assert self._listener is None
+ util.debug('starting listener and thread for sending handles')
+ self._listener = Listener(authkey=process.current_process().authkey)
+ self._address = self._listener.address
+ t = threading.Thread(target=self._serve)
+ t.daemon = True
+ t.start()
+ self._thread = t
+
+ def _serve(self):
+ if hasattr(signal, 'pthread_sigmask'):
+ signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
+ while 1:
+ try:
+ with self._listener.accept() as conn:
+ msg = conn.recv()
+ if msg is None:
+ break
+ key, destination_pid = msg
+ send, close = self._cache.pop(key)
+ try:
+ send(conn, destination_pid)
+ finally:
+ close()
+ except:
+ if not util.is_exiting():
+ sys.excepthook(*sys.exc_info())
+
+
+_resource_sharer = _ResourceSharer()
+stop = _resource_sharer.stop
diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py
new file mode 100644
index 0000000..de7738e
--- /dev/null
+++ b/Lib/multiprocessing/semaphore_tracker.py
@@ -0,0 +1,143 @@
+#
+# On Unix we run a server process which keeps track of unlinked
+# semaphores. The server ignores SIGINT and SIGTERM and reads from a
+# pipe. Every other process of the program has a copy of the writable
+# end of the pipe, so we get EOF when all other processes have exited.
+# Then the server process unlinks any remaining semaphore names.
+#
+# This is important because the system only supports a limited number
+# of named semaphores, and they will not be automatically removed till
+# the next reboot. Without this semaphore tracker process, "killall
+# python" would probably leave unlinked semaphores.
+#
+
+import os
+import signal
+import sys
+import threading
+import warnings
+import _multiprocessing
+
+from . import spawn
+from . import util
+
+__all__ = ['ensure_running', 'register', 'unregister']
+
+
+class SemaphoreTracker(object):
+
+ def __init__(self):
+ self._lock = threading.Lock()
+ self._fd = None
+
+ def getfd(self):
+ self.ensure_running()
+ return self._fd
+
+ def ensure_running(self):
+ '''Make sure that semaphore tracker process is running.
+
+ This can be run from any process. Usually a child process will use
+ the semaphore created by its parent.'''
+ with self._lock:
+ if self._fd is not None:
+ return
+ fds_to_pass = []
+ try:
+ fds_to_pass.append(sys.stderr.fileno())
+ except Exception:
+ pass
+ cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)'
+ r, w = os.pipe()
+ try:
+ fds_to_pass.append(r)
+ # process will out live us, so no need to wait on pid
+ exe = spawn.get_executable()
+ args = [exe] + util._args_from_interpreter_flags()
+ args += ['-c', cmd % r]
+ util.spawnv_passfds(exe, args, fds_to_pass)
+ except:
+ os.close(w)
+ raise
+ else:
+ self._fd = w
+ finally:
+ os.close(r)
+
+ def register(self, name):
+ '''Register name of semaphore with semaphore tracker.'''
+ self._send('REGISTER', name)
+
+ def unregister(self, name):
+ '''Unregister name of semaphore with semaphore tracker.'''
+ self._send('UNREGISTER', name)
+
+ def _send(self, cmd, name):
+ self.ensure_running()
+ msg = '{0}:{1}\n'.format(cmd, name).encode('ascii')
+ if len(name) > 512:
+ # posix guarantees that writes to a pipe of less than PIPE_BUF
+ # bytes are atomic, and that PIPE_BUF >= 512
+ raise ValueError('name too long')
+ nbytes = os.write(self._fd, msg)
+ assert nbytes == len(msg)
+
+
+_semaphore_tracker = SemaphoreTracker()
+ensure_running = _semaphore_tracker.ensure_running
+register = _semaphore_tracker.register
+unregister = _semaphore_tracker.unregister
+getfd = _semaphore_tracker.getfd
+
+
+def main(fd):
+ '''Run semaphore tracker.'''
+ # protect the process from ^C and "killall python" etc
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
+
+ for f in (sys.stdin, sys.stdout):
+ try:
+ f.close()
+ except Exception:
+ pass
+
+ cache = set()
+ try:
+ # keep track of registered/unregistered semaphores
+ with open(fd, 'rb') as f:
+ for line in f:
+ try:
+ cmd, name = line.strip().split(b':')
+ if cmd == b'REGISTER':
+ cache.add(name)
+ elif cmd == b'UNREGISTER':
+ cache.remove(name)
+ else:
+ raise RuntimeError('unrecognized command %r' % cmd)
+ except Exception:
+ try:
+ sys.excepthook(*sys.exc_info())
+ except:
+ pass
+ finally:
+ # all processes have terminated; cleanup any remaining semaphores
+ if cache:
+ try:
+ warnings.warn('semaphore_tracker: There appear to be %d '
+ 'leaked semaphores to clean up at shutdown' %
+ len(cache))
+ except Exception:
+ pass
+ for name in cache:
+ # For some reason the process which created and registered this
+ # semaphore has failed to unregister it. Presumably it has died.
+ # We therefore unlink it.
+ try:
+ name = name.decode('ascii')
+ try:
+ _multiprocessing.sem_unlink(name)
+ except Exception as e:
+ warnings.warn('semaphore_tracker: %r: %s' % (name, e))
+ finally:
+ pass
diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py
index a358ed4..0c17825 100644
--- a/Lib/multiprocessing/sharedctypes.py
+++ b/Lib/multiprocessing/sharedctypes.py
@@ -10,8 +10,11 @@
import ctypes
import weakref
-from multiprocessing import heap, RLock
-from multiprocessing.forking import assert_spawning, ForkingPickler
+from . import heap
+from . import get_context
+
+from .context import assert_spawning
+from .reduction import ForkingPickler
__all__ = ['RawValue', 'RawArray', 'Value', 'Array', 'copy', 'synchronized']
@@ -63,7 +66,7 @@ def RawArray(typecode_or_type, size_or_initializer):
result.__init__(*size_or_initializer)
return result
-def Value(typecode_or_type, *args, lock=True):
+def Value(typecode_or_type, *args, lock=True, ctx=None):
'''
Return a synchronization wrapper for a Value
'''
@@ -71,12 +74,13 @@ def Value(typecode_or_type, *args, lock=True):
if lock is False:
return obj
if lock in (True, None):
- lock = RLock()
+ ctx = ctx or get_context()
+ lock = ctx.RLock()
if not hasattr(lock, 'acquire'):
raise AttributeError("'%r' has no method 'acquire'" % lock)
- return synchronized(obj, lock)
+ return synchronized(obj, lock, ctx=ctx)
-def Array(typecode_or_type, size_or_initializer, *, lock=True):
+def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None):
'''
Return a synchronization wrapper for a RawArray
'''
@@ -84,25 +88,27 @@ def Array(typecode_or_type, size_or_initializer, *, lock=True):
if lock is False:
return obj
if lock in (True, None):
- lock = RLock()
+ ctx = ctx or get_context()
+ lock = ctx.RLock()
if not hasattr(lock, 'acquire'):
raise AttributeError("'%r' has no method 'acquire'" % lock)
- return synchronized(obj, lock)
+ return synchronized(obj, lock, ctx=ctx)
def copy(obj):
new_obj = _new_value(type(obj))
ctypes.pointer(new_obj)[0] = obj
return new_obj
-def synchronized(obj, lock=None):
+def synchronized(obj, lock=None, ctx=None):
assert not isinstance(obj, SynchronizedBase), 'object already synchronized'
+ ctx = ctx or get_context()
if isinstance(obj, ctypes._SimpleCData):
- return Synchronized(obj, lock)
+ return Synchronized(obj, lock, ctx)
elif isinstance(obj, ctypes.Array):
if obj._type_ is ctypes.c_char:
- return SynchronizedString(obj, lock)
- return SynchronizedArray(obj, lock)
+ return SynchronizedString(obj, lock, ctx)
+ return SynchronizedArray(obj, lock, ctx)
else:
cls = type(obj)
try:
@@ -112,7 +118,7 @@ def synchronized(obj, lock=None):
d = dict((name, make_property(name)) for name in names)
classname = 'Synchronized' + cls.__name__
scls = class_cache[cls] = type(classname, (SynchronizedBase,), d)
- return scls(obj, lock)
+ return scls(obj, lock, ctx)
#
# Functions for pickling/unpickling
@@ -172,9 +178,13 @@ class_cache = weakref.WeakKeyDictionary()
class SynchronizedBase(object):
- def __init__(self, obj, lock=None):
+ def __init__(self, obj, lock=None, ctx=None):
self._obj = obj
- self._lock = lock or RLock()
+ if lock:
+ self._lock = lock
+ else:
+ ctx = ctx or get_context(force=True)
+ self._lock = ctx.RLock()
self.acquire = self._lock.acquire
self.release = self._lock.release
diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py
new file mode 100644
index 0000000..336e479
--- /dev/null
+++ b/Lib/multiprocessing/spawn.py
@@ -0,0 +1,287 @@
+#
+# Code used to start processes when using the spawn or forkserver
+# start methods.
+#
+# multiprocessing/spawn.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import os
+import pickle
+import sys
+import runpy
+import types
+
+from . import get_start_method, set_start_method
+from . import process
+from . import util
+
+__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
+ 'get_preparation_data', 'get_command_line', 'import_main_path']
+
+#
+# _python_exe is the assumed path to the python executable.
+# People embedding Python want to modify it.
+#
+
+if sys.platform != 'win32':
+ WINEXE = False
+ WINSERVICE = False
+else:
+ WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
+ WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
+
+if WINSERVICE:
+ _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
+else:
+ _python_exe = sys.executable
+
+def set_executable(exe):
+ global _python_exe
+ _python_exe = exe
+
+def get_executable():
+ return _python_exe
+
+#
+#
+#
+
+def is_forking(argv):
+ '''
+ Return whether commandline indicates we are forking
+ '''
+ if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
+ return True
+ else:
+ return False
+
+
+def freeze_support():
+ '''
+ Run code for process object if this in not the main process
+ '''
+ if is_forking(sys.argv):
+ kwds = {}
+ for arg in sys.argv[2:]:
+ name, value = arg.split('=')
+ if value == 'None':
+ kwds[name] = None
+ else:
+ kwds[name] = int(value)
+ spawn_main(**kwds)
+ sys.exit()
+
+
+def get_command_line(**kwds):
+ '''
+ Returns prefix of command line used for spawning a child process
+ '''
+ if getattr(sys, 'frozen', False):
+ return ([sys.executable, '--multiprocessing-fork'] +
+ ['%s=%r' % item for item in kwds.items()])
+ else:
+ prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
+ prog %= ', '.join('%s=%r' % item for item in kwds.items())
+ opts = util._args_from_interpreter_flags()
+ return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
+
+
+def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
+ '''
+ Run code specifed by data received over pipe
+ '''
+ assert is_forking(sys.argv)
+ if sys.platform == 'win32':
+ import msvcrt
+ from .reduction import steal_handle
+ new_handle = steal_handle(parent_pid, pipe_handle)
+ fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
+ else:
+ from . import semaphore_tracker
+ semaphore_tracker._semaphore_tracker._fd = tracker_fd
+ fd = pipe_handle
+ exitcode = _main(fd)
+ sys.exit(exitcode)
+
+
+def _main(fd):
+ with os.fdopen(fd, 'rb', closefd=True) as from_parent:
+ process.current_process()._inheriting = True
+ try:
+ preparation_data = pickle.load(from_parent)
+ prepare(preparation_data)
+ self = pickle.load(from_parent)
+ finally:
+ del process.current_process()._inheriting
+ return self._bootstrap()
+
+
+def _check_not_importing_main():
+ if getattr(process.current_process(), '_inheriting', False):
+ raise RuntimeError('''
+ An attempt has been made to start a new process before the
+ current process has finished its bootstrapping phase.
+
+ This probably means that you are not using fork to start your
+ child processes and you have forgotten to use the proper idiom
+ in the main module:
+
+ if __name__ == '__main__':
+ freeze_support()
+ ...
+
+ The "freeze_support()" line can be omitted if the program
+ is not going to be frozen to produce an executable.''')
+
+
+def get_preparation_data(name):
+ '''
+ Return info about parent needed by child to unpickle process object
+ '''
+ _check_not_importing_main()
+ d = dict(
+ log_to_stderr=util._log_to_stderr,
+ authkey=process.current_process().authkey,
+ )
+
+ if util._logger is not None:
+ d['log_level'] = util._logger.getEffectiveLevel()
+
+ sys_path=sys.path.copy()
+ try:
+ i = sys_path.index('')
+ except ValueError:
+ pass
+ else:
+ sys_path[i] = process.ORIGINAL_DIR
+
+ d.update(
+ name=name,
+ sys_path=sys_path,
+ sys_argv=sys.argv,
+ orig_dir=process.ORIGINAL_DIR,
+ dir=os.getcwd(),
+ start_method=get_start_method(),
+ )
+
+ # Figure out whether to initialise main in the subprocess as a module
+ # or through direct execution (or to leave it alone entirely)
+ main_module = sys.modules['__main__']
+ main_mod_name = getattr(main_module.__spec__, "name", None)
+ if main_mod_name is not None:
+ d['init_main_from_name'] = main_mod_name
+ elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
+ main_path = getattr(main_module, '__file__', None)
+ if main_path is not None:
+ if (not os.path.isabs(main_path) and
+ process.ORIGINAL_DIR is not None):
+ main_path = os.path.join(process.ORIGINAL_DIR, main_path)
+ d['init_main_from_path'] = os.path.normpath(main_path)
+
+ return d
+
+#
+# Prepare current process
+#
+
+old_main_modules = []
+
+def prepare(data):
+ '''
+ Try to get current process ready to unpickle process object
+ '''
+ if 'name' in data:
+ process.current_process().name = data['name']
+
+ if 'authkey' in data:
+ process.current_process().authkey = data['authkey']
+
+ if 'log_to_stderr' in data and data['log_to_stderr']:
+ util.log_to_stderr()
+
+ if 'log_level' in data:
+ util.get_logger().setLevel(data['log_level'])
+
+ if 'sys_path' in data:
+ sys.path = data['sys_path']
+
+ if 'sys_argv' in data:
+ sys.argv = data['sys_argv']
+
+ if 'dir' in data:
+ os.chdir(data['dir'])
+
+ if 'orig_dir' in data:
+ process.ORIGINAL_DIR = data['orig_dir']
+
+ if 'start_method' in data:
+ set_start_method(data['start_method'])
+
+ if 'init_main_from_name' in data:
+ _fixup_main_from_name(data['init_main_from_name'])
+ elif 'init_main_from_path' in data:
+ _fixup_main_from_path(data['init_main_from_path'])
+
+# Multiprocessing module helpers to fix up the main module in
+# spawned subprocesses
+def _fixup_main_from_name(mod_name):
+ # __main__.py files for packages, directories, zip archives, etc, run
+ # their "main only" code unconditionally, so we don't even try to
+ # populate anything in __main__, nor do we make any changes to
+ # __main__ attributes
+ current_main = sys.modules['__main__']
+ if mod_name == "__main__" or mod_name.endswith(".__main__"):
+ return
+
+ # If this process was forked, __main__ may already be populated
+ if getattr(current_main.__spec__, "name", None) == mod_name:
+ return
+
+ # Otherwise, __main__ may contain some non-main code where we need to
+ # support unpickling it properly. We rerun it as __mp_main__ and make
+ # the normal __main__ an alias to that
+ old_main_modules.append(current_main)
+ main_module = types.ModuleType("__mp_main__")
+ main_content = runpy.run_module(mod_name,
+ run_name="__mp_main__",
+ alter_sys=True)
+ main_module.__dict__.update(main_content)
+ sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def _fixup_main_from_path(main_path):
+ # If this process was forked, __main__ may already be populated
+ current_main = sys.modules['__main__']
+
+ # Unfortunately, the main ipython launch script historically had no
+ # "if __name__ == '__main__'" guard, so we work around that
+ # by treating it like a __main__.py file
+ # See https://github.com/ipython/ipython/issues/4698
+ main_name = os.path.splitext(os.path.basename(main_path))[0]
+ if main_name == 'ipython':
+ return
+
+ # Otherwise, if __file__ already has the setting we expect,
+ # there's nothing more to do
+ if getattr(current_main, '__file__', None) == main_path:
+ return
+
+ # If the parent process has sent a path through rather than a module
+ # name we assume it is an executable script that may contain
+ # non-main code that needs to be executed
+ old_main_modules.append(current_main)
+ main_module = types.ModuleType("__mp_main__")
+ main_content = runpy.run_path(main_path,
+ run_name="__mp_main__")
+ main_module.__dict__.update(main_content)
+ sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
+
+
+def import_main_path(main_path):
+ '''
+ Set sys.modules['__main__'] to module at main_path
+ '''
+ _fixup_main_from_path(main_path)
diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py
index 0faca78..dea1cbd 100644
--- a/Lib/multiprocessing/synchronize.py
+++ b/Lib/multiprocessing/synchronize.py
@@ -13,18 +13,20 @@ __all__ = [
import threading
import sys
-
+import tempfile
import _multiprocessing
-from multiprocessing.process import current_process
-from multiprocessing.util import register_after_fork, debug
-from multiprocessing.forking import assert_spawning, Popen
+
from time import time as _time
+from . import context
+from . import process
+from . import util
+
# Try to import the mp.synchronize module cleanly, if it fails
# raise ImportError for platforms lacking a working sem_open implementation.
# See issue 3770
try:
- from _multiprocessing import SemLock
+ from _multiprocessing import SemLock, sem_unlink
except (ImportError):
raise ImportError("This platform lacks a functioning sem_open" +
" implementation, therefore, the required" +
@@ -44,15 +46,47 @@ SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
class SemLock(object):
- def __init__(self, kind, value, maxvalue):
- sl = self._semlock = _multiprocessing.SemLock(kind, value, maxvalue)
- debug('created semlock with handle %s' % sl.handle)
+ _rand = tempfile._RandomNameSequence()
+
+ def __init__(self, kind, value, maxvalue, *, ctx):
+ if ctx is None:
+ ctx = context._default_context.get_context()
+ name = ctx.get_start_method()
+ unlink_now = sys.platform == 'win32' or name == 'fork'
+ for i in range(100):
+ try:
+ sl = self._semlock = _multiprocessing.SemLock(
+ kind, value, maxvalue, self._make_name(),
+ unlink_now)
+ except FileExistsError:
+ pass
+ else:
+ break
+ else:
+ raise FileExistsError('cannot find name for semaphore')
+
+ util.debug('created semlock with handle %s' % sl.handle)
self._make_methods()
if sys.platform != 'win32':
def _after_fork(obj):
obj._semlock._after_fork()
- register_after_fork(self, _after_fork)
+ util.register_after_fork(self, _after_fork)
+
+ if self._semlock.name is not None:
+ # We only get here if we are on Unix with forking
+ # disabled. When the object is garbage collected or the
+ # process shuts down we unlink the semaphore name
+ from .semaphore_tracker import register
+ register(self._semlock.name)
+ util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
+ exitpriority=0)
+
+ @staticmethod
+ def _cleanup(name):
+ from .semaphore_tracker import unregister
+ sem_unlink(name)
+ unregister(name)
def _make_methods(self):
self.acquire = self._semlock.acquire
@@ -65,23 +99,32 @@ class SemLock(object):
return self._semlock.__exit__(*args)
def __getstate__(self):
- assert_spawning(self)
+ context.assert_spawning(self)
sl = self._semlock
- return (Popen.duplicate_for_child(sl.handle), sl.kind, sl.maxvalue)
+ if sys.platform == 'win32':
+ h = context.get_spawning_popen().duplicate_for_child(sl.handle)
+ else:
+ h = sl.handle
+ return (h, sl.kind, sl.maxvalue, sl.name)
def __setstate__(self, state):
self._semlock = _multiprocessing.SemLock._rebuild(*state)
- debug('recreated blocker with handle %r' % state[0])
+ util.debug('recreated blocker with handle %r' % state[0])
self._make_methods()
+ @staticmethod
+ def _make_name():
+ return '%s-%s' % (process.current_process()._config['semprefix'],
+ next(SemLock._rand))
+
#
# Semaphore
#
class Semaphore(SemLock):
- def __init__(self, value=1):
- SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
+ def __init__(self, value=1, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)
def get_value(self):
return self._semlock._get_value()
@@ -99,8 +142,8 @@ class Semaphore(SemLock):
class BoundedSemaphore(Semaphore):
- def __init__(self, value=1):
- SemLock.__init__(self, SEMAPHORE, value, value)
+ def __init__(self, value=1, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
def __repr__(self):
try:
@@ -116,13 +159,13 @@ class BoundedSemaphore(Semaphore):
class Lock(SemLock):
- def __init__(self):
- SemLock.__init__(self, SEMAPHORE, 1, 1)
+ def __init__(self, *, ctx):
+ SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx)
def __repr__(self):
try:
if self._semlock._is_mine():
- name = current_process().name
+ name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
elif self._semlock._get_value() == 1:
@@ -141,13 +184,13 @@ class Lock(SemLock):
class RLock(SemLock):
- def __init__(self):
- SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1)
+ def __init__(self, *, ctx):
+ SemLock.__init__(self, RECURSIVE_MUTEX, 1, 1, ctx=ctx)
def __repr__(self):
try:
if self._semlock._is_mine():
- name = current_process().name
+ name = process.current_process().name
if threading.current_thread().name != 'MainThread':
name += '|' + threading.current_thread().name
count = self._semlock._count()
@@ -167,15 +210,15 @@ class RLock(SemLock):
class Condition(object):
- def __init__(self, lock=None):
- self._lock = lock or RLock()
- self._sleeping_count = Semaphore(0)
- self._woken_count = Semaphore(0)
- self._wait_semaphore = Semaphore(0)
+ def __init__(self, lock=None, *, ctx):
+ self._lock = lock or ctx.RLock()
+ self._sleeping_count = ctx.Semaphore(0)
+ self._woken_count = ctx.Semaphore(0)
+ self._wait_semaphore = ctx.Semaphore(0)
self._make_methods()
def __getstate__(self):
- assert_spawning(self)
+ context.assert_spawning(self)
return (self._lock, self._sleeping_count,
self._woken_count, self._wait_semaphore)
@@ -289,9 +332,9 @@ class Condition(object):
class Event(object):
- def __init__(self):
- self._cond = Condition(Lock())
- self._flag = Semaphore(0)
+ def __init__(self, *, ctx):
+ self._cond = ctx.Condition(ctx.Lock())
+ self._flag = ctx.Semaphore(0)
def is_set(self):
self._cond.acquire()
@@ -340,11 +383,11 @@ class Event(object):
class Barrier(threading.Barrier):
- def __init__(self, parties, action=None, timeout=None):
+ def __init__(self, parties, action=None, timeout=None, *, ctx):
import struct
- from multiprocessing.heap import BufferWrapper
+ from .heap import BufferWrapper
wrapper = BufferWrapper(struct.calcsize('i') * 2)
- cond = Condition()
+ cond = ctx.Condition()
self.__setstate__((parties, action, timeout, cond, wrapper))
self._state = 0
self._count = 0
diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py
index f5862b4..0b695e4 100644
--- a/Lib/multiprocessing/util.py
+++ b/Lib/multiprocessing/util.py
@@ -7,8 +7,6 @@
# Licensed to PSF under a Contributor Agreement.
#
-import sys
-import functools
import os
import itertools
import weakref
@@ -17,13 +15,13 @@ import threading # we want threading to install it's
# cleanup function before multiprocessing does
from subprocess import _args_from_interpreter_flags
-from multiprocessing.process import current_process, active_children
+from . import process
__all__ = [
'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger',
'log_to_stderr', 'get_temp_dir', 'register_after_fork',
'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
- 'SUBDEBUG', 'SUBWARNING',
+ 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
]
#
@@ -71,8 +69,6 @@ def get_logger():
_logger = logging.getLogger(LOGGER_NAME)
_logger.propagate = 0
- logging.addLevelName(SUBDEBUG, 'SUBDEBUG')
- logging.addLevelName(SUBWARNING, 'SUBWARNING')
# XXX multiprocessing should cleanup before logging
if hasattr(atexit, 'unregister'):
@@ -111,13 +107,14 @@ def log_to_stderr(level=None):
def get_temp_dir():
# get name of a temp directory which will be automatically cleaned up
- if current_process()._tempdir is None:
+ tempdir = process.current_process()._config.get('tempdir')
+ if tempdir is None:
import shutil, tempfile
tempdir = tempfile.mkdtemp(prefix='pymp-')
info('created temp directory %s', tempdir)
Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100)
- current_process()._tempdir = tempdir
- return current_process()._tempdir
+ process.current_process()._config['tempdir'] = tempdir
+ return tempdir
#
# Support for reinitialization of objects when bootstrapping a child process
@@ -273,8 +270,8 @@ def is_exiting():
_exiting = False
def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
- active_children=active_children,
- current_process=current_process):
+ active_children=process.active_children,
+ current_process=process.current_process):
# We hold on to references to functions in the arglist due to the
# situation described below, where this function is called after this
# module's globals are destroyed.
@@ -303,7 +300,7 @@ def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
# #9207.
for p in active_children():
- if p._daemonic:
+ if p.daemon:
info('calling terminate() for daemon %s', p.name)
p._popen.terminate()
@@ -335,3 +332,36 @@ class ForkAwareLocal(threading.local):
register_after_fork(self, lambda obj : obj.__dict__.clear())
def __reduce__(self):
return type(self), ()
+
+#
+# Close fds except those specified
+#
+
+try:
+ MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+ MAXFD = 256
+
+def close_all_fds_except(fds):
+ fds = list(fds) + [-1, MAXFD]
+ fds.sort()
+ assert fds[-1] == MAXFD, 'fd too large'
+ for i in range(len(fds) - 1):
+ os.closerange(fds[i]+1, fds[i+1])
+
+#
+# Start a program with only specified fds kept open
+#
+
+def spawnv_passfds(path, args, passfds):
+ import _posixsubprocess
+ passfds = sorted(passfds)
+ errpipe_read, errpipe_write = os.pipe()
+ try:
+ return _posixsubprocess.fork_exec(
+ args, [os.fsencode(path)], True, passfds, None, None,
+ -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
+ False, False, None)
+ finally:
+ os.close(errpipe_read)
+ os.close(errpipe_write)
diff --git a/Lib/netrc.py b/Lib/netrc.py
index 2aa48f3..bbb3d23 100644
--- a/Lib/netrc.py
+++ b/Lib/netrc.py
@@ -3,8 +3,6 @@
# Module and documentation by Eric S. Raymond, 21 Dec 1998
import os, shlex, stat
-if os.name == 'posix':
- import pwd
__all__ = ["netrc", "NetrcParseError"]
@@ -28,7 +26,7 @@ class netrc:
try:
file = os.path.join(os.environ['HOME'], ".netrc")
except KeyError:
- raise IOError("Could not find .netrc: $HOME is not set")
+ raise OSError("Could not find .netrc: $HOME is not set")
self.hosts = {}
self.macros = {}
with open(file) as fp:
@@ -92,6 +90,7 @@ class netrc:
if os.name == 'posix' and default_netrc:
prop = os.fstat(fp.fileno())
if prop.st_uid != os.getuid():
+ import pwd
try:
fowner = pwd.getpwuid(prop.st_uid)[0]
except KeyError:
diff --git a/Lib/nntplib.py b/Lib/nntplib.py
index 02cc37c..e2c6579 100644
--- a/Lib/nntplib.py
+++ b/Lib/nntplib.py
@@ -279,7 +279,7 @@ def _unparse_datetime(dt, legacy=False):
if _have_ssl:
- def _encrypt_on(sock, context):
+ def _encrypt_on(sock, context, hostname):
"""Wrap a socket in SSL/TLS. Arguments:
- sock: Socket to wrap
- context: SSL context to use for the encrypted connection
@@ -288,10 +288,9 @@ if _have_ssl:
"""
# Generate a default SSL context if none was passed.
if context is None:
- context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
- # SSLv2 considered harmful.
- context.options |= ssl.OP_NO_SSLv2
- return context.wrap_socket(sock)
+ context = ssl._create_stdlib_context()
+ server_hostname = hostname if ssl.HAS_SNI else None
+ return context.wrap_socket(sock, server_hostname=server_hostname)
# The classes themselves
@@ -366,7 +365,7 @@ class _NNTPBase:
if is_connected():
try:
self.quit()
- except (socket.error, EOFError):
+ except (OSError, EOFError):
pass
finally:
if is_connected():
@@ -956,7 +955,7 @@ class _NNTPBase:
if auth:
user = auth[0]
password = auth[2]
- except IOError:
+ except OSError:
pass
# Perform NNTP authentication if needed.
if not user:
@@ -1007,7 +1006,7 @@ class _NNTPBase:
resp = self._shortcmd('STARTTLS')
if resp.startswith('382'):
self.file.close()
- self.sock = _encrypt_on(self.sock, context)
+ self.sock = _encrypt_on(self.sock, context, self.host)
self.file = self.sock.makefile("rwb")
self.tls_on = True
# Capabilities may change after TLS starts up, so ask for them
@@ -1067,7 +1066,7 @@ if _have_ssl:
in default port and the `ssl_context` argument for SSL connections.
"""
self.sock = socket.create_connection((host, port), timeout)
- self.sock = _encrypt_on(self.sock, ssl_context)
+ self.sock = _encrypt_on(self.sock, ssl_context, host)
file = self.sock.makefile("rwb")
_NNTPBase.__init__(self, file, host,
readermode=readermode, timeout=timeout)
@@ -1086,7 +1085,6 @@ if _have_ssl:
# Test retrieval when run as a script.
if __name__ == '__main__':
import argparse
- from email.utils import parsedate
parser = argparse.ArgumentParser(description="""\
nntplib built-in demo - display the latest articles in a newsgroup""")
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index 303e586..af3fb87 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -17,7 +17,7 @@ __all__ = ["normcase","isabs","join","splitdrive","split","splitext",
"ismount", "expanduser","expandvars","normpath","abspath",
"splitunc","curdir","pardir","sep","pathsep","defpath","altsep",
"extsep","devnull","realpath","supports_unicode_filenames","relpath",
- "samefile", "sameopenfile",]
+ "samefile", "sameopenfile", "samestat",]
# strings representing various path-related bits and pieces
# These are primarily for export; internally, they are hardcoded.
@@ -30,9 +30,6 @@ altsep = '/'
defpath = '.;C:\\bin'
if 'ce' in sys.builtin_module_names:
defpath = '\\Windows'
-elif 'os2' in sys.builtin_module_names:
- # OS/2 w/ VACPP
- altsep = '/'
devnull = 'nul'
def _get_empty(path):
@@ -260,12 +257,11 @@ def dirname(p):
def islink(path):
"""Test whether a path is a symbolic link.
- This will always return false for Windows prior to 6.0
- and for OS/2.
+ This will always return false for Windows prior to 6.0.
"""
try:
st = os.lstat(path)
- except (os.error, AttributeError):
+ except (OSError, AttributeError):
return False
return stat.S_ISLNK(st.st_mode)
@@ -275,20 +271,39 @@ def lexists(path):
"""Test whether a path exists. Returns True for broken symbolic links"""
try:
st = os.lstat(path)
- except (os.error, WindowsError):
+ except OSError:
return False
return True
-# Is a path a mount point? Either a root (with or without drive letter)
-# or an UNC path with at most a / or \ after the mount point.
-
+# Is a path a mount point?
+# Any drive letter root (eg c:\)
+# Any share UNC (eg \\server\share)
+# Any volume mounted on a filesystem folder
+#
+# No one method detects all three situations. Historically we've lexically
+# detected drive letter roots and share UNCs. The canonical approach to
+# detecting mounted volumes (querying the reparse tag) fails for the most
+# common case: drive letter roots. The alternative which uses GetVolumePathName
+# fails if the drive letter is the result of a SUBST.
+try:
+ from nt import _getvolumepathname
+except ImportError:
+ _getvolumepathname = None
def ismount(path):
- """Test whether a path is a mount point (defined as root of drive)"""
+ """Test whether a path is a mount point (a drive root, the root of a
+ share, or a mounted volume)"""
seps = _get_bothseps(path)
+ path = abspath(path)
root, rest = splitdrive(path)
if root and root[0] in seps:
return (not rest) or (rest in seps)
- return rest in seps
+ if rest in seps:
+ return True
+
+ if _getvolumepathname:
+ return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+ else:
+ return False
# Expand paths beginning with '~' or '~user'.
@@ -530,7 +545,7 @@ else: # use native Windows method on Windows
if path: # Empty path must return current working directory.
try:
path = _getfullpathname(path)
- except WindowsError:
+ except OSError:
pass # Bad path - return unchanged.
elif isinstance(path, bytes):
path = os.getcwdb()
@@ -598,23 +613,6 @@ except (AttributeError, ImportError):
def _getfinalpathname(f):
return normcase(abspath(f))
-def samefile(f1, f2):
- "Test whether two pathnames reference the same actual file"
- return _getfinalpathname(f1) == _getfinalpathname(f2)
-
-
-try:
- from nt import _getfileinformation
-except ImportError:
- # On other operating systems, just return the fd and see that
- # it compares equal in sameopenfile.
- def _getfileinformation(fd):
- return fd
-
-def sameopenfile(f1, f2):
- """Test whether two file objects reference the same file"""
- return _getfileinformation(f1) == _getfileinformation(f2)
-
try:
# The genericpath.isdir implementation uses os.stat and checks the mode
diff --git a/Lib/nturl2path.py b/Lib/nturl2path.py
index 511dcec..5a6d44a 100644
--- a/Lib/nturl2path.py
+++ b/Lib/nturl2path.py
@@ -23,7 +23,7 @@ def url2pathname(url):
comp = url.split('|')
if len(comp) != 2 or comp[0][-1] not in string.ascii_letters:
error = 'Bad URL: ' + url
- raise IOError(error)
+ raise OSError(error)
drive = comp[0][-1].upper()
components = comp[1].split('/')
path = drive + ':'
@@ -55,7 +55,7 @@ def pathname2url(p):
comp = p.split(':')
if len(comp) != 2 or len(comp[0]) > 1:
error = 'Bad path: ' + p
- raise IOError(error)
+ raise OSError(error)
drive = urllib.parse.quote(comp[0].upper())
components = comp[1].split('\\')
diff --git a/Lib/opcode.py b/Lib/opcode.py
index a639fe3..0bd1ee6 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -8,6 +8,19 @@ __all__ = ["cmp_op", "hasconst", "hasname", "hasjrel", "hasjabs",
"haslocal", "hascompare", "hasfree", "opname", "opmap",
"HAVE_ARGUMENT", "EXTENDED_ARG", "hasnargs"]
+# It's a chicken-and-egg I'm afraid:
+# We're imported before _opcode's made.
+# With exception unheeded
+# (stack_effect is not needed)
+# Both our chickens and eggs are allayed.
+# --Larry Hastings, 2013/11/23
+
+try:
+ from _opcode import stack_effect
+ __all__.append('stack_effect')
+except ImportError:
+ pass
+
cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is',
'is not', 'exception match', 'BAD')
@@ -84,7 +97,6 @@ def_op('BINARY_XOR', 65)
def_op('BINARY_OR', 66)
def_op('INPLACE_POWER', 67)
def_op('GET_ITER', 68)
-def_op('STORE_LOCALS', 69)
def_op('PRINT_EXPR', 70)
def_op('LOAD_BUILD_CLASS', 71)
@@ -179,6 +191,9 @@ def_op('LIST_APPEND', 145)
def_op('SET_ADD', 146)
def_op('MAP_ADD', 147)
+def_op('LOAD_CLASSDEREF', 148)
+hasfree.append(148)
+
def_op('EXTENDED_ARG', 144)
EXTENDED_ARG = 144
diff --git a/Lib/operator.py b/Lib/operator.py
new file mode 100644
index 0000000..b60349f
--- /dev/null
+++ b/Lib/operator.py
@@ -0,0 +1,411 @@
+"""
+Operator Interface
+
+This module exports a set of functions corresponding to the intrinsic
+operators of Python. For example, operator.add(x, y) is equivalent
+to the expression x+y. The function names are those used for special
+methods; variants without leading and trailing '__' are also provided
+for convenience.
+
+This is the pure Python implementation of the module.
+"""
+
+__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf',
+ 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand',
+ 'iconcat', 'ifloordiv', 'ilshift', 'imod', 'imul', 'index',
+ 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', 'is_',
+ 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le',
+ 'length_hint', 'lshift', 'lt', 'methodcaller', 'mod', 'mul', 'ne',
+ 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub',
+ 'truediv', 'truth', 'xor']
+
+from builtins import abs as _abs
+
+
+# Comparison Operations *******************************************************#
+
+def lt(a, b):
+ "Same as a < b."
+ return a < b
+
+def le(a, b):
+ "Same as a <= b."
+ return a <= b
+
+def eq(a, b):
+ "Same as a == b."
+ return a == b
+
+def ne(a, b):
+ "Same as a != b."
+ return a != b
+
+def ge(a, b):
+ "Same as a >= b."
+ return a >= b
+
+def gt(a, b):
+ "Same as a > b."
+ return a > b
+
+# Logical Operations **********************************************************#
+
+def not_(a):
+ "Same as not a."
+ return not a
+
+def truth(a):
+ "Return True if a is true, False otherwise."
+ return True if a else False
+
+def is_(a, b):
+ "Same as a is b."
+ return a is b
+
+def is_not(a, b):
+ "Same as a is not b."
+ return a is not b
+
+# Mathematical/Bitwise Operations *********************************************#
+
+def abs(a):
+ "Same as abs(a)."
+ return _abs(a)
+
+def add(a, b):
+ "Same as a + b."
+ return a + b
+
+def and_(a, b):
+ "Same as a & b."
+ return a & b
+
+def floordiv(a, b):
+ "Same as a // b."
+ return a // b
+
+def index(a):
+ "Same as a.__index__()."
+ return a.__index__()
+
+def inv(a):
+ "Same as ~a."
+ return ~a
+invert = inv
+
+def lshift(a, b):
+ "Same as a << b."
+ return a << b
+
+def mod(a, b):
+ "Same as a % b."
+ return a % b
+
+def mul(a, b):
+ "Same as a * b."
+ return a * b
+
+def neg(a):
+ "Same as -a."
+ return -a
+
+def or_(a, b):
+ "Same as a | b."
+ return a | b
+
+def pos(a):
+ "Same as +a."
+ return +a
+
+def pow(a, b):
+ "Same as a ** b."
+ return a ** b
+
+def rshift(a, b):
+ "Same as a >> b."
+ return a >> b
+
+def sub(a, b):
+ "Same as a - b."
+ return a - b
+
+def truediv(a, b):
+ "Same as a / b."
+ return a / b
+
+def xor(a, b):
+ "Same as a ^ b."
+ return a ^ b
+
+# Sequence Operations *********************************************************#
+
+def concat(a, b):
+ "Same as a + b, for a and b sequences."
+ if not hasattr(a, '__getitem__'):
+ msg = "'%s' object can't be concatenated" % type(a).__name__
+ raise TypeError(msg)
+ return a + b
+
+def contains(a, b):
+ "Same as b in a (note reversed operands)."
+ return b in a
+
+def countOf(a, b):
+ "Return the number of times b occurs in a."
+ count = 0
+ for i in a:
+ if i == b:
+ count += 1
+ return count
+
+def delitem(a, b):
+ "Same as del a[b]."
+ del a[b]
+
+def getitem(a, b):
+ "Same as a[b]."
+ return a[b]
+
+def indexOf(a, b):
+ "Return the first index of b in a."
+ for i, j in enumerate(a):
+ if j == b:
+ return i
+ else:
+ raise ValueError('sequence.index(x): x not in sequence')
+
+def setitem(a, b, c):
+ "Same as a[b] = c."
+ a[b] = c
+
+def length_hint(obj, default=0):
+ """
+ Return an estimate of the number of items in obj.
+ This is useful for presizing containers when building from an iterable.
+
+ If the object supports len(), the result will be exact. Otherwise, it may
+ over- or under-estimate by an arbitrary amount. The result will be an
+ integer >= 0.
+ """
+ if not isinstance(default, int):
+ msg = ("'%s' object cannot be interpreted as an integer" %
+ type(default).__name__)
+ raise TypeError(msg)
+
+ try:
+ return len(obj)
+ except TypeError:
+ pass
+
+ try:
+ hint = type(obj).__length_hint__
+ except AttributeError:
+ return default
+
+ try:
+ val = hint(obj)
+ except TypeError:
+ return default
+ if val is NotImplemented:
+ return default
+ if not isinstance(val, int):
+ msg = ('__length_hint__ must be integer, not %s' %
+ type(val).__name__)
+ raise TypeError(msg)
+ if val < 0:
+ msg = '__length_hint__() should return >= 0'
+ raise ValueError(msg)
+ return val
+
+# Generalized Lookup Objects **************************************************#
+
+class attrgetter:
+ """
+ Return a callable object that fetches the given attribute(s) from its operand.
+ After f = attrgetter('name'), the call f(r) returns r.name.
+ After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).
+ After h = attrgetter('name.first', 'name.last'), the call h(r) returns
+ (r.name.first, r.name.last).
+ """
+ def __init__(self, attr, *attrs):
+ if not attrs:
+ if not isinstance(attr, str):
+ raise TypeError('attribute name must be a string')
+ names = attr.split('.')
+ def func(obj):
+ for name in names:
+ obj = getattr(obj, name)
+ return obj
+ self._call = func
+ else:
+ getters = tuple(map(attrgetter, (attr,) + attrs))
+ def func(obj):
+ return tuple(getter(obj) for getter in getters)
+ self._call = func
+
+ def __call__(self, obj):
+ return self._call(obj)
+
+class itemgetter:
+ """
+ Return a callable object that fetches the given item(s) from its operand.
+ After f = itemgetter(2), the call f(r) returns r[2].
+ After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])
+ """
+ def __init__(self, item, *items):
+ if not items:
+ def func(obj):
+ return obj[item]
+ self._call = func
+ else:
+ items = (item,) + items
+ def func(obj):
+ return tuple(obj[i] for i in items)
+ self._call = func
+
+ def __call__(self, obj):
+ return self._call(obj)
+
+class methodcaller:
+ """
+ Return a callable object that calls the given method on its operand.
+ After f = methodcaller('name'), the call f(r) returns r.name().
+ After g = methodcaller('name', 'date', foo=1), the call g(r) returns
+ r.name('date', foo=1).
+ """
+
+ def __init__(*args, **kwargs):
+ if len(args) < 2:
+ msg = "methodcaller needs at least one argument, the method name"
+ raise TypeError(msg)
+ self = args[0]
+ self._name = args[1]
+ self._args = args[2:]
+ self._kwargs = kwargs
+
+ def __call__(self, obj):
+ return getattr(obj, self._name)(*self._args, **self._kwargs)
+
+# In-place Operations *********************************************************#
+
+def iadd(a, b):
+ "Same as a += b."
+ a += b
+ return a
+
+def iand(a, b):
+ "Same as a &= b."
+ a &= b
+ return a
+
+def iconcat(a, b):
+ "Same as a += b, for a and b sequences."
+ if not hasattr(a, '__getitem__'):
+ msg = "'%s' object can't be concatenated" % type(a).__name__
+ raise TypeError(msg)
+ a += b
+ return a
+
+def ifloordiv(a, b):
+ "Same as a //= b."
+ a //= b
+ return a
+
+def ilshift(a, b):
+ "Same as a <<= b."
+ a <<= b
+ return a
+
+def imod(a, b):
+ "Same as a %= b."
+ a %= b
+ return a
+
+def imul(a, b):
+ "Same as a *= b."
+ a *= b
+ return a
+
+def ior(a, b):
+ "Same as a |= b."
+ a |= b
+ return a
+
+def ipow(a, b):
+ "Same as a **= b."
+ a **=b
+ return a
+
+def irshift(a, b):
+ "Same as a >>= b."
+ a >>= b
+ return a
+
+def isub(a, b):
+ "Same as a -= b."
+ a -= b
+ return a
+
+def itruediv(a, b):
+ "Same as a /= b."
+ a /= b
+ return a
+
+def ixor(a, b):
+ "Same as a ^= b."
+ a ^= b
+ return a
+
+
+try:
+ from _operator import *
+except ImportError:
+ pass
+else:
+ from _operator import __doc__
+
+# All of these "__func__ = func" assignments have to happen after importing
+# from _operator to make sure they're set to the right function
+__lt__ = lt
+__le__ = le
+__eq__ = eq
+__ne__ = ne
+__ge__ = ge
+__gt__ = gt
+__not__ = not_
+__abs__ = abs
+__add__ = add
+__and__ = and_
+__floordiv__ = floordiv
+__index__ = index
+__inv__ = inv
+__invert__ = invert
+__lshift__ = lshift
+__mod__ = mod
+__mul__ = mul
+__neg__ = neg
+__or__ = or_
+__pos__ = pos
+__pow__ = pow
+__rshift__ = rshift
+__sub__ = sub
+__truediv__ = truediv
+__xor__ = xor
+__concat__ = concat
+__contains__ = contains
+__delitem__ = delitem
+__getitem__ = getitem
+__setitem__ = setitem
+__iadd__ = iadd
+__iand__ = iand
+__iconcat__ = iconcat
+__ifloordiv__ = ifloordiv
+__ilshift__ = ilshift
+__imod__ = imod
+__imul__ = imul
+__ior__ = ior
+__ipow__ = ipow
+__irshift__ = irshift
+__isub__ = isub
+__itruediv__ = itruediv
+__ixor__ = ixor
diff --git a/Lib/os.py b/Lib/os.py
index 87689cc..fa0fbb9 100644
--- a/Lib/os.py
+++ b/Lib/os.py
@@ -1,9 +1,9 @@
r"""OS routines for Mac, NT, or Posix depending on what system we're on.
This exports:
- - all functions from posix, nt, os2, or ce, e.g. unlink, stat, etc.
+ - all functions from posix, nt or ce, e.g. unlink, stat, etc.
- os.path is either posixpath or ntpath
- - os.name is either 'posix', 'nt', 'os2' or 'ce'.
+ - os.name is either 'posix', 'nt' or 'ce'.
- os.curdir is a string representing the current directory ('.' or ':')
- os.pardir is a string representing the parent directory ('..' or '::')
- os.sep is the (or a most common) pathname separator ('/' or ':' or '\\')
@@ -81,30 +81,6 @@ elif 'nt' in _names:
except ImportError:
pass
-elif 'os2' in _names:
- name = 'os2'
- linesep = '\r\n'
- from os2 import *
- try:
- from os2 import _exit
- __all__.append('_exit')
- except ImportError:
- pass
- if sys.version.find('EMX GCC') == -1:
- import ntpath as path
- else:
- import os2emxpath as path
- from _emx_link import link
-
- import os2
- __all__.extend(_get_exports_list(os2))
- del os2
-
- try:
- from os2 import _have_functions
- except ImportError:
- pass
-
elif 'ce' in _names:
name = 'ce'
linesep = '\r\n'
@@ -240,7 +216,7 @@ def _get_masked_mode(mode):
# (Inspired by Eric Raymond; the doc strings are mostly his)
def makedirs(name, mode=0o777, exist_ok=False):
- """makedirs(path [, mode=0o777][, exist_ok=False])
+ """makedirs(name [, mode=0o777][, exist_ok=False])
Super-mkdir; create a leaf directory and all intermediate ones.
Works like mkdir, except that any intermediate path segment (not
@@ -256,10 +232,9 @@ def makedirs(name, mode=0o777, exist_ok=False):
if head and tail and not path.exists(head):
try:
makedirs(head, mode, exist_ok)
- except OSError as e:
+ except FileExistsError:
# be happy if someone already created the path
- if e.errno != errno.EEXIST:
- raise
+ pass
cdir = curdir
if isinstance(tail, bytes):
cdir = bytes(curdir, 'ASCII')
@@ -285,7 +260,7 @@ def makedirs(name, mode=0o777, exist_ok=False):
raise
def removedirs(name):
- """removedirs(path)
+ """removedirs(name)
Super-rmdir; remove a leaf directory and all empty intermediate
ones. Works like rmdir except that, if the leaf directory is
@@ -302,7 +277,7 @@ def removedirs(name):
while head and tail:
try:
rmdir(head)
- except error:
+ except OSError:
break
head, tail = path.split(head)
@@ -329,7 +304,7 @@ def renames(old, new):
if head and tail:
try:
removedirs(head)
- except error:
+ except OSError:
pass
__all__.extend(["makedirs", "removedirs", "renames"])
@@ -365,7 +340,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
By default errors from the os.listdir() call are ignored. If
optional arg 'onerror' is specified, it should be a function; it
- will be called with one argument, an os.error instance. It can
+ will be called with one argument, an OSError instance. It can
report the error to continue with the walk, or raise the exception
to abort the walk. Note that the filename is available as the
filename attribute of the exception object.
@@ -399,10 +374,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False):
# minor reason when (say) a thousand readable directories are still
# left to visit. That logic is copied here.
try:
- # Note that listdir and error are globals in this module due
+ # Note that listdir is global in this module due
# to earlier import-*.
names = listdir(top)
- except error as err:
+ except OSError as err:
if onerror is not None:
onerror(err)
return
@@ -504,7 +479,7 @@ if {open, stat} <= supports_dir_fd and {listdir, stat} <= supports_fd:
try:
orig_st = stat(name, dir_fd=topfd, follow_symlinks=follow_symlinks)
dirfd = open(name, O_RDONLY, dir_fd=topfd)
- except error as err:
+ except OSError as err:
if onerror is not None:
onerror(err)
return
@@ -599,7 +574,7 @@ def _execvpe(file, args, env=None):
fullname = path.join(dir, file)
try:
exec_func(fullname, *argrest)
- except error as e:
+ except OSError as e:
last_exc = e
tb = sys.exc_info()[2]
if (e.errno != errno.ENOENT and e.errno != errno.ENOTDIR
@@ -656,7 +631,7 @@ def get_exec_path(env=None):
# Change environ to automatically call putenv(), unsetenv if they exist.
-from collections.abc import MutableMapping
+from _collections_abc import MutableMapping
class _Environ(MutableMapping):
def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv):
@@ -716,17 +691,19 @@ try:
except NameError:
_putenv = lambda key, value: None
else:
- __all__.append("putenv")
+ if "putenv" not in __all__:
+ __all__.append("putenv")
try:
_unsetenv = unsetenv
except NameError:
_unsetenv = lambda key: _putenv(key, "")
else:
- __all__.append("unsetenv")
+ if "unsetenv" not in __all__:
+ __all__.append("unsetenv")
def _createenviron():
- if name in ('os2', 'nt'):
+ if name == 'nt':
# Where Env Var Names Must Be UPPERCASE
def check_str(value):
if not isinstance(value, str):
@@ -766,7 +743,7 @@ def getenv(key, default=None):
key, default and the result are str."""
return environ.get(key, default)
-supports_bytes_environ = name not in ('os2', 'nt')
+supports_bytes_environ = (name != 'nt')
__all__.extend(("getenv", "supports_bytes_environ"))
if supports_bytes_environ:
@@ -865,7 +842,7 @@ if _exists("fork") and not _exists("spawnv") and _exists("execv"):
elif WIFEXITED(sts):
return WEXITSTATUS(sts)
else:
- raise error("Not stopped, signaled or exited???")
+ raise OSError("Not stopped, signaled or exited???")
def spawnv(mode, file, args):
"""spawnv(mode, file, args) -> integer
@@ -908,6 +885,10 @@ If mode == P_WAIT return the process's exit code if it exits normally;
otherwise return -SIG, where SIG is the signal that killed it. """
return _spawnvef(mode, file, args, env, execvpe)
+
+ __all__.extend(["spawnv", "spawnve", "spawnvp", "spawnvpe"])
+
+
if _exists("spawnv"):
# These aren't supplied by the basic Windows code
# but can be easily implemented in Python
@@ -933,7 +914,7 @@ otherwise return -SIG, where SIG is the signal that killed it. """
return spawnve(mode, file, args[:-1], env)
- __all__.extend(["spawnv", "spawnve", "spawnl", "spawnle",])
+ __all__.extend(["spawnl", "spawnle"])
if _exists("spawnvp"):
@@ -961,34 +942,8 @@ otherwise return -SIG, where SIG is the signal that killed it. """
return spawnvpe(mode, file, args[:-1], env)
- __all__.extend(["spawnvp", "spawnvpe", "spawnlp", "spawnlpe",])
-
-import copyreg as _copyreg
-
-def _make_stat_result(tup, dict):
- return stat_result(tup, dict)
+ __all__.extend(["spawnlp", "spawnlpe"])
-def _pickle_stat_result(sr):
- (type, args) = sr.__reduce__()
- return (_make_stat_result, args)
-
-try:
- _copyreg.pickle(stat_result, _pickle_stat_result, _make_stat_result)
-except NameError: # stat_result may not exist
- pass
-
-def _make_statvfs_result(tup, dict):
- return statvfs_result(tup, dict)
-
-def _pickle_statvfs_result(sr):
- (type, args) = sr.__reduce__()
- return (_make_statvfs_result, args)
-
-try:
- _copyreg.pickle(statvfs_result, _pickle_statvfs_result,
- _make_statvfs_result)
-except NameError: # statvfs_result may not exist
- pass
# Supply os.popen()
def popen(cmd, mode="r", buffering=-1):
diff --git a/Lib/os2emxpath.py b/Lib/os2emxpath.py
deleted file mode 100644
index 0ccbf8a..0000000
--- a/Lib/os2emxpath.py
+++ /dev/null
@@ -1,158 +0,0 @@
-# Module 'os2emxpath' -- common operations on OS/2 pathnames
-"""Common pathname manipulations, OS/2 EMX version.
-
-Instead of importing this module directly, import os and refer to this
-module as os.path.
-"""
-
-import os
-import stat
-from genericpath import *
-from ntpath import (expanduser, expandvars, isabs, islink, splitdrive,
- splitext, split)
-
-__all__ = ["normcase","isabs","join","splitdrive","split","splitext",
- "basename","dirname","commonprefix","getsize","getmtime",
- "getatime","getctime", "islink","exists","lexists","isdir","isfile",
- "ismount","expanduser","expandvars","normpath","abspath",
- "splitunc","curdir","pardir","sep","pathsep","defpath","altsep",
- "extsep","devnull","realpath","supports_unicode_filenames"]
-
-# strings representing various path-related bits and pieces
-curdir = '.'
-pardir = '..'
-extsep = '.'
-sep = '/'
-altsep = '\\'
-pathsep = ';'
-defpath = '.;C:\\bin'
-devnull = 'nul'
-
-# Normalize the case of a pathname and map slashes to backslashes.
-# Other normalizations (such as optimizing '../' away) are not done
-# (this is done by normpath).
-
-def normcase(s):
- """Normalize case of pathname.
-
- Makes all characters lowercase and all altseps into seps."""
- if not isinstance(s, (bytes, str)):
- raise TypeError("normcase() argument must be str or bytes, "
- "not '{}'".format(s.__class__.__name__))
- return s.replace('\\', '/').lower()
-
-
-# Join two (or more) paths.
-
-def join(a, *p):
- """Join two or more pathname components, inserting sep as needed"""
- path = a
- for b in p:
- if isabs(b):
- path = b
- elif path == '' or path[-1:] in '/\\:':
- path = path + b
- else:
- path = path + '/' + b
- return path
-
-
-# Parse UNC paths
-def splitunc(p):
- """Split a pathname into UNC mount point and relative path specifiers.
-
- Return a 2-tuple (unc, rest); either part may be empty.
- If unc is not empty, it has the form '//host/mount' (or similar
- using backslashes). unc+rest is always the input path.
- Paths containing drive letters never have an UNC part.
- """
- if p[1:2] == ':':
- return '', p # Drive letter present
- firstTwo = p[0:2]
- if firstTwo == '/' * 2 or firstTwo == '\\' * 2:
- # is a UNC path:
- # vvvvvvvvvvvvvvvvvvvv equivalent to drive letter
- # \\machine\mountpoint\directories...
- # directory ^^^^^^^^^^^^^^^
- normp = normcase(p)
- index = normp.find('/', 2)
- if index == -1:
- ##raise RuntimeError, 'illegal UNC path: "' + p + '"'
- return ("", p)
- index = normp.find('/', index + 1)
- if index == -1:
- index = len(p)
- return p[:index], p[index:]
- return '', p
-
-
-# Return the tail (basename) part of a path.
-
-def basename(p):
- """Returns the final component of a pathname"""
- return split(p)[1]
-
-
-# Return the head (dirname) part of a path.
-
-def dirname(p):
- """Returns the directory component of a pathname"""
- return split(p)[0]
-
-
-# alias exists to lexists
-lexists = exists
-
-
-# Is a path a directory?
-
-# Is a path a mount point? Either a root (with or without drive letter)
-# or an UNC path with at most a / or \ after the mount point.
-
-def ismount(path):
- """Test whether a path is a mount point (defined as root of drive)"""
- unc, rest = splitunc(path)
- if unc:
- return rest in ("", "/", "\\")
- p = splitdrive(path)[1]
- return len(p) == 1 and p[0] in '/\\'
-
-
-# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.
-
-def normpath(path):
- """Normalize path, eliminating double slashes, etc."""
- path = path.replace('\\', '/')
- prefix, path = splitdrive(path)
- while path[:1] == '/':
- prefix = prefix + '/'
- path = path[1:]
- comps = path.split('/')
- i = 0
- while i < len(comps):
- if comps[i] == '.':
- del comps[i]
- elif comps[i] == '..' and i > 0 and comps[i-1] not in ('', '..'):
- del comps[i-1:i+1]
- i = i - 1
- elif comps[i] == '' and i > 0 and comps[i-1] != '':
- del comps[i]
- else:
- i = i + 1
- # If the path is now empty, substitute '.'
- if not prefix and not comps:
- comps.append('.')
- return prefix + '/'.join(comps)
-
-
-# Return an absolute path.
-def abspath(path):
- """Return the absolute version of a path"""
- if not isabs(path):
- path = join(os.getcwd(), path)
- return normpath(path)
-
-# realpath is a no-op on systems without islink support
-realpath = abspath
-
-supports_unicode_filenames = False
diff --git a/Lib/pathlib.py b/Lib/pathlib.py
new file mode 100644
index 0000000..8d08e8b
--- /dev/null
+++ b/Lib/pathlib.py
@@ -0,0 +1,1288 @@
+import fnmatch
+import functools
+import io
+import ntpath
+import os
+import posixpath
+import re
+import sys
+from collections import Sequence
+from contextlib import contextmanager
+from errno import EINVAL, ENOENT
+from operator import attrgetter
+from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO
+from urllib.parse import quote_from_bytes as urlquote_from_bytes
+
+
+supports_symlinks = True
+try:
+ import nt
+except ImportError:
+ nt = None
+else:
+ if sys.getwindowsversion()[:2] >= (6, 0):
+ from nt import _getfinalpathname
+ else:
+ supports_symlinks = False
+ _getfinalpathname = None
+
+
+__all__ = [
+ "PurePath", "PurePosixPath", "PureWindowsPath",
+ "Path", "PosixPath", "WindowsPath",
+ ]
+
+#
+# Internals
+#
+
+def _is_wildcard_pattern(pat):
+ # Whether this pattern needs actual matching using fnmatch, or can
+ # be looked up directly as a file.
+ return "*" in pat or "?" in pat or "[" in pat
+
+
+class _Flavour(object):
+ """A flavour implements a particular (platform-specific) set of path
+ semantics."""
+
+ def __init__(self):
+ self.join = self.sep.join
+
+ def parse_parts(self, parts):
+ parsed = []
+ sep = self.sep
+ altsep = self.altsep
+ drv = root = ''
+ it = reversed(parts)
+ for part in it:
+ if not part:
+ continue
+ if altsep:
+ part = part.replace(altsep, sep)
+ drv, root, rel = self.splitroot(part)
+ if sep in rel:
+ for x in reversed(rel.split(sep)):
+ if x and x != '.':
+ parsed.append(sys.intern(x))
+ else:
+ if rel and rel != '.':
+ parsed.append(sys.intern(rel))
+ if drv or root:
+ if not drv:
+ # If no drive is present, try to find one in the previous
+ # parts. This makes the result of parsing e.g.
+ # ("C:", "/", "a") reasonably intuitive.
+ for part in it:
+ drv = self.splitroot(part)[0]
+ if drv:
+ break
+ break
+ if drv or root:
+ parsed.append(drv + root)
+ parsed.reverse()
+ return drv, root, parsed
+
+ def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2):
+ """
+ Join the two paths represented by the respective
+ (drive, root, parts) tuples. Return a new (drive, root, parts) tuple.
+ """
+ if root2:
+ if not drv2 and drv:
+ return drv, root2, [drv + root2] + parts2[1:]
+ elif drv2:
+ if drv2 == drv or self.casefold(drv2) == self.casefold(drv):
+ # Same drive => second path is relative to the first
+ return drv, root, parts + parts2[1:]
+ else:
+ # Second path is non-anchored (common case)
+ return drv, root, parts + parts2
+ return drv2, root2, parts2
+
+
+class _WindowsFlavour(_Flavour):
+ # Reference for Windows paths can be found at
+ # http://msdn.microsoft.com/en-us/library/aa365247%28v=vs.85%29.aspx
+
+ sep = '\\'
+ altsep = '/'
+ has_drv = True
+ pathmod = ntpath
+
+ is_supported = (nt is not None)
+
+ drive_letters = (
+ set(chr(x) for x in range(ord('a'), ord('z') + 1)) |
+ set(chr(x) for x in range(ord('A'), ord('Z') + 1))
+ )
+ ext_namespace_prefix = '\\\\?\\'
+
+ reserved_names = (
+ {'CON', 'PRN', 'AUX', 'NUL'} |
+ {'COM%d' % i for i in range(1, 10)} |
+ {'LPT%d' % i for i in range(1, 10)}
+ )
+
+ # Interesting findings about extended paths:
+ # - '\\?\c:\a', '//?/c:\a' and '//?/c:/a' are all supported
+ # but '\\?\c:/a' is not
+ # - extended paths are always absolute; "relative" extended paths will
+ # fail.
+
+ def splitroot(self, part, sep=sep):
+ first = part[0:1]
+ second = part[1:2]
+ if (second == sep and first == sep):
+ # XXX extended paths should also disable the collapsing of "."
+ # components (according to MSDN docs).
+ prefix, part = self._split_extended_path(part)
+ first = part[0:1]
+ second = part[1:2]
+ else:
+ prefix = ''
+ third = part[2:3]
+ if (second == sep and first == sep and third != sep):
+ # is a UNC path:
+ # vvvvvvvvvvvvvvvvvvvvv root
+ # \\machine\mountpoint\directory\etc\...
+ # directory ^^^^^^^^^^^^^^
+ index = part.find(sep, 2)
+ if index != -1:
+ index2 = part.find(sep, index + 1)
+ # a UNC path can't have two slashes in a row
+ # (after the initial two)
+ if index2 != index + 1:
+ if index2 == -1:
+ index2 = len(part)
+ if prefix:
+ return prefix + part[1:index2], sep, part[index2+1:]
+ else:
+ return part[:index2], sep, part[index2+1:]
+ drv = root = ''
+ if second == ':' and first in self.drive_letters:
+ drv = part[:2]
+ part = part[2:]
+ first = third
+ if first == sep:
+ root = first
+ part = part.lstrip(sep)
+ return prefix + drv, root, part
+
+ def casefold(self, s):
+ return s.lower()
+
+ def casefold_parts(self, parts):
+ return [p.lower() for p in parts]
+
+ def resolve(self, path):
+ s = str(path)
+ if not s:
+ return os.getcwd()
+ if _getfinalpathname is not None:
+ return self._ext_to_normal(_getfinalpathname(s))
+ # Means fallback on absolute
+ return None
+
+ def _split_extended_path(self, s, ext_prefix=ext_namespace_prefix):
+ prefix = ''
+ if s.startswith(ext_prefix):
+ prefix = s[:4]
+ s = s[4:]
+ if s.startswith('UNC\\'):
+ prefix += s[:3]
+ s = '\\' + s[3:]
+ return prefix, s
+
+ def _ext_to_normal(self, s):
+ # Turn back an extended path into a normal DOS-like path
+ return self._split_extended_path(s)[1]
+
+ def is_reserved(self, parts):
+ # NOTE: the rules for reserved names seem somewhat complicated
+ # (e.g. r"..\NUL" is reserved but not r"foo\NUL").
+ # We err on the side of caution and return True for paths which are
+ # not considered reserved by Windows.
+ if not parts:
+ return False
+ if parts[0].startswith('\\\\'):
+ # UNC paths are never reserved
+ return False
+ return parts[-1].partition('.')[0].upper() in self.reserved_names
+
+ def make_uri(self, path):
+ # Under Windows, file URIs use the UTF-8 encoding.
+ drive = path.drive
+ if len(drive) == 2 and drive[1] == ':':
+ # It's a path on a local drive => 'file:///c:/a/b'
+ rest = path.as_posix()[2:].lstrip('/')
+ return 'file:///%s/%s' % (
+ drive, urlquote_from_bytes(rest.encode('utf-8')))
+ else:
+ # It's a path on a network drive => 'file://host/share/a/b'
+ return 'file:' + urlquote_from_bytes(path.as_posix().encode('utf-8'))
+
+
+class _PosixFlavour(_Flavour):
+ sep = '/'
+ altsep = ''
+ has_drv = False
+ pathmod = posixpath
+
+ is_supported = (os.name != 'nt')
+
+ def splitroot(self, part, sep=sep):
+ if part and part[0] == sep:
+ stripped_part = part.lstrip(sep)
+ # According to POSIX path resolution:
+ # http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap04.html#tag_04_11
+ # "A pathname that begins with two successive slashes may be
+ # interpreted in an implementation-defined manner, although more
+ # than two leading slashes shall be treated as a single slash".
+ if len(part) - len(stripped_part) == 2:
+ return '', sep * 2, stripped_part
+ else:
+ return '', sep, stripped_part
+ else:
+ return '', '', part
+
+ def casefold(self, s):
+ return s
+
+ def casefold_parts(self, parts):
+ return parts
+
+ def resolve(self, path):
+ sep = self.sep
+ accessor = path._accessor
+ seen = {}
+ def _resolve(path, rest):
+ if rest.startswith(sep):
+ path = ''
+
+ for name in rest.split(sep):
+ if not name or name == '.':
+ # current dir
+ continue
+ if name == '..':
+ # parent dir
+ path, _, _ = path.rpartition(sep)
+ continue
+ newpath = path + sep + name
+ if newpath in seen:
+ # Already seen this path
+ path = seen[newpath]
+ if path is not None:
+ # use cached value
+ continue
+ # The symlink is not resolved, so we must have a symlink loop.
+ raise RuntimeError("Symlink loop from %r" % newpath)
+ # Resolve the symbolic link
+ try:
+ target = accessor.readlink(newpath)
+ except OSError as e:
+ if e.errno != EINVAL:
+ raise
+ # Not a symlink
+ path = newpath
+ else:
+ seen[newpath] = None # not resolved symlink
+ path = _resolve(path, target)
+ seen[newpath] = path # resolved symlink
+
+ return path
+ # NOTE: according to POSIX, getcwd() cannot contain path components
+ # which are symlinks.
+ base = '' if path.is_absolute() else os.getcwd()
+ return _resolve(base, str(path)) or sep
+
+ def is_reserved(self, parts):
+ return False
+
+ def make_uri(self, path):
+ # We represent the path using the local filesystem encoding,
+ # for portability to other applications.
+ bpath = bytes(path)
+ return 'file://' + urlquote_from_bytes(bpath)
+
+
+_windows_flavour = _WindowsFlavour()
+_posix_flavour = _PosixFlavour()
+
+
+class _Accessor:
+ """An accessor implements a particular (system-specific or not) way of
+ accessing paths on the filesystem."""
+
+
+class _NormalAccessor(_Accessor):
+
+ def _wrap_strfunc(strfunc):
+ @functools.wraps(strfunc)
+ def wrapped(pathobj, *args):
+ return strfunc(str(pathobj), *args)
+ return staticmethod(wrapped)
+
+ def _wrap_binary_strfunc(strfunc):
+ @functools.wraps(strfunc)
+ def wrapped(pathobjA, pathobjB, *args):
+ return strfunc(str(pathobjA), str(pathobjB), *args)
+ return staticmethod(wrapped)
+
+ stat = _wrap_strfunc(os.stat)
+
+ lstat = _wrap_strfunc(os.lstat)
+
+ open = _wrap_strfunc(os.open)
+
+ listdir = _wrap_strfunc(os.listdir)
+
+ chmod = _wrap_strfunc(os.chmod)
+
+ if hasattr(os, "lchmod"):
+ lchmod = _wrap_strfunc(os.lchmod)
+ else:
+ def lchmod(self, pathobj, mode):
+ raise NotImplementedError("lchmod() not available on this system")
+
+ mkdir = _wrap_strfunc(os.mkdir)
+
+ unlink = _wrap_strfunc(os.unlink)
+
+ rmdir = _wrap_strfunc(os.rmdir)
+
+ rename = _wrap_binary_strfunc(os.rename)
+
+ replace = _wrap_binary_strfunc(os.replace)
+
+ if nt:
+ if supports_symlinks:
+ symlink = _wrap_binary_strfunc(os.symlink)
+ else:
+ def symlink(a, b, target_is_directory):
+ raise NotImplementedError("symlink() not available on this system")
+ else:
+ # Under POSIX, os.symlink() takes two args
+ @staticmethod
+ def symlink(a, b, target_is_directory):
+ return os.symlink(str(a), str(b))
+
+ utime = _wrap_strfunc(os.utime)
+
+ # Helper for resolve()
+ def readlink(self, path):
+ return os.readlink(path)
+
+
+_normal_accessor = _NormalAccessor()
+
+
+#
+# Globbing helpers
+#
+
+@contextmanager
+def _cached(func):
+ try:
+ func.__cached__
+ yield func
+ except AttributeError:
+ cache = {}
+ def wrapper(*args):
+ try:
+ return cache[args]
+ except KeyError:
+ value = cache[args] = func(*args)
+ return value
+ wrapper.__cached__ = True
+ try:
+ yield wrapper
+ finally:
+ cache.clear()
+
+def _make_selector(pattern_parts):
+ pat = pattern_parts[0]
+ child_parts = pattern_parts[1:]
+ if pat == '**':
+ cls = _RecursiveWildcardSelector
+ elif '**' in pat:
+ raise ValueError("Invalid pattern: '**' can only be an entire path component")
+ elif _is_wildcard_pattern(pat):
+ cls = _WildcardSelector
+ else:
+ cls = _PreciseSelector
+ return cls(pat, child_parts)
+
+if hasattr(functools, "lru_cache"):
+ _make_selector = functools.lru_cache()(_make_selector)
+
+
+class _Selector:
+ """A selector matches a specific glob pattern part against the children
+ of a given path."""
+
+ def __init__(self, child_parts):
+ self.child_parts = child_parts
+ if child_parts:
+ self.successor = _make_selector(child_parts)
+ else:
+ self.successor = _TerminatingSelector()
+
+ def select_from(self, parent_path):
+ """Iterate over all child paths of `parent_path` matched by this
+ selector. This can contain parent_path itself."""
+ path_cls = type(parent_path)
+ is_dir = path_cls.is_dir
+ exists = path_cls.exists
+ listdir = parent_path._accessor.listdir
+ return self._select_from(parent_path, is_dir, exists, listdir)
+
+
+class _TerminatingSelector:
+
+ def _select_from(self, parent_path, is_dir, exists, listdir):
+ yield parent_path
+
+
+class _PreciseSelector(_Selector):
+
+ def __init__(self, name, child_parts):
+ self.name = name
+ _Selector.__init__(self, child_parts)
+
+ def _select_from(self, parent_path, is_dir, exists, listdir):
+ if not is_dir(parent_path):
+ return
+ path = parent_path._make_child_relpath(self.name)
+ if exists(path):
+ for p in self.successor._select_from(path, is_dir, exists, listdir):
+ yield p
+
+
+class _WildcardSelector(_Selector):
+
+ def __init__(self, pat, child_parts):
+ self.pat = re.compile(fnmatch.translate(pat))
+ _Selector.__init__(self, child_parts)
+
+ def _select_from(self, parent_path, is_dir, exists, listdir):
+ if not is_dir(parent_path):
+ return
+ cf = parent_path._flavour.casefold
+ for name in listdir(parent_path):
+ casefolded = cf(name)
+ if self.pat.match(casefolded):
+ path = parent_path._make_child_relpath(name)
+ for p in self.successor._select_from(path, is_dir, exists, listdir):
+ yield p
+
+
+class _RecursiveWildcardSelector(_Selector):
+
+ def __init__(self, pat, child_parts):
+ _Selector.__init__(self, child_parts)
+
+ def _iterate_directories(self, parent_path, is_dir, listdir):
+ yield parent_path
+ for name in listdir(parent_path):
+ path = parent_path._make_child_relpath(name)
+ if is_dir(path):
+ for p in self._iterate_directories(path, is_dir, listdir):
+ yield p
+
+ def _select_from(self, parent_path, is_dir, exists, listdir):
+ if not is_dir(parent_path):
+ return
+ with _cached(listdir) as listdir:
+ yielded = set()
+ try:
+ successor_select = self.successor._select_from
+ for starting_point in self._iterate_directories(parent_path, is_dir, listdir):
+ for p in successor_select(starting_point, is_dir, exists, listdir):
+ if p not in yielded:
+ yield p
+ yielded.add(p)
+ finally:
+ yielded.clear()
+
+
+#
+# Public API
+#
+
+class _PathParents(Sequence):
+ """This object provides sequence-like access to the logical ancestors
+ of a path. Don't try to construct it yourself."""
+ __slots__ = ('_pathcls', '_drv', '_root', '_parts')
+
+ def __init__(self, path):
+ # We don't store the instance to avoid reference cycles
+ self._pathcls = type(path)
+ self._drv = path._drv
+ self._root = path._root
+ self._parts = path._parts
+
+ def __len__(self):
+ if self._drv or self._root:
+ return len(self._parts) - 1
+ else:
+ return len(self._parts)
+
+ def __getitem__(self, idx):
+ if idx < 0 or idx >= len(self):
+ raise IndexError(idx)
+ return self._pathcls._from_parsed_parts(self._drv, self._root,
+ self._parts[:-idx - 1])
+
+ def __repr__(self):
+ return "<{}.parents>".format(self._pathcls.__name__)
+
+
+class PurePath(object):
+ """PurePath represents a filesystem path and offers operations which
+ don't imply any actual filesystem I/O. Depending on your system,
+ instantiating a PurePath will return either a PurePosixPath or a
+ PureWindowsPath object. You can also instantiate either of these classes
+ directly, regardless of your system.
+ """
+ __slots__ = (
+ '_drv', '_root', '_parts',
+ '_str', '_hash', '_pparts', '_cached_cparts',
+ )
+
+ def __new__(cls, *args):
+ """Construct a PurePath from one or several strings and or existing
+ PurePath objects. The strings and path objects are combined so as
+ to yield a canonicalized path, which is incorporated into the
+ new PurePath object.
+ """
+ if cls is PurePath:
+ cls = PureWindowsPath if os.name == 'nt' else PurePosixPath
+ return cls._from_parts(args)
+
+ def __reduce__(self):
+ # Using the parts tuple helps share interned path parts
+ # when pickling related paths.
+ return (self.__class__, tuple(self._parts))
+
+ @classmethod
+ def _parse_args(cls, args):
+ # This is useful when you don't want to create an instance, just
+ # canonicalize some constructor arguments.
+ parts = []
+ for a in args:
+ if isinstance(a, PurePath):
+ parts += a._parts
+ elif isinstance(a, str):
+ # Assuming a str
+ parts.append(a)
+ else:
+ raise TypeError(
+ "argument should be a path or str object, not %r"
+ % type(a))
+ return cls._flavour.parse_parts(parts)
+
+ @classmethod
+ def _from_parts(cls, args, init=True):
+ # We need to call _parse_args on the instance, so as to get the
+ # right flavour.
+ self = object.__new__(cls)
+ drv, root, parts = self._parse_args(args)
+ self._drv = drv
+ self._root = root
+ self._parts = parts
+ if init:
+ self._init()
+ return self
+
+ @classmethod
+ def _from_parsed_parts(cls, drv, root, parts, init=True):
+ self = object.__new__(cls)
+ self._drv = drv
+ self._root = root
+ self._parts = parts
+ if init:
+ self._init()
+ return self
+
+ @classmethod
+ def _format_parsed_parts(cls, drv, root, parts):
+ if drv or root:
+ return drv + root + cls._flavour.join(parts[1:])
+ else:
+ return cls._flavour.join(parts)
+
+ def _init(self):
+ # Overriden in concrete Path
+ pass
+
+ def _make_child(self, args):
+ drv, root, parts = self._parse_args(args)
+ drv, root, parts = self._flavour.join_parsed_parts(
+ self._drv, self._root, self._parts, drv, root, parts)
+ return self._from_parsed_parts(drv, root, parts)
+
+ def __str__(self):
+ """Return the string representation of the path, suitable for
+ passing to system calls."""
+ try:
+ return self._str
+ except AttributeError:
+ self._str = self._format_parsed_parts(self._drv, self._root,
+ self._parts) or '.'
+ return self._str
+
+ def as_posix(self):
+ """Return the string representation of the path with forward (/)
+ slashes."""
+ f = self._flavour
+ return str(self).replace(f.sep, '/')
+
+ def __bytes__(self):
+ """Return the bytes representation of the path. This is only
+ recommended to use under Unix."""
+ return os.fsencode(str(self))
+
+ def __repr__(self):
+ return "{}({!r})".format(self.__class__.__name__, self.as_posix())
+
+ def as_uri(self):
+ """Return the path as a 'file' URI."""
+ if not self.is_absolute():
+ raise ValueError("relative path can't be expressed as a file URI")
+ return self._flavour.make_uri(self)
+
+ @property
+ def _cparts(self):
+ # Cached casefolded parts, for hashing and comparison
+ try:
+ return self._cached_cparts
+ except AttributeError:
+ self._cached_cparts = self._flavour.casefold_parts(self._parts)
+ return self._cached_cparts
+
+ def __eq__(self, other):
+ if not isinstance(other, PurePath):
+ return NotImplemented
+ return self._cparts == other._cparts and self._flavour is other._flavour
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ try:
+ return self._hash
+ except AttributeError:
+ self._hash = hash(tuple(self._cparts))
+ return self._hash
+
+ def __lt__(self, other):
+ if not isinstance(other, PurePath) or self._flavour is not other._flavour:
+ return NotImplemented
+ return self._cparts < other._cparts
+
+ def __le__(self, other):
+ if not isinstance(other, PurePath) or self._flavour is not other._flavour:
+ return NotImplemented
+ return self._cparts <= other._cparts
+
+ def __gt__(self, other):
+ if not isinstance(other, PurePath) or self._flavour is not other._flavour:
+ return NotImplemented
+ return self._cparts > other._cparts
+
+ def __ge__(self, other):
+ if not isinstance(other, PurePath) or self._flavour is not other._flavour:
+ return NotImplemented
+ return self._cparts >= other._cparts
+
+ drive = property(attrgetter('_drv'),
+ doc="""The drive prefix (letter or UNC path), if any.""")
+
+ root = property(attrgetter('_root'),
+ doc="""The root of the path, if any.""")
+
+ @property
+ def anchor(self):
+ """The concatenation of the drive and root, or ''."""
+ anchor = self._drv + self._root
+ return anchor
+
+ @property
+ def name(self):
+ """The final path component, if any."""
+ parts = self._parts
+ if len(parts) == (1 if (self._drv or self._root) else 0):
+ return ''
+ return parts[-1]
+
+ @property
+ def suffix(self):
+ """The final component's last suffix, if any."""
+ name = self.name
+ i = name.rfind('.')
+ if 0 < i < len(name) - 1:
+ return name[i:]
+ else:
+ return ''
+
+ @property
+ def suffixes(self):
+ """A list of the final component's suffixes, if any."""
+ name = self.name
+ if name.endswith('.'):
+ return []
+ name = name.lstrip('.')
+ return ['.' + suffix for suffix in name.split('.')[1:]]
+
+ @property
+ def stem(self):
+ """The final path component, minus its last suffix."""
+ name = self.name
+ i = name.rfind('.')
+ if 0 < i < len(name) - 1:
+ return name[:i]
+ else:
+ return name
+
+ def with_name(self, name):
+ """Return a new path with the file name changed."""
+ if not self.name:
+ raise ValueError("%r has an empty name" % (self,))
+ return self._from_parsed_parts(self._drv, self._root,
+ self._parts[:-1] + [name])
+
+ def with_suffix(self, suffix):
+ """Return a new path with the file suffix changed (or added, if none)."""
+ # XXX if suffix is None, should the current suffix be removed?
+ drv, root, parts = self._flavour.parse_parts((suffix,))
+ if drv or root or len(parts) != 1:
+ raise ValueError("Invalid suffix %r" % (suffix))
+ suffix = parts[0]
+ if not suffix.startswith('.'):
+ raise ValueError("Invalid suffix %r" % (suffix))
+ name = self.name
+ if not name:
+ raise ValueError("%r has an empty name" % (self,))
+ old_suffix = self.suffix
+ if not old_suffix:
+ name = name + suffix
+ else:
+ name = name[:-len(old_suffix)] + suffix
+ return self._from_parsed_parts(self._drv, self._root,
+ self._parts[:-1] + [name])
+
+ def relative_to(self, *other):
+ """Return the relative path to another path identified by the passed
+ arguments. If the operation is not possible (because this is not
+ a subpath of the other path), raise ValueError.
+ """
+ # For the purpose of this method, drive and root are considered
+ # separate parts, i.e.:
+ # Path('c:/').relative_to('c:') gives Path('/')
+ # Path('c:/').relative_to('/') raise ValueError
+ if not other:
+ raise TypeError("need at least one argument")
+ parts = self._parts
+ drv = self._drv
+ root = self._root
+ if root:
+ abs_parts = [drv, root] + parts[1:]
+ else:
+ abs_parts = parts
+ to_drv, to_root, to_parts = self._parse_args(other)
+ if to_root:
+ to_abs_parts = [to_drv, to_root] + to_parts[1:]
+ else:
+ to_abs_parts = to_parts
+ n = len(to_abs_parts)
+ cf = self._flavour.casefold_parts
+ if (root or drv) if n == 0 else cf(abs_parts[:n]) != cf(to_abs_parts):
+ formatted = self._format_parsed_parts(to_drv, to_root, to_parts)
+ raise ValueError("{!r} does not start with {!r}"
+ .format(str(self), str(formatted)))
+ return self._from_parsed_parts('', root if n == 1 else '',
+ abs_parts[n:])
+
+ @property
+ def parts(self):
+ """An object providing sequence-like access to the
+ components in the filesystem path."""
+ # We cache the tuple to avoid building a new one each time .parts
+ # is accessed. XXX is this necessary?
+ try:
+ return self._pparts
+ except AttributeError:
+ self._pparts = tuple(self._parts)
+ return self._pparts
+
+ def joinpath(self, *args):
+ """Combine this path with one or several arguments, and return a
+ new path representing either a subpath (if all arguments are relative
+ paths) or a totally different path (if one of the arguments is
+ anchored).
+ """
+ return self._make_child(args)
+
+ def __truediv__(self, key):
+ return self._make_child((key,))
+
+ def __rtruediv__(self, key):
+ return self._from_parts([key] + self._parts)
+
+ @property
+ def parent(self):
+ """The logical parent of the path."""
+ drv = self._drv
+ root = self._root
+ parts = self._parts
+ if len(parts) == 1 and (drv or root):
+ return self
+ return self._from_parsed_parts(drv, root, parts[:-1])
+
+ @property
+ def parents(self):
+ """A sequence of this path's logical parents."""
+ return _PathParents(self)
+
+ def is_absolute(self):
+ """True if the path is absolute (has both a root and, if applicable,
+ a drive)."""
+ if not self._root:
+ return False
+ return not self._flavour.has_drv or bool(self._drv)
+
+ def is_reserved(self):
+ """Return True if the path contains one of the special names reserved
+ by the system, if any."""
+ return self._flavour.is_reserved(self._parts)
+
+ def match(self, path_pattern):
+ """
+ Return True if this path matches the given pattern.
+ """
+ cf = self._flavour.casefold
+ path_pattern = cf(path_pattern)
+ drv, root, pat_parts = self._flavour.parse_parts((path_pattern,))
+ if not pat_parts:
+ raise ValueError("empty pattern")
+ if drv and drv != cf(self._drv):
+ return False
+ if root and root != cf(self._root):
+ return False
+ parts = self._cparts
+ if drv or root:
+ if len(pat_parts) != len(parts):
+ return False
+ pat_parts = pat_parts[1:]
+ elif len(pat_parts) > len(parts):
+ return False
+ for part, pat in zip(reversed(parts), reversed(pat_parts)):
+ if not fnmatch.fnmatchcase(part, pat):
+ return False
+ return True
+
+
+class PurePosixPath(PurePath):
+ _flavour = _posix_flavour
+ __slots__ = ()
+
+
+class PureWindowsPath(PurePath):
+ _flavour = _windows_flavour
+ __slots__ = ()
+
+
+# Filesystem-accessing classes
+
+
+class Path(PurePath):
+ __slots__ = (
+ '_accessor',
+ '_closed',
+ )
+
+ def __new__(cls, *args, **kwargs):
+ if cls is Path:
+ cls = WindowsPath if os.name == 'nt' else PosixPath
+ self = cls._from_parts(args, init=False)
+ if not self._flavour.is_supported:
+ raise NotImplementedError("cannot instantiate %r on your system"
+ % (cls.__name__,))
+ self._init()
+ return self
+
+ def _init(self,
+ # Private non-constructor arguments
+ template=None,
+ ):
+ self._closed = False
+ if template is not None:
+ self._accessor = template._accessor
+ else:
+ self._accessor = _normal_accessor
+
+ def _make_child_relpath(self, part):
+ # This is an optimization used for dir walking. `part` must be
+ # a single part relative to this path.
+ parts = self._parts + [part]
+ return self._from_parsed_parts(self._drv, self._root, parts)
+
+ def __enter__(self):
+ if self._closed:
+ self._raise_closed()
+ return self
+
+ def __exit__(self, t, v, tb):
+ self._closed = True
+
+ def _raise_closed(self):
+ raise ValueError("I/O operation on closed path")
+
+ def _opener(self, name, flags, mode=0o666):
+ # A stub for the opener argument to built-in open()
+ return self._accessor.open(self, flags, mode)
+
+ def _raw_open(self, flags, mode=0o777):
+ """
+ Open the file pointed by this path and return a file descriptor,
+ as os.open() does.
+ """
+ if self._closed:
+ self._raise_closed()
+ return self._accessor.open(self, flags, mode)
+
+ # Public API
+
+ @classmethod
+ def cwd(cls):
+ """Return a new path pointing to the current working directory
+ (as returned by os.getcwd()).
+ """
+ return cls(os.getcwd())
+
+ def iterdir(self):
+ """Iterate over the files in this directory. Does not yield any
+ result for the special paths '.' and '..'.
+ """
+ if self._closed:
+ self._raise_closed()
+ for name in self._accessor.listdir(self):
+ if name in {'.', '..'}:
+ # Yielding a path object for these makes little sense
+ continue
+ yield self._make_child_relpath(name)
+ if self._closed:
+ self._raise_closed()
+
+ def glob(self, pattern):
+ """Iterate over this subtree and yield all existing files (of any
+ kind, including directories) matching the given pattern.
+ """
+ pattern = self._flavour.casefold(pattern)
+ drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
+ if drv or root:
+ raise NotImplementedError("Non-relative patterns are unsupported")
+ selector = _make_selector(tuple(pattern_parts))
+ for p in selector.select_from(self):
+ yield p
+
+ def rglob(self, pattern):
+ """Recursively yield all existing files (of any kind, including
+ directories) matching the given pattern, anywhere in this subtree.
+ """
+ pattern = self._flavour.casefold(pattern)
+ drv, root, pattern_parts = self._flavour.parse_parts((pattern,))
+ if drv or root:
+ raise NotImplementedError("Non-relative patterns are unsupported")
+ selector = _make_selector(("**",) + tuple(pattern_parts))
+ for p in selector.select_from(self):
+ yield p
+
+ def absolute(self):
+ """Return an absolute version of this path. This function works
+ even if the path doesn't point to anything.
+
+ No normalization is done, i.e. all '.' and '..' will be kept along.
+ Use resolve() to get the canonical path to a file.
+ """
+ # XXX untested yet!
+ if self._closed:
+ self._raise_closed()
+ if self.is_absolute():
+ return self
+ # FIXME this must defer to the specific flavour (and, under Windows,
+ # use nt._getfullpathname())
+ obj = self._from_parts([os.getcwd()] + self._parts, init=False)
+ obj._init(template=self)
+ return obj
+
+ def resolve(self):
+ """
+ Make the path absolute, resolving all symlinks on the way and also
+ normalizing it (for example turning slashes into backslashes under
+ Windows).
+ """
+ if self._closed:
+ self._raise_closed()
+ s = self._flavour.resolve(self)
+ if s is None:
+ # No symlink resolution => for consistency, raise an error if
+ # the path doesn't exist or is forbidden
+ self.stat()
+ s = str(self.absolute())
+ # Now we have no symlinks in the path, it's safe to normalize it.
+ normed = self._flavour.pathmod.normpath(s)
+ obj = self._from_parts((normed,), init=False)
+ obj._init(template=self)
+ return obj
+
+ def stat(self):
+ """
+ Return the result of the stat() system call on this path, like
+ os.stat() does.
+ """
+ return self._accessor.stat(self)
+
+ def owner(self):
+ """
+ Return the login name of the file owner.
+ """
+ import pwd
+ return pwd.getpwuid(self.stat().st_uid).pw_name
+
+ def group(self):
+ """
+ Return the group name of the file gid.
+ """
+ import grp
+ return grp.getgrgid(self.stat().st_gid).gr_name
+
+ def open(self, mode='r', buffering=-1, encoding=None,
+ errors=None, newline=None):
+ """
+ Open the file pointed by this path and return a file object, as
+ the built-in open() function does.
+ """
+ if self._closed:
+ self._raise_closed()
+ return io.open(str(self), mode, buffering, encoding, errors, newline,
+ opener=self._opener)
+
+ def touch(self, mode=0o666, exist_ok=True):
+ """
+ Create this file with the given access mode, if it doesn't exist.
+ """
+ if self._closed:
+ self._raise_closed()
+ if exist_ok:
+ # First try to bump modification time
+ # Implementation note: GNU touch uses the UTIME_NOW option of
+ # the utimensat() / futimens() functions.
+ try:
+ self._accessor.utime(self, None)
+ except OSError:
+ # Avoid exception chaining
+ pass
+ else:
+ return
+ flags = os.O_CREAT | os.O_WRONLY
+ if not exist_ok:
+ flags |= os.O_EXCL
+ fd = self._raw_open(flags, mode)
+ os.close(fd)
+
+ def mkdir(self, mode=0o777, parents=False):
+ if self._closed:
+ self._raise_closed()
+ if not parents:
+ self._accessor.mkdir(self, mode)
+ else:
+ try:
+ self._accessor.mkdir(self, mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ self.parent.mkdir(parents=True)
+ self._accessor.mkdir(self, mode)
+
+ def chmod(self, mode):
+ """
+ Change the permissions of the path, like os.chmod().
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.chmod(self, mode)
+
+ def lchmod(self, mode):
+ """
+ Like chmod(), except if the path points to a symlink, the symlink's
+ permissions are changed, rather than its target's.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.lchmod(self, mode)
+
+ def unlink(self):
+ """
+ Remove this file or link.
+ If the path is a directory, use rmdir() instead.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.unlink(self)
+
+ def rmdir(self):
+ """
+ Remove this directory. The directory must be empty.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.rmdir(self)
+
+ def lstat(self):
+ """
+ Like stat(), except if the path points to a symlink, the symlink's
+ status information is returned, rather than its target's.
+ """
+ if self._closed:
+ self._raise_closed()
+ return self._accessor.lstat(self)
+
+ def rename(self, target):
+ """
+ Rename this path to the given path.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.rename(self, target)
+
+ def replace(self, target):
+ """
+ Rename this path to the given path, clobbering the existing
+ destination if it exists.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.replace(self, target)
+
+ def symlink_to(self, target, target_is_directory=False):
+ """
+ Make this path a symlink pointing to the given path.
+ Note the order of arguments (self, target) is the reverse of os.symlink's.
+ """
+ if self._closed:
+ self._raise_closed()
+ self._accessor.symlink(target, self, target_is_directory)
+
+ # Convenience functions for querying the stat results
+
+ def exists(self):
+ """
+ Whether this path exists.
+ """
+ try:
+ self.stat()
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ return False
+ return True
+
+ def is_dir(self):
+ """
+ Whether this path is a directory.
+ """
+ try:
+ return S_ISDIR(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+ def is_file(self):
+ """
+ Whether this path is a regular file (also True for symlinks pointing
+ to regular files).
+ """
+ try:
+ return S_ISREG(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+ def is_symlink(self):
+ """
+ Whether this path is a symbolic link.
+ """
+ try:
+ return S_ISLNK(self.lstat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist
+ return False
+
+ def is_block_device(self):
+ """
+ Whether this path is a block device.
+ """
+ try:
+ return S_ISBLK(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+ def is_char_device(self):
+ """
+ Whether this path is a character device.
+ """
+ try:
+ return S_ISCHR(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+ def is_fifo(self):
+ """
+ Whether this path is a FIFO.
+ """
+ try:
+ return S_ISFIFO(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+ def is_socket(self):
+ """
+ Whether this path is a socket.
+ """
+ try:
+ return S_ISSOCK(self.stat().st_mode)
+ except OSError as e:
+ if e.errno != ENOENT:
+ raise
+ # Path doesn't exist or is a broken symlink
+ # (see https://bitbucket.org/pitrou/pathlib/issue/12/)
+ return False
+
+
+class PosixPath(Path, PurePosixPath):
+ __slots__ = ()
+
+class WindowsPath(Path, PureWindowsPath):
+ __slots__ = ()
diff --git a/Lib/pdb.py b/Lib/pdb.py
index 80cba9d..dd7ceb8 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -92,21 +92,14 @@ def find_function(funcname, filename):
cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
try:
fp = open(filename)
- except IOError:
+ except OSError:
return None
# consumer of this info expects the first line to be 1
- lineno = 1
- answer = None
- while True:
- line = fp.readline()
- if line == '':
- break
- if cre.match(line):
- answer = funcname, filename, lineno
- break
- lineno += 1
- fp.close()
- return answer
+ with fp:
+ for lineno, line in enumerate(fp, start=1):
+ if cre.match(line):
+ return funcname, filename, lineno
+ return None
def getsourcelines(obj):
lines, lineno = inspect.findsource(obj)
@@ -170,12 +163,12 @@ class Pdb(bdb.Bdb, cmd.Cmd):
try:
with open(os.path.join(envHome, ".pdbrc")) as rcFile:
self.rcLines.extend(rcFile)
- except IOError:
+ except OSError:
pass
try:
with open(".pdbrc") as rcFile:
self.rcLines.extend(rcFile)
- except IOError:
+ except OSError:
pass
self.commands = {} # associates a command list to breakpoint numbers
@@ -304,8 +297,16 @@ class Pdb(bdb.Bdb, cmd.Cmd):
return
exc_type, exc_value, exc_traceback = exc_info
frame.f_locals['__exception__'] = exc_type, exc_value
- self.message(traceback.format_exception_only(exc_type,
- exc_value)[-1].strip())
+
+ # An 'Internal StopIteration' exception is an exception debug event
+ # issued by the interpreter when handling a subgenerator run with
+ # 'yield from' or a generator controled by a for loop. No exception has
+ # actually occured in this case. The debugger uses this debug event to
+ # stop when the debuggee is returning from such generators.
+ prefix = 'Internal ' if (not exc_traceback
+ and exc_type is StopIteration) else ''
+ self.message('%s%s' % (prefix,
+ traceback.format_exception_only(exc_type, exc_value)[-1].strip()))
self.interaction(frame, exc_traceback)
# General interaction function
@@ -1163,15 +1164,13 @@ class Pdb(bdb.Bdb, cmd.Cmd):
return _rstr('** raised %s **' % err)
def do_p(self, arg):
- """p(rint) expression
+ """p expression
Print the value of the expression.
"""
try:
self.message(repr(self._getval(arg)))
except:
pass
- # make "print" an alias of "p" since print isn't a Python statement anymore
- do_print = do_p
def do_pp(self, arg):
"""pp expression
@@ -1245,7 +1244,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
breaklist = self.get_file_breaks(filename)
try:
lines, lineno = getsourcelines(self.curframe)
- except IOError as err:
+ except OSError as err:
self.error(err)
return
self._print_lines(lines, lineno, breaklist, self.curframe)
@@ -1261,7 +1260,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
return
try:
lines, lineno = getsourcelines(obj)
- except (IOError, TypeError) as err:
+ except (OSError, TypeError) as err:
self.error(err)
return
self._print_lines(lines, lineno)
@@ -1392,7 +1391,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
placed in the .pdbrc file):
# Print instance variables (usage "pi classInst")
- alias pi for k in %1.__dict__.keys(): print "%1.",k,"=",%1.__dict__[k]
+ alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])
# Print instance variables in self
alias ps pi self
"""
@@ -1550,7 +1549,7 @@ if __doc__ is not None:
'help', 'where', 'down', 'up', 'break', 'tbreak', 'clear', 'disable',
'enable', 'ignore', 'condition', 'commands', 'step', 'next', 'until',
'jump', 'return', 'retval', 'run', 'continue', 'list', 'longlist',
- 'args', 'print', 'pp', 'whatis', 'source', 'display', 'undisplay',
+ 'args', 'p', 'pp', 'whatis', 'source', 'display', 'undisplay',
'interact', 'alias', 'unalias', 'debug', 'quit',
]
diff --git a/Lib/pickle.py b/Lib/pickle.py
index 386ffba..3fc2596 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -23,12 +23,13 @@ Misc variables:
"""
-from types import FunctionType, BuiltinFunctionType
+from types import FunctionType
from copyreg import dispatch_table
from copyreg import _extension_registry, _inverted_registry, _extension_cache
-import marshal
+from itertools import islice
import sys
-import struct
+from sys import maxsize
+from struct import pack, unpack
import re
import io
import codecs
@@ -41,28 +42,24 @@ __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
bytes_types = (bytes, bytearray)
# These are purely informational; no code uses these.
-format_version = "3.0" # File format version we write
+format_version = "4.0" # File format version we write
compatible_formats = ["1.0", # Original protocol 0
"1.1", # Protocol 0 with INST added
"1.2", # Original protocol 1
"1.3", # Protocol 1 with BINFLOAT added
"2.0", # Protocol 2
"3.0", # Protocol 3
+ "4.0", # Protocol 4
] # Old format versions we can read
# This is the highest protocol number we know how to read.
-HIGHEST_PROTOCOL = 3
+HIGHEST_PROTOCOL = 4
# The protocol we write by default. May be less than HIGHEST_PROTOCOL.
# We intentionally write a protocol that Python 2.x cannot read;
# there are too many issues with that.
DEFAULT_PROTOCOL = 3
-# Why use struct.pack() for pickling but marshal.loads() for
-# unpickling? struct.pack() is 40% faster than marshal.dumps(), but
-# marshal.loads() is twice as fast as struct.unpack()!
-mloads = marshal.loads
-
class PickleError(Exception):
"""A common base class for the other pickling exceptions."""
pass
@@ -168,7 +165,181 @@ _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
BINBYTES = b'B' # push bytes; counted binary string argument
SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
-__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)])
+# Protocol 4
+SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
+BINUNICODE8 = b'\x8d' # push very long string
+BINBYTES8 = b'\x8e' # push very long bytes string
+EMPTY_SET = b'\x8f' # push empty set on the stack
+ADDITEMS = b'\x90' # modify set by adding topmost stack items
+FROZENSET = b'\x91' # build frozenset from topmost stack items
+NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
+STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
+MEMOIZE = b'\x94' # store top of the stack in memo
+FRAME = b'\x95' # indicate the beginning of a new frame
+
+__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
+
+
+class _Framer:
+
+ _FRAME_SIZE_TARGET = 64 * 1024
+
+ def __init__(self, file_write):
+ self.file_write = file_write
+ self.current_frame = None
+
+ def start_framing(self):
+ self.current_frame = io.BytesIO()
+
+ def end_framing(self):
+ if self.current_frame and self.current_frame.tell() > 0:
+ self.commit_frame(force=True)
+ self.current_frame = None
+
+ def commit_frame(self, force=False):
+ if self.current_frame:
+ f = self.current_frame
+ if f.tell() >= self._FRAME_SIZE_TARGET or force:
+ with f.getbuffer() as data:
+ n = len(data)
+ write = self.file_write
+ write(FRAME)
+ write(pack("<Q", n))
+ write(data)
+ f.seek(0)
+ f.truncate()
+
+ def write(self, data):
+ if self.current_frame:
+ return self.current_frame.write(data)
+ else:
+ return self.file_write(data)
+
+
+class _Unframer:
+
+ def __init__(self, file_read, file_readline, file_tell=None):
+ self.file_read = file_read
+ self.file_readline = file_readline
+ self.current_frame = None
+
+ def read(self, n):
+ if self.current_frame:
+ data = self.current_frame.read(n)
+ if not data and n != 0:
+ self.current_frame = None
+ return self.file_read(n)
+ if len(data) < n:
+ raise UnpicklingError(
+ "pickle exhausted before end of frame")
+ return data
+ else:
+ return self.file_read(n)
+
+ def readline(self):
+ if self.current_frame:
+ data = self.current_frame.readline()
+ if not data:
+ self.current_frame = None
+ return self.file_readline()
+ if data[-1] != b'\n':
+ raise UnpicklingError(
+ "pickle exhausted before end of frame")
+ return data
+ else:
+ return self.file_readline()
+
+ def load_frame(self, frame_size):
+ if self.current_frame and self.current_frame.read() != b'':
+ raise UnpicklingError(
+ "beginning of a new frame before end of current frame")
+ self.current_frame = io.BytesIO(self.file_read(frame_size))
+
+
+# Tools used for pickling.
+
+def _getattribute(obj, name, allow_qualname=False):
+ dotted_path = name.split(".")
+ if not allow_qualname and len(dotted_path) > 1:
+ raise AttributeError("Can't get qualified attribute {!r} on {!r}; " +
+ "use protocols >= 4 to enable support"
+ .format(name, obj))
+ for subpath in dotted_path:
+ if subpath == '<locals>':
+ raise AttributeError("Can't get local attribute {!r} on {!r}"
+ .format(name, obj))
+ try:
+ obj = getattr(obj, subpath)
+ except AttributeError:
+ raise AttributeError("Can't get attribute {!r} on {!r}"
+ .format(name, obj))
+ return obj
+
+def whichmodule(obj, name, allow_qualname=False):
+ """Find the module an object belong to."""
+ module_name = getattr(obj, '__module__', None)
+ if module_name is not None:
+ return module_name
+ for module_name, module in sys.modules.items():
+ if module_name == '__main__' or module is None:
+ continue
+ try:
+ if _getattribute(module, name, allow_qualname) is obj:
+ return module_name
+ except AttributeError:
+ pass
+ return '__main__'
+
+def encode_long(x):
+ r"""Encode a long to a two's complement little-endian binary string.
+ Note that 0 is a special case, returning an empty string, to save a
+ byte in the LONG1 pickling context.
+
+ >>> encode_long(0)
+ b''
+ >>> encode_long(255)
+ b'\xff\x00'
+ >>> encode_long(32767)
+ b'\xff\x7f'
+ >>> encode_long(-256)
+ b'\x00\xff'
+ >>> encode_long(-32768)
+ b'\x00\x80'
+ >>> encode_long(-128)
+ b'\x80'
+ >>> encode_long(127)
+ b'\x7f'
+ >>>
+ """
+ if x == 0:
+ return b''
+ nbytes = (x.bit_length() >> 3) + 1
+ result = x.to_bytes(nbytes, byteorder='little', signed=True)
+ if x < 0 and nbytes > 1:
+ if result[-1] == 0xff and (result[-2] & 0x80) != 0:
+ result = result[:-1]
+ return result
+
+def decode_long(data):
+ r"""Decode a long from a two's complement little-endian binary string.
+
+ >>> decode_long(b'')
+ 0
+ >>> decode_long(b"\xff\x00")
+ 255
+ >>> decode_long(b"\xff\x7f")
+ 32767
+ >>> decode_long(b"\x00\xff")
+ -256
+ >>> decode_long(b"\x00\x80")
+ -32768
+ >>> decode_long(b"\x80")
+ -128
+ >>> decode_long(b"\x7f")
+ 127
+ """
+ return int.from_bytes(data, byteorder='little', signed=True)
+
# Pickling machinery
@@ -177,24 +348,25 @@ class _Pickler:
def __init__(self, file, protocol=None, *, fix_imports=True):
"""This takes a binary file for writing a pickle data stream.
- The optional protocol argument tells the pickler to use the
- given protocol; supported protocols are 0, 1, 2, 3. The default
- protocol is 3; a backward-incompatible protocol designed for
- Python 3.0.
+ The optional *protocol* argument tells the pickler to use the
+ given protocol; supported protocols are 0, 1, 2, 3 and 4. The
+ default protocol is 3; a backward-incompatible protocol designed
+ for Python 3.
Specifying a negative protocol version selects the highest
protocol version supported. The higher the protocol used, the
more recent the version of Python needed to read the pickle
produced.
- The file argument must have a write() method that accepts a single
- bytes argument. It can thus be a file object opened for binary
- writing, a io.BytesIO instance, or any other custom object that
- meets this interface.
+ The *file* argument must have a write() method that accepts a
+ single bytes argument. It can thus be a file object opened for
+ binary writing, a io.BytesIO instance, or any other custom
+ object that meets this interface.
- If fix_imports is True and protocol is less than 3, pickle will try to
- map the new Python 3.x names to the old module names used in Python
- 2.x, so that the pickle data stream is readable with Python 2.x.
+ If *fix_imports* is True and *protocol* is less than 3, pickle
+ will try to map the new Python 3 names to the old module names
+ used in Python 2, so that the pickle data stream is readable
+ with Python 2.
"""
if protocol is None:
protocol = DEFAULT_PROTOCOL
@@ -203,9 +375,11 @@ class _Pickler:
elif not 0 <= protocol <= HIGHEST_PROTOCOL:
raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
try:
- self.write = file.write
+ self._file_write = file.write
except AttributeError:
raise TypeError("file must have a 'write' attribute")
+ self.framer = _Framer(self._file_write)
+ self.write = self.framer.write
self.memo = {}
self.proto = int(protocol)
self.bin = protocol >= 1
@@ -216,10 +390,9 @@ class _Pickler:
"""Clears the pickler's "memo".
The memo is the data structure that remembers which objects the
- pickler has already seen, so that shared or recursive objects are
- pickled by reference and not by value. This method is useful when
- re-using picklers.
-
+ pickler has already seen, so that shared or recursive objects
+ are pickled by reference and not by value. This method is
+ useful when re-using picklers.
"""
self.memo.clear()
@@ -227,13 +400,16 @@ class _Pickler:
"""Write a pickled representation of obj to the open file."""
# Check whether Pickler was initialized correctly. This is
# only needed to mimic the behavior of _pickle.Pickler.dump().
- if not hasattr(self, "write"):
+ if not hasattr(self, "_file_write"):
raise PicklingError("Pickler.__init__() was not called by "
"%s.__init__()" % (self.__class__.__name__,))
if self.proto >= 2:
- self.write(PROTO + bytes([self.proto]))
+ self.write(PROTO + pack("<B", self.proto))
+ if self.proto >= 4:
+ self.framer.start_framing()
self.save(obj)
self.write(STOP)
+ self.framer.end_framing()
def memoize(self, obj):
"""Store an object in the memo."""
@@ -253,31 +429,35 @@ class _Pickler:
if self.fast:
return
assert id(obj) not in self.memo
- memo_len = len(self.memo)
- self.write(self.put(memo_len))
- self.memo[id(obj)] = memo_len, obj
+ idx = len(self.memo)
+ self.write(self.put(idx))
+ self.memo[id(obj)] = idx, obj
# Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
- def put(self, i, pack=struct.pack):
- if self.bin:
- if i < 256:
- return BINPUT + bytes([i])
+ def put(self, idx):
+ if self.proto >= 4:
+ return MEMOIZE
+ elif self.bin:
+ if idx < 256:
+ return BINPUT + pack("<B", idx)
else:
- return LONG_BINPUT + pack("<I", i)
-
- return PUT + repr(i).encode("ascii") + b'\n'
+ return LONG_BINPUT + pack("<I", idx)
+ else:
+ return PUT + repr(idx).encode("ascii") + b'\n'
# Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
- def get(self, i, pack=struct.pack):
+ def get(self, i):
if self.bin:
if i < 256:
- return BINGET + bytes([i])
+ return BINGET + pack("<B", i)
else:
return LONG_BINGET + pack("<I", i)
return GET + repr(i).encode("ascii") + b'\n'
def save(self, obj, save_persistent_id=True):
+ self.framer.commit_frame()
+
# Check for persistent id (defined by a subclass)
pid = self.persistent_id(obj)
if pid is not None and save_persistent_id:
@@ -286,20 +466,20 @@ class _Pickler:
# Check the memo
x = self.memo.get(id(obj))
- if x:
+ if x is not None:
self.write(self.get(x[0]))
return
# Check the type dispatch table
t = type(obj)
f = self.dispatch.get(t)
- if f:
+ if f is not None:
f(self, obj) # Call unbound method with explicit self
return
# Check private dispatch table if any, or else copyreg.dispatch_table
reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
- if reduce:
+ if reduce is not None:
rv = reduce(obj)
else:
# Check for a class with a custom metaclass; treat as regular class
@@ -313,11 +493,11 @@ class _Pickler:
# Check for a __reduce_ex__ method, fall back to __reduce__
reduce = getattr(obj, "__reduce_ex__", None)
- if reduce:
+ if reduce is not None:
rv = reduce(self.proto)
else:
reduce = getattr(obj, "__reduce__", None)
- if reduce:
+ if reduce is not None:
rv = reduce()
else:
raise PicklingError("Can't pickle %r object: %r" %
@@ -353,24 +533,33 @@ class _Pickler:
else:
self.write(PERSID + str(pid).encode("ascii") + b'\n')
- def save_reduce(self, func, args, state=None,
- listitems=None, dictitems=None, obj=None):
+ def save_reduce(self, func, args, state=None, listitems=None,
+ dictitems=None, obj=None):
# This API is called by some subclasses
- # Assert that args is a tuple
if not isinstance(args, tuple):
- raise PicklingError("args from save_reduce() should be a tuple")
-
- # Assert that func is callable
+ raise PicklingError("args from save_reduce() must be a tuple")
if not callable(func):
- raise PicklingError("func from save_reduce() should be callable")
+ raise PicklingError("func from save_reduce() must be callable")
save = self.save
write = self.write
- # Protocol 2 special case: if func's name is __newobj__, use NEWOBJ
- if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__":
- # A __reduce__ implementation can direct protocol 2 to
+ func_name = getattr(func, "__name__", "")
+ if self.proto >= 4 and func_name == "__newobj_ex__":
+ cls, args, kwargs = args
+ if not hasattr(cls, "__new__"):
+ raise PicklingError("args[0] from {} args has no __new__"
+ .format(func_name))
+ if obj is not None and cls is not obj.__class__:
+ raise PicklingError("args[0] from {} args has the wrong class"
+ .format(func_name))
+ save(cls)
+ save(args)
+ save(kwargs)
+ write(NEWOBJ_EX)
+ elif self.proto >= 2 and func_name == "__newobj__":
+ # A __reduce__ implementation can direct protocol 2 or newer to
# use the more efficient NEWOBJ opcode, while still
# allowing protocol 0 and 1 to work normally. For this to
# work, the function returned by __reduce__ should be
@@ -413,7 +602,13 @@ class _Pickler:
write(REDUCE)
if obj is not None:
- self.memoize(obj)
+ # If the object is already in the memo, this means it is
+ # recursive. In this case, throw away everything we put on the
+ # stack, and fetch the object back from the memo.
+ if id(obj) in self.memo:
+ write(POP + self.get(self.memo[id(obj)][0]))
+ else:
+ self.memoize(obj)
# More new special cases (that work with older protocols as
# well): when __reduce__ returns a tuple with 4 or 5 items,
@@ -438,22 +633,14 @@ class _Pickler:
self.write(NONE)
dispatch[type(None)] = save_none
- def save_ellipsis(self, obj):
- self.save_global(Ellipsis, 'Ellipsis')
- dispatch[type(Ellipsis)] = save_ellipsis
-
- def save_notimplemented(self, obj):
- self.save_global(NotImplemented, 'NotImplemented')
- dispatch[type(NotImplemented)] = save_notimplemented
-
def save_bool(self, obj):
if self.proto >= 2:
- self.write(obj and NEWTRUE or NEWFALSE)
+ self.write(NEWTRUE if obj else NEWFALSE)
else:
- self.write(obj and TRUE or FALSE)
+ self.write(TRUE if obj else FALSE)
dispatch[bool] = save_bool
- def save_long(self, obj, pack=struct.pack):
+ def save_long(self, obj):
if self.bin:
# If the int is small enough to fit in a signed 4-byte 2's-comp
# format, we can store it more efficiently than the general
@@ -461,93 +648,95 @@ class _Pickler:
# First one- and two-byte unsigned ints:
if obj >= 0:
if obj <= 0xff:
- self.write(BININT1 + bytes([obj]))
+ self.write(BININT1 + pack("<B", obj))
return
if obj <= 0xffff:
- self.write(BININT2 + bytes([obj&0xff, obj>>8]))
+ self.write(BININT2 + pack("<H", obj))
return
# Next check for 4-byte signed ints:
- high_bits = obj >> 31 # note that Python shift sign-extends
- if high_bits == 0 or high_bits == -1:
- # All high bits are copies of bit 2**31, so the value
- # fits in a 4-byte signed int.
+ if -0x80000000 <= obj <= 0x7fffffff:
self.write(BININT + pack("<i", obj))
return
if self.proto >= 2:
encoded = encode_long(obj)
n = len(encoded)
if n < 256:
- self.write(LONG1 + bytes([n]) + encoded)
+ self.write(LONG1 + pack("<B", n) + encoded)
else:
self.write(LONG4 + pack("<i", n) + encoded)
return
self.write(LONG + repr(obj).encode("ascii") + b'L\n')
dispatch[int] = save_long
- def save_float(self, obj, pack=struct.pack):
+ def save_float(self, obj):
if self.bin:
self.write(BINFLOAT + pack('>d', obj))
else:
self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
dispatch[float] = save_float
- def save_bytes(self, obj, pack=struct.pack):
+ def save_bytes(self, obj):
if self.proto < 3:
- if len(obj) == 0:
+ if not obj: # bytes object is empty
self.save_reduce(bytes, (), obj=obj)
else:
self.save_reduce(codecs.encode,
(str(obj, 'latin1'), 'latin1'), obj=obj)
return
n = len(obj)
- if n < 256:
- self.write(SHORT_BINBYTES + bytes([n]) + bytes(obj))
+ if n <= 0xff:
+ self.write(SHORT_BINBYTES + pack("<B", n) + obj)
+ elif n > 0xffffffff and self.proto >= 4:
+ self.write(BINBYTES8 + pack("<Q", n) + obj)
else:
- self.write(BINBYTES + pack("<I", n) + bytes(obj))
+ self.write(BINBYTES + pack("<I", n) + obj)
self.memoize(obj)
dispatch[bytes] = save_bytes
- def save_str(self, obj, pack=struct.pack):
+ def save_str(self, obj):
if self.bin:
encoded = obj.encode('utf-8', 'surrogatepass')
n = len(encoded)
- self.write(BINUNICODE + pack("<I", n) + encoded)
+ if n <= 0xff and self.proto >= 4:
+ self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
+ elif n > 0xffffffff and self.proto >= 4:
+ self.write(BINUNICODE8 + pack("<Q", n) + encoded)
+ else:
+ self.write(BINUNICODE + pack("<I", n) + encoded)
else:
obj = obj.replace("\\", "\\u005c")
obj = obj.replace("\n", "\\u000a")
- self.write(UNICODE + bytes(obj.encode('raw-unicode-escape')) +
+ self.write(UNICODE + obj.encode('raw-unicode-escape') +
b'\n')
self.memoize(obj)
dispatch[str] = save_str
def save_tuple(self, obj):
- write = self.write
- proto = self.proto
-
- n = len(obj)
- if n == 0:
- if proto:
- write(EMPTY_TUPLE)
+ if not obj: # tuple is empty
+ if self.bin:
+ self.write(EMPTY_TUPLE)
else:
- write(MARK + TUPLE)
+ self.write(MARK + TUPLE)
return
+ n = len(obj)
save = self.save
memo = self.memo
- if n <= 3 and proto >= 2:
+ if n <= 3 and self.proto >= 2:
for element in obj:
save(element)
# Subtle. Same as in the big comment below.
if id(obj) in memo:
get = self.get(memo[id(obj)][0])
- write(POP * n + get)
+ self.write(POP * n + get)
else:
- write(_tuplesize2code[n])
+ self.write(_tuplesize2code[n])
self.memoize(obj)
return
# proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
# has more than 3 elements.
+ write = self.write
write(MARK)
for element in obj:
save(element)
@@ -561,25 +750,23 @@ class _Pickler:
# could have been done in the "for element" loop instead, but
# recursive tuples are a rare thing.
get = self.get(memo[id(obj)][0])
- if proto:
+ if self.bin:
write(POP_MARK + get)
else: # proto 0 -- POP_MARK not available
write(POP * (n+1) + get)
return
# No recursion.
- self.write(TUPLE)
+ write(TUPLE)
self.memoize(obj)
dispatch[tuple] = save_tuple
def save_list(self, obj):
- write = self.write
-
if self.bin:
- write(EMPTY_LIST)
+ self.write(EMPTY_LIST)
else: # proto 0 -- can't use EMPTY_LIST
- write(MARK + LIST)
+ self.write(MARK + LIST)
self.memoize(obj)
self._batch_appends(obj)
@@ -599,17 +786,9 @@ class _Pickler:
write(APPEND)
return
- items = iter(items)
- r = range(self._BATCHSIZE)
- while items is not None:
- tmp = []
- for i in r:
- try:
- x = next(items)
- tmp.append(x)
- except StopIteration:
- items = None
- break
+ it = iter(items)
+ while True:
+ tmp = list(islice(it, self._BATCHSIZE))
n = len(tmp)
if n > 1:
write(MARK)
@@ -620,14 +799,14 @@ class _Pickler:
save(tmp[0])
write(APPEND)
# else tmp is empty, and we're done
+ if n < self._BATCHSIZE:
+ return
def save_dict(self, obj):
- write = self.write
-
if self.bin:
- write(EMPTY_DICT)
+ self.write(EMPTY_DICT)
else: # proto 0 -- can't use EMPTY_DICT
- write(MARK + DICT)
+ self.write(MARK + DICT)
self.memoize(obj)
self._batch_setitems(obj.items())
@@ -648,16 +827,9 @@ class _Pickler:
write(SETITEM)
return
- items = iter(items)
- r = range(self._BATCHSIZE)
- while items is not None:
- tmp = []
- for i in r:
- try:
- tmp.append(next(items))
- except StopIteration:
- items = None
- break
+ it = iter(items)
+ while True:
+ tmp = list(islice(it, self._BATCHSIZE))
n = len(tmp)
if n > 1:
write(MARK)
@@ -671,55 +843,109 @@ class _Pickler:
save(v)
write(SETITEM)
# else tmp is empty, and we're done
+ if n < self._BATCHSIZE:
+ return
+
+ def save_set(self, obj):
+ save = self.save
+ write = self.write
+
+ if self.proto < 4:
+ self.save_reduce(set, (list(obj),), obj=obj)
+ return
+
+ write(EMPTY_SET)
+ self.memoize(obj)
+
+ it = iter(obj)
+ while True:
+ batch = list(islice(it, self._BATCHSIZE))
+ n = len(batch)
+ if n > 0:
+ write(MARK)
+ for item in batch:
+ save(item)
+ write(ADDITEMS)
+ if n < self._BATCHSIZE:
+ return
+ dispatch[set] = save_set
+
+ def save_frozenset(self, obj):
+ save = self.save
+ write = self.write
+
+ if self.proto < 4:
+ self.save_reduce(frozenset, (list(obj),), obj=obj)
+ return
+
+ write(MARK)
+ for item in obj:
+ save(item)
+
+ if id(obj) in self.memo:
+ # If the object is already in the memo, this means it is
+ # recursive. In this case, throw away everything we put on the
+ # stack, and fetch the object back from the memo.
+ write(POP_MARK + self.get(self.memo[id(obj)][0]))
+ return
- def save_global(self, obj, name=None, pack=struct.pack):
+ write(FROZENSET)
+ self.memoize(obj)
+ dispatch[frozenset] = save_frozenset
+
+ def save_global(self, obj, name=None):
write = self.write
memo = self.memo
+ if name is None and self.proto >= 4:
+ name = getattr(obj, '__qualname__', None)
if name is None:
name = obj.__name__
- module = getattr(obj, "__module__", None)
- if module is None:
- module = whichmodule(obj, name)
-
+ module_name = whichmodule(obj, name, allow_qualname=self.proto >= 4)
try:
- __import__(module, level=0)
- mod = sys.modules[module]
- klass = getattr(mod, name)
+ __import__(module_name, level=0)
+ module = sys.modules[module_name]
+ obj2 = _getattribute(module, name, allow_qualname=self.proto >= 4)
except (ImportError, KeyError, AttributeError):
raise PicklingError(
"Can't pickle %r: it's not found as %s.%s" %
- (obj, module, name))
+ (obj, module_name, name))
else:
- if klass is not obj:
+ if obj2 is not obj:
raise PicklingError(
"Can't pickle %r: it's not the same object as %s.%s" %
- (obj, module, name))
+ (obj, module_name, name))
if self.proto >= 2:
- code = _extension_registry.get((module, name))
+ code = _extension_registry.get((module_name, name))
if code:
assert code > 0
if code <= 0xff:
- write(EXT1 + bytes([code]))
+ write(EXT1 + pack("<B", code))
elif code <= 0xffff:
- write(EXT2 + bytes([code&0xff, code>>8]))
+ write(EXT2 + pack("<H", code))
else:
write(EXT4 + pack("<i", code))
return
# Non-ASCII identifiers are supported only with protocols >= 3.
- if self.proto >= 3:
- write(GLOBAL + bytes(module, "utf-8") + b'\n' +
+ if self.proto >= 4:
+ self.save(module_name)
+ self.save(name)
+ write(STACK_GLOBAL)
+ elif self.proto >= 3:
+ write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
else:
if self.fix_imports:
- if (module, name) in _compat_pickle.REVERSE_NAME_MAPPING:
- module, name = _compat_pickle.REVERSE_NAME_MAPPING[(module, name)]
- if module in _compat_pickle.REVERSE_IMPORT_MAPPING:
- module = _compat_pickle.REVERSE_IMPORT_MAPPING[module]
+ r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
+ r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
+ if (module_name, name) in r_name_mapping:
+ module_name, name = r_name_mapping[(module_name, name)]
+ if module_name in r_import_mapping:
+ module_name = r_import_mapping[module_name]
try:
- write(GLOBAL + bytes(module, "ascii") + b'\n' +
+ write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
raise PicklingError(
@@ -738,58 +964,8 @@ class _Pickler:
return self.save_global(obj)
dispatch[FunctionType] = save_global
- dispatch[BuiltinFunctionType] = save_global
dispatch[type] = save_type
-# Pickling helpers
-
-def _keep_alive(x, memo):
- """Keeps a reference to the object x in the memo.
-
- Because we remember objects by their id, we have
- to assure that possibly temporary objects are kept
- alive by referencing them.
- We store a reference at the id of the memo, which should
- normally not be used unless someone tries to deepcopy
- the memo itself...
- """
- try:
- memo[id(memo)].append(x)
- except KeyError:
- # aha, this is the first one :-)
- memo[id(memo)]=[x]
-
-
-# A cache for whichmodule(), mapping a function object to the name of
-# the module in which the function was found.
-
-classmap = {} # called classmap for backwards compatibility
-
-def whichmodule(func, funcname):
- """Figure out the module in which a function occurs.
-
- Search sys.modules for the module.
- Cache in classmap.
- Return a module name.
- If the function cannot be found, return "__main__".
- """
- # Python functions should always get an __module__ from their globals.
- mod = getattr(func, "__module__", None)
- if mod is not None:
- return mod
- if func in classmap:
- return classmap[func]
-
- for name, module in list(sys.modules.items()):
- if module is None:
- continue # skip dummy package entries
- if name != '__main__' and getattr(module, funcname, None) is func:
- break
- else:
- name = '__main__'
- classmap[func] = name
- return name
-
# Unpickling machinery
@@ -799,8 +975,14 @@ class _Unpickler:
encoding="ASCII", errors="strict"):
"""This takes a binary file for reading a pickle data stream.
- The protocol version of the pickle is detected automatically, so no
- proto argument is needed.
+ The protocol version of the pickle is detected automatically, so
+ no proto argument is needed.
+
+ The argument *file* must have two methods, a read() method that
+ takes an integer argument, and a readline() method that requires
+ no arguments. Both methods should return bytes. Thus *file*
+ can be a binary file object opened for reading, a io.BytesIO
+ object, or any other custom object that meets this interface.
The file-like object must have two methods, a read() method
that takes an integer argument, and a readline() method that
@@ -809,16 +991,17 @@ class _Unpickler:
reading, a BytesIO object, or any other custom object that
meets this interface.
- Optional keyword arguments are *fix_imports*, *encoding* and *errors*,
- which are used to control compatiblity support for pickle stream
- generated by Python 2.x. If *fix_imports* is True, pickle will try to
- map the old Python 2.x names to the new names used in Python 3.x. The
- *encoding* and *errors* tell pickle how to decode 8-bit string
- instances pickled by Python 2.x; these default to 'ASCII' and
- 'strict', respectively.
+ Optional keyword arguments are *fix_imports*, *encoding* and
+ *errors*, which are used to control compatiblity support for
+ pickle stream generated by Python 2. If *fix_imports* is True,
+ pickle will try to map the old Python 2 names to the new names
+ used in Python 3. The *encoding* and *errors* tell pickle how
+ to decode 8-bit string instances pickled by Python 2; these
+ default to 'ASCII' and 'strict', respectively. *encoding* can be
+ 'bytes' to read theses 8-bit string instances as bytes objects.
"""
- self.readline = file.readline
- self.read = file.read
+ self._file_readline = file.readline
+ self._file_read = file.read
self.memo = {}
self.encoding = encoding
self.errors = errors
@@ -832,16 +1015,20 @@ class _Unpickler:
"""
# Check whether Unpickler was initialized correctly. This is
# only needed to mimic the behavior of _pickle.Unpickler.dump().
- if not hasattr(self, "read"):
+ if not hasattr(self, "_file_read"):
raise UnpicklingError("Unpickler.__init__() was not called by "
"%s.__init__()" % (self.__class__.__name__,))
+ self._unframer = _Unframer(self._file_read, self._file_readline)
+ self.read = self._unframer.read
+ self.readline = self._unframer.readline
self.mark = object() # any new unique object
self.stack = []
self.append = self.stack.append
+ self.proto = 0
read = self.read
dispatch = self.dispatch
try:
- while 1:
+ while True:
key = read(1)
if not key:
raise EOFError
@@ -871,12 +1058,19 @@ class _Unpickler:
dispatch = {}
def load_proto(self):
- proto = ord(self.read(1))
+ proto = self.read(1)[0]
if not 0 <= proto <= HIGHEST_PROTOCOL:
raise ValueError("unsupported pickle protocol: %d" % proto)
self.proto = proto
dispatch[PROTO[0]] = load_proto
+ def load_frame(self):
+ frame_size, = unpack('<Q', self.read(8))
+ if frame_size > sys.maxsize:
+ raise ValueError("frame size > sys.maxsize: %d" % frame_size)
+ self._unframer.load_frame(frame_size)
+ dispatch[FRAME[0]] = load_frame
+
def load_persid(self):
pid = self.readline()[:-1].decode("ascii")
self.append(self.persistent_load(pid))
@@ -906,43 +1100,40 @@ class _Unpickler:
elif data == TRUE[1:]:
val = True
else:
- try:
- val = int(data, 0)
- except ValueError:
- val = int(data, 0)
+ val = int(data, 0)
self.append(val)
dispatch[INT[0]] = load_int
def load_binint(self):
- self.append(mloads(b'i' + self.read(4)))
+ self.append(unpack('<i', self.read(4))[0])
dispatch[BININT[0]] = load_binint
def load_binint1(self):
- self.append(ord(self.read(1)))
+ self.append(self.read(1)[0])
dispatch[BININT1[0]] = load_binint1
def load_binint2(self):
- self.append(mloads(b'i' + self.read(2) + b'\000\000'))
+ self.append(unpack('<H', self.read(2))[0])
dispatch[BININT2[0]] = load_binint2
def load_long(self):
- val = self.readline()[:-1].decode("ascii")
- if val and val[-1] == 'L':
+ val = self.readline()[:-1]
+ if val and val[-1] == b'L'[0]:
val = val[:-1]
self.append(int(val, 0))
dispatch[LONG[0]] = load_long
def load_long1(self):
- n = ord(self.read(1))
+ n = self.read(1)[0]
data = self.read(n)
self.append(decode_long(data))
dispatch[LONG1[0]] = load_long1
def load_long4(self):
- n = mloads(b'i' + self.read(4))
+ n, = unpack('<i', self.read(4))
if n < 0:
# Corrupt or hostile pickle -- we never write one like this
- raise UnpicklingError("LONG pickle has negative byte count");
+ raise UnpicklingError("LONG pickle has negative byte count")
data = self.read(n)
self.append(decode_long(data))
dispatch[LONG4[0]] = load_long4
@@ -951,39 +1142,43 @@ class _Unpickler:
self.append(float(self.readline()[:-1]))
dispatch[FLOAT[0]] = load_float
- def load_binfloat(self, unpack=struct.unpack):
+ def load_binfloat(self):
self.append(unpack('>d', self.read(8))[0])
dispatch[BINFLOAT[0]] = load_binfloat
+ def _decode_string(self, value):
+ # Used to allow strings from Python 2 to be decoded either as
+ # bytes or Unicode strings. This should be used only with the
+ # STRING, BINSTRING and SHORT_BINSTRING opcodes.
+ if self.encoding == "bytes":
+ return value
+ else:
+ return value.decode(self.encoding, self.errors)
+
def load_string(self):
- orig = self.readline()
- rep = orig[:-1]
- for q in (b'"', b"'"): # double or single quote
- if rep.startswith(q):
- if len(rep) < 2 or not rep.endswith(q):
- raise ValueError("insecure string pickle")
- rep = rep[len(q):-len(q)]
- break
+ data = self.readline()[:-1]
+ # Strip outermost quotes
+ if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
+ data = data[1:-1]
else:
- raise ValueError("insecure string pickle: %r" % orig)
- self.append(codecs.escape_decode(rep)[0]
- .decode(self.encoding, self.errors))
+ raise UnpicklingError("the STRING opcode argument must be quoted")
+ self.append(self._decode_string(codecs.escape_decode(data)[0]))
dispatch[STRING[0]] = load_string
def load_binstring(self):
# Deprecated BINSTRING uses signed 32-bit length
- len = mloads(b'i' + self.read(4))
+ len, = unpack('<i', self.read(4))
if len < 0:
- raise UnpicklingError("BINSTRING pickle has negative byte count");
+ raise UnpicklingError("BINSTRING pickle has negative byte count")
data = self.read(len)
- value = str(data, self.encoding, self.errors)
- self.append(value)
+ self.append(self._decode_string(data))
dispatch[BINSTRING[0]] = load_binstring
- def load_binbytes(self, unpack=struct.unpack, maxsize=sys.maxsize):
+ def load_binbytes(self):
len, = unpack('<I', self.read(4))
if len > maxsize:
- raise UnpicklingError("BINBYTES exceeds system's maximum size of %d bytes" % maxsize);
+ raise UnpicklingError("BINBYTES exceeds system's maximum size "
+ "of %d bytes" % maxsize)
self.append(self.read(len))
dispatch[BINBYTES[0]] = load_binbytes
@@ -991,25 +1186,38 @@ class _Unpickler:
self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
dispatch[UNICODE[0]] = load_unicode
- def load_binunicode(self, unpack=struct.unpack, maxsize=sys.maxsize):
+ def load_binunicode(self):
len, = unpack('<I', self.read(4))
if len > maxsize:
- raise UnpicklingError("BINUNICODE exceeds system's maximum size of %d bytes" % maxsize);
+ raise UnpicklingError("BINUNICODE exceeds system's maximum size "
+ "of %d bytes" % maxsize)
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
dispatch[BINUNICODE[0]] = load_binunicode
+ def load_binunicode8(self):
+ len, = unpack('<Q', self.read(8))
+ if len > maxsize:
+ raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
+ "of %d bytes" % maxsize)
+ self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
+ dispatch[BINUNICODE8[0]] = load_binunicode8
+
def load_short_binstring(self):
- len = ord(self.read(1))
- data = bytes(self.read(len))
- value = str(data, self.encoding, self.errors)
- self.append(value)
+ len = self.read(1)[0]
+ data = self.read(len)
+ self.append(self._decode_string(data))
dispatch[SHORT_BINSTRING[0]] = load_short_binstring
def load_short_binbytes(self):
- len = ord(self.read(1))
- self.append(bytes(self.read(len)))
+ len = self.read(1)[0]
+ self.append(self.read(len))
dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
+ def load_short_binunicode(self):
+ len = self.read(1)[0]
+ self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
+ dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
+
def load_tuple(self):
k = self.marker()
self.stack[k:] = [tuple(self.stack[k+1:])]
@@ -1039,6 +1247,15 @@ class _Unpickler:
self.append({})
dispatch[EMPTY_DICT[0]] = load_empty_dictionary
+ def load_empty_set(self):
+ self.append(set())
+ dispatch[EMPTY_SET[0]] = load_empty_set
+
+ def load_frozenset(self):
+ k = self.marker()
+ self.stack[k:] = [frozenset(self.stack[k+1:])]
+ dispatch[FROZENSET[0]] = load_frozenset
+
def load_list(self):
k = self.marker()
self.stack[k:] = [self.stack[k+1:]]
@@ -1046,12 +1263,9 @@ class _Unpickler:
def load_dict(self):
k = self.marker()
- d = {}
items = self.stack[k+1:]
- for i in range(0, len(items), 2):
- key = items[i]
- value = items[i+1]
- d[key] = value
+ d = {items[i]: items[i+1]
+ for i in range(0, len(items), 2)}
self.stack[k:] = [d]
dispatch[DICT[0]] = load_dict
@@ -1090,11 +1304,19 @@ class _Unpickler:
def load_newobj(self):
args = self.stack.pop()
- cls = self.stack[-1]
+ cls = self.stack.pop()
obj = cls.__new__(cls, *args)
- self.stack[-1] = obj
+ self.append(obj)
dispatch[NEWOBJ[0]] = load_newobj
+ def load_newobj_ex(self):
+ kwargs = self.stack.pop()
+ args = self.stack.pop()
+ cls = self.stack.pop()
+ obj = cls.__new__(cls, *args, **kwargs)
+ self.append(obj)
+ dispatch[NEWOBJ_EX[0]] = load_newobj_ex
+
def load_global(self):
module = self.readline()[:-1].decode("utf-8")
name = self.readline()[:-1].decode("utf-8")
@@ -1102,18 +1324,26 @@ class _Unpickler:
self.append(klass)
dispatch[GLOBAL[0]] = load_global
+ def load_stack_global(self):
+ name = self.stack.pop()
+ module = self.stack.pop()
+ if type(name) is not str or type(module) is not str:
+ raise UnpicklingError("STACK_GLOBAL requires str")
+ self.append(self.find_class(module, name))
+ dispatch[STACK_GLOBAL[0]] = load_stack_global
+
def load_ext1(self):
- code = ord(self.read(1))
+ code = self.read(1)[0]
self.get_extension(code)
dispatch[EXT1[0]] = load_ext1
def load_ext2(self):
- code = mloads(b'i' + self.read(2) + b'\000\000')
+ code, = unpack('<H', self.read(2))
self.get_extension(code)
dispatch[EXT2[0]] = load_ext2
def load_ext4(self):
- code = mloads(b'i' + self.read(4))
+ code, = unpack('<i', self.read(4))
self.get_extension(code)
dispatch[EXT4[0]] = load_ext4
@@ -1127,7 +1357,7 @@ class _Unpickler:
if not key:
if code <= 0: # note that 0 is forbidden
# Corrupt or hostile pickle.
- raise UnpicklingError("EXT specifies code <= 0");
+ raise UnpicklingError("EXT specifies code <= 0")
raise ValueError("unregistered extension code %d" % code)
obj = self.find_class(*key)
_extension_cache[code] = obj
@@ -1141,9 +1371,8 @@ class _Unpickler:
if module in _compat_pickle.IMPORT_MAPPING:
module = _compat_pickle.IMPORT_MAPPING[module]
__import__(module, level=0)
- mod = sys.modules[module]
- klass = getattr(mod, name)
- return klass
+ return _getattribute(sys.modules[module], name,
+ allow_qualname=self.proto >= 4)
def load_reduce(self):
stack = self.stack
@@ -1181,7 +1410,7 @@ class _Unpickler:
self.append(self.memo[i])
dispatch[BINGET[0]] = load_binget
- def load_long_binget(self, unpack=struct.unpack):
+ def load_long_binget(self):
i, = unpack('<I', self.read(4))
self.append(self.memo[i])
dispatch[LONG_BINGET[0]] = load_long_binget
@@ -1200,13 +1429,18 @@ class _Unpickler:
self.memo[i] = self.stack[-1]
dispatch[BINPUT[0]] = load_binput
- def load_long_binput(self, unpack=struct.unpack, maxsize=sys.maxsize):
+ def load_long_binput(self):
i, = unpack('<I', self.read(4))
if i > maxsize:
raise ValueError("negative LONG_BINPUT argument")
self.memo[i] = self.stack[-1]
dispatch[LONG_BINPUT[0]] = load_long_binput
+ def load_memoize(self):
+ memo = self.memo
+ memo[len(memo)] = self.stack[-1]
+ dispatch[MEMOIZE[0]] = load_memoize
+
def load_append(self):
stack = self.stack
value = stack.pop()
@@ -1246,12 +1480,26 @@ class _Unpickler:
del stack[mark:]
dispatch[SETITEMS[0]] = load_setitems
+ def load_additems(self):
+ stack = self.stack
+ mark = self.marker()
+ set_obj = stack[mark - 1]
+ items = stack[mark + 1:]
+ if isinstance(set_obj, set):
+ set_obj.update(items)
+ else:
+ add = set_obj.add
+ for item in items:
+ add(item)
+ del stack[mark:]
+ dispatch[ADDITEMS[0]] = load_additems
+
def load_build(self):
stack = self.stack
state = stack.pop()
inst = stack[-1]
setstate = getattr(inst, "__setstate__", None)
- if setstate:
+ if setstate is not None:
setstate(state)
return
slotstate = None
@@ -1279,86 +1527,46 @@ class _Unpickler:
raise _Stop(value)
dispatch[STOP[0]] = load_stop
-# Encode/decode ints.
-
-def encode_long(x):
- r"""Encode a long to a two's complement little-endian binary string.
- Note that 0 is a special case, returning an empty string, to save a
- byte in the LONG1 pickling context.
-
- >>> encode_long(0)
- b''
- >>> encode_long(255)
- b'\xff\x00'
- >>> encode_long(32767)
- b'\xff\x7f'
- >>> encode_long(-256)
- b'\x00\xff'
- >>> encode_long(-32768)
- b'\x00\x80'
- >>> encode_long(-128)
- b'\x80'
- >>> encode_long(127)
- b'\x7f'
- >>>
- """
- if x == 0:
- return b''
- nbytes = (x.bit_length() >> 3) + 1
- result = x.to_bytes(nbytes, byteorder='little', signed=True)
- if x < 0 and nbytes > 1:
- if result[-1] == 0xff and (result[-2] & 0x80) != 0:
- result = result[:-1]
- return result
-
-def decode_long(data):
- r"""Decode an int from a two's complement little-endian binary string.
-
- >>> decode_long(b'')
- 0
- >>> decode_long(b"\xff\x00")
- 255
- >>> decode_long(b"\xff\x7f")
- 32767
- >>> decode_long(b"\x00\xff")
- -256
- >>> decode_long(b"\x00\x80")
- -32768
- >>> decode_long(b"\x80")
- -128
- >>> decode_long(b"\x7f")
- 127
- """
- return int.from_bytes(data, byteorder='little', signed=True)
# Shorthands
-def dump(obj, file, protocol=None, *, fix_imports=True):
- Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
+def _dump(obj, file, protocol=None, *, fix_imports=True):
+ _Pickler(file, protocol, fix_imports=fix_imports).dump(obj)
-def dumps(obj, protocol=None, *, fix_imports=True):
+def _dumps(obj, protocol=None, *, fix_imports=True):
f = io.BytesIO()
- Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
+ _Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
res = f.getvalue()
assert isinstance(res, bytes_types)
return res
-def load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
- return Unpickler(file, fix_imports=fix_imports,
+def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict"):
+ return _Unpickler(file, fix_imports=fix_imports,
encoding=encoding, errors=errors).load()
-def loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
+def _loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
if isinstance(s, str):
raise TypeError("Can't load pickle from unicode string")
file = io.BytesIO(s)
- return Unpickler(file, fix_imports=fix_imports,
- encoding=encoding, errors=errors).load()
+ return _Unpickler(file, fix_imports=fix_imports,
+ encoding=encoding, errors=errors).load()
# Use the faster _pickle if possible
try:
- from _pickle import *
+ from _pickle import (
+ PickleError,
+ PicklingError,
+ UnpicklingError,
+ Pickler,
+ Unpickler,
+ dump,
+ dumps,
+ load,
+ loads
+ )
except ImportError:
Pickler, Unpickler = _Pickler, _Unpickler
+ dump, dumps, load, loads = _dump, _dumps, _load, _loads
# Doctest
def _test():
diff --git a/Lib/pickletools.py b/Lib/pickletools.py
index 612fa8f..71c2aa1 100644
--- a/Lib/pickletools.py
+++ b/Lib/pickletools.py
@@ -11,6 +11,7 @@ dis(pickle, out=None, memo=None, indentlevel=4)
'''
import codecs
+import io
import pickle
import re
import sys
@@ -34,119 +35,118 @@ bytes_types = pickle.bytes_types
# by a later GET.
-"""
-"A pickle" is a program for a virtual pickle machine (PM, but more accurately
-called an unpickling machine). It's a sequence of opcodes, interpreted by the
-PM, building an arbitrarily complex Python object.
-
-For the most part, the PM is very simple: there are no looping, testing, or
-conditional instructions, no arithmetic and no function calls. Opcodes are
-executed once each, from first to last, until a STOP opcode is reached.
-
-The PM has two data areas, "the stack" and "the memo".
-
-Many opcodes push Python objects onto the stack; e.g., INT pushes a Python
-integer object on the stack, whose value is gotten from a decimal string
-literal immediately following the INT opcode in the pickle bytestream. Other
-opcodes take Python objects off the stack. The result of unpickling is
-whatever object is left on the stack when the final STOP opcode is executed.
-
-The memo is simply an array of objects, or it can be implemented as a dict
-mapping little integers to objects. The memo serves as the PM's "long term
-memory", and the little integers indexing the memo are akin to variable
-names. Some opcodes pop a stack object into the memo at a given index,
-and others push a memo object at a given index onto the stack again.
-
-At heart, that's all the PM has. Subtleties arise for these reasons:
-
-+ Object identity. Objects can be arbitrarily complex, and subobjects
- may be shared (for example, the list [a, a] refers to the same object a
- twice). It can be vital that unpickling recreate an isomorphic object
- graph, faithfully reproducing sharing.
-
-+ Recursive objects. For example, after "L = []; L.append(L)", L is a
- list, and L[0] is the same list. This is related to the object identity
- point, and some sequences of pickle opcodes are subtle in order to
- get the right result in all cases.
-
-+ Things pickle doesn't know everything about. Examples of things pickle
- does know everything about are Python's builtin scalar and container
- types, like ints and tuples. They generally have opcodes dedicated to
- them. For things like module references and instances of user-defined
- classes, pickle's knowledge is limited. Historically, many enhancements
- have been made to the pickle protocol in order to do a better (faster,
- and/or more compact) job on those.
-
-+ Backward compatibility and micro-optimization. As explained below,
- pickle opcodes never go away, not even when better ways to do a thing
- get invented. The repertoire of the PM just keeps growing over time.
- For example, protocol 0 had two opcodes for building Python integers (INT
- and LONG), protocol 1 added three more for more-efficient pickling of short
- integers, and protocol 2 added two more for more-efficient pickling of
- long integers (before protocol 2, the only ways to pickle a Python long
- took time quadratic in the number of digits, for both pickling and
- unpickling). "Opcode bloat" isn't so much a subtlety as a source of
- wearying complication.
-
-
-Pickle protocols:
-
-For compatibility, the meaning of a pickle opcode never changes. Instead new
-pickle opcodes get added, and each version's unpickler can handle all the
-pickle opcodes in all protocol versions to date. So old pickles continue to
-be readable forever. The pickler can generally be told to restrict itself to
-the subset of opcodes available under previous protocol versions too, so that
-users can create pickles under the current version readable by older
-versions. However, a pickle does not contain its version number embedded
-within it. If an older unpickler tries to read a pickle using a later
-protocol, the result is most likely an exception due to seeing an unknown (in
-the older unpickler) opcode.
-
-The original pickle used what's now called "protocol 0", and what was called
-"text mode" before Python 2.3. The entire pickle bytestream is made up of
-printable 7-bit ASCII characters, plus the newline character, in protocol 0.
-That's why it was called text mode. Protocol 0 is small and elegant, but
-sometimes painfully inefficient.
-
-The second major set of additions is now called "protocol 1", and was called
-"binary mode" before Python 2.3. This added many opcodes with arguments
-consisting of arbitrary bytes, including NUL bytes and unprintable "high bit"
-bytes. Binary mode pickles can be substantially smaller than equivalent
-text mode pickles, and sometimes faster too; e.g., BININT represents a 4-byte
-int as 4 bytes following the opcode, which is cheaper to unpickle than the
-(perhaps) 11-character decimal string attached to INT. Protocol 1 also added
-a number of opcodes that operate on many stack elements at once (like APPENDS
-and SETITEMS), and "shortcut" opcodes (like EMPTY_DICT and EMPTY_TUPLE).
-
-The third major set of additions came in Python 2.3, and is called "protocol
-2". This added:
-
-- A better way to pickle instances of new-style classes (NEWOBJ).
-
-- A way for a pickle to identify its protocol (PROTO).
-
-- Time- and space- efficient pickling of long ints (LONG{1,4}).
-
-- Shortcuts for small tuples (TUPLE{1,2,3}}.
-
-- Dedicated opcodes for bools (NEWTRUE, NEWFALSE).
-
-- The "extension registry", a vector of popular objects that can be pushed
- efficiently by index (EXT{1,2,4}). This is akin to the memo and GET, but
- the registry contents are predefined (there's nothing akin to the memo's
- PUT).
-
-Another independent change with Python 2.3 is the abandonment of any
-pretense that it might be safe to load pickles received from untrusted
-parties -- no sufficient security analysis has been done to guarantee
-this and there isn't a use case that warrants the expense of such an
-analysis.
-
-To this end, all tests for __safe_for_unpickling__ or for
-copyreg.safe_constructors are removed from the unpickling code.
-References to these variables in the descriptions below are to be seen
-as describing unpickling in Python 2.2 and before.
-"""
+# "A pickle" is a program for a virtual pickle machine (PM, but more accurately
+# called an unpickling machine). It's a sequence of opcodes, interpreted by the
+# PM, building an arbitrarily complex Python object.
+#
+# For the most part, the PM is very simple: there are no looping, testing, or
+# conditional instructions, no arithmetic and no function calls. Opcodes are
+# executed once each, from first to last, until a STOP opcode is reached.
+#
+# The PM has two data areas, "the stack" and "the memo".
+#
+# Many opcodes push Python objects onto the stack; e.g., INT pushes a Python
+# integer object on the stack, whose value is gotten from a decimal string
+# literal immediately following the INT opcode in the pickle bytestream. Other
+# opcodes take Python objects off the stack. The result of unpickling is
+# whatever object is left on the stack when the final STOP opcode is executed.
+#
+# The memo is simply an array of objects, or it can be implemented as a dict
+# mapping little integers to objects. The memo serves as the PM's "long term
+# memory", and the little integers indexing the memo are akin to variable
+# names. Some opcodes pop a stack object into the memo at a given index,
+# and others push a memo object at a given index onto the stack again.
+#
+# At heart, that's all the PM has. Subtleties arise for these reasons:
+#
+# + Object identity. Objects can be arbitrarily complex, and subobjects
+# may be shared (for example, the list [a, a] refers to the same object a
+# twice). It can be vital that unpickling recreate an isomorphic object
+# graph, faithfully reproducing sharing.
+#
+# + Recursive objects. For example, after "L = []; L.append(L)", L is a
+# list, and L[0] is the same list. This is related to the object identity
+# point, and some sequences of pickle opcodes are subtle in order to
+# get the right result in all cases.
+#
+# + Things pickle doesn't know everything about. Examples of things pickle
+# does know everything about are Python's builtin scalar and container
+# types, like ints and tuples. They generally have opcodes dedicated to
+# them. For things like module references and instances of user-defined
+# classes, pickle's knowledge is limited. Historically, many enhancements
+# have been made to the pickle protocol in order to do a better (faster,
+# and/or more compact) job on those.
+#
+# + Backward compatibility and micro-optimization. As explained below,
+# pickle opcodes never go away, not even when better ways to do a thing
+# get invented. The repertoire of the PM just keeps growing over time.
+# For example, protocol 0 had two opcodes for building Python integers (INT
+# and LONG), protocol 1 added three more for more-efficient pickling of short
+# integers, and protocol 2 added two more for more-efficient pickling of
+# long integers (before protocol 2, the only ways to pickle a Python long
+# took time quadratic in the number of digits, for both pickling and
+# unpickling). "Opcode bloat" isn't so much a subtlety as a source of
+# wearying complication.
+#
+#
+# Pickle protocols:
+#
+# For compatibility, the meaning of a pickle opcode never changes. Instead new
+# pickle opcodes get added, and each version's unpickler can handle all the
+# pickle opcodes in all protocol versions to date. So old pickles continue to
+# be readable forever. The pickler can generally be told to restrict itself to
+# the subset of opcodes available under previous protocol versions too, so that
+# users can create pickles under the current version readable by older
+# versions. However, a pickle does not contain its version number embedded
+# within it. If an older unpickler tries to read a pickle using a later
+# protocol, the result is most likely an exception due to seeing an unknown (in
+# the older unpickler) opcode.
+#
+# The original pickle used what's now called "protocol 0", and what was called
+# "text mode" before Python 2.3. The entire pickle bytestream is made up of
+# printable 7-bit ASCII characters, plus the newline character, in protocol 0.
+# That's why it was called text mode. Protocol 0 is small and elegant, but
+# sometimes painfully inefficient.
+#
+# The second major set of additions is now called "protocol 1", and was called
+# "binary mode" before Python 2.3. This added many opcodes with arguments
+# consisting of arbitrary bytes, including NUL bytes and unprintable "high bit"
+# bytes. Binary mode pickles can be substantially smaller than equivalent
+# text mode pickles, and sometimes faster too; e.g., BININT represents a 4-byte
+# int as 4 bytes following the opcode, which is cheaper to unpickle than the
+# (perhaps) 11-character decimal string attached to INT. Protocol 1 also added
+# a number of opcodes that operate on many stack elements at once (like APPENDS
+# and SETITEMS), and "shortcut" opcodes (like EMPTY_DICT and EMPTY_TUPLE).
+#
+# The third major set of additions came in Python 2.3, and is called "protocol
+# 2". This added:
+#
+# - A better way to pickle instances of new-style classes (NEWOBJ).
+#
+# - A way for a pickle to identify its protocol (PROTO).
+#
+# - Time- and space- efficient pickling of long ints (LONG{1,4}).
+#
+# - Shortcuts for small tuples (TUPLE{1,2,3}}.
+#
+# - Dedicated opcodes for bools (NEWTRUE, NEWFALSE).
+#
+# - The "extension registry", a vector of popular objects that can be pushed
+# efficiently by index (EXT{1,2,4}). This is akin to the memo and GET, but
+# the registry contents are predefined (there's nothing akin to the memo's
+# PUT).
+#
+# Another independent change with Python 2.3 is the abandonment of any
+# pretense that it might be safe to load pickles received from untrusted
+# parties -- no sufficient security analysis has been done to guarantee
+# this and there isn't a use case that warrants the expense of such an
+# analysis.
+#
+# To this end, all tests for __safe_for_unpickling__ or for
+# copyreg.safe_constructors are removed from the unpickling code.
+# References to these variables in the descriptions below are to be seen
+# as describing unpickling in Python 2.2 and before.
+
# Meta-rule: Descriptions are stored in instances of descriptor objects,
# with plain constructors. No meta-language is defined from which
@@ -169,6 +169,7 @@ UP_TO_NEWLINE = -1
TAKEN_FROM_ARGUMENT1 = -2 # num bytes is 1-byte unsigned int
TAKEN_FROM_ARGUMENT4 = -3 # num bytes is 4-byte signed little-endian int
TAKEN_FROM_ARGUMENT4U = -4 # num bytes is 4-byte unsigned little-endian int
+TAKEN_FROM_ARGUMENT8U = -5 # num bytes is 8-byte unsigned little-endian int
class ArgumentDescriptor(object):
__slots__ = (
@@ -176,7 +177,7 @@ class ArgumentDescriptor(object):
'name',
# length of argument, in bytes; an int; UP_TO_NEWLINE and
- # TAKEN_FROM_ARGUMENT{1,4} are negative values for variable-length
+ # TAKEN_FROM_ARGUMENT{1,4,8} are negative values for variable-length
# cases
'n',
@@ -197,7 +198,8 @@ class ArgumentDescriptor(object):
n in (UP_TO_NEWLINE,
TAKEN_FROM_ARGUMENT1,
TAKEN_FROM_ARGUMENT4,
- TAKEN_FROM_ARGUMENT4U))
+ TAKEN_FROM_ARGUMENT4U,
+ TAKEN_FROM_ARGUMENT8U))
self.n = n
self.reader = reader
@@ -289,6 +291,27 @@ uint4 = ArgumentDescriptor(
doc="Four-byte unsigned integer, little-endian.")
+def read_uint8(f):
+ r"""
+ >>> import io
+ >>> read_uint8(io.BytesIO(b'\xff\x00\x00\x00\x00\x00\x00\x00'))
+ 255
+ >>> read_uint8(io.BytesIO(b'\xff' * 8)) == 2**64-1
+ True
+ """
+
+ data = f.read(8)
+ if len(data) == 8:
+ return _unpack("<Q", data)[0]
+ raise ValueError("not enough data in stream to read uint8")
+
+uint8 = ArgumentDescriptor(
+ name='uint8',
+ n=8,
+ reader=read_uint8,
+ doc="Eight-byte unsigned integer, little-endian.")
+
+
def read_stringnl(f, decode=True, stripquotes=True):
r"""
>>> import io
@@ -382,6 +405,36 @@ stringnl_noescape_pair = ArgumentDescriptor(
a single blank separating the two strings.
""")
+
+def read_string1(f):
+ r"""
+ >>> import io
+ >>> read_string1(io.BytesIO(b"\x00"))
+ ''
+ >>> read_string1(io.BytesIO(b"\x03abcdef"))
+ 'abc'
+ """
+
+ n = read_uint1(f)
+ assert n >= 0
+ data = f.read(n)
+ if len(data) == n:
+ return data.decode("latin-1")
+ raise ValueError("expected %d bytes in a string1, but only %d remain" %
+ (n, len(data)))
+
+string1 = ArgumentDescriptor(
+ name="string1",
+ n=TAKEN_FROM_ARGUMENT1,
+ reader=read_string1,
+ doc="""A counted string.
+
+ The first argument is a 1-byte unsigned int giving the number
+ of bytes in the string, and the second argument is that many
+ bytes.
+ """)
+
+
def read_string4(f):
r"""
>>> import io
@@ -416,28 +469,28 @@ string4 = ArgumentDescriptor(
""")
-def read_string1(f):
+def read_bytes1(f):
r"""
>>> import io
- >>> read_string1(io.BytesIO(b"\x00"))
- ''
- >>> read_string1(io.BytesIO(b"\x03abcdef"))
- 'abc'
+ >>> read_bytes1(io.BytesIO(b"\x00"))
+ b''
+ >>> read_bytes1(io.BytesIO(b"\x03abcdef"))
+ b'abc'
"""
n = read_uint1(f)
assert n >= 0
data = f.read(n)
if len(data) == n:
- return data.decode("latin-1")
- raise ValueError("expected %d bytes in a string1, but only %d remain" %
+ return data
+ raise ValueError("expected %d bytes in a bytes1, but only %d remain" %
(n, len(data)))
-string1 = ArgumentDescriptor(
- name="string1",
+bytes1 = ArgumentDescriptor(
+ name="bytes1",
n=TAKEN_FROM_ARGUMENT1,
- reader=read_string1,
- doc="""A counted string.
+ reader=read_bytes1,
+ doc="""A counted bytes string.
The first argument is a 1-byte unsigned int giving the number
of bytes in the string, and the second argument is that many
@@ -487,6 +540,7 @@ def read_bytes4(f):
"""
n = read_uint4(f)
+ assert n >= 0
if n > sys.maxsize:
raise ValueError("bytes4 byte count > sys.maxsize: %d" % n)
data = f.read(n)
@@ -506,6 +560,40 @@ bytes4 = ArgumentDescriptor(
""")
+def read_bytes8(f):
+ r"""
+ >>> import io, struct, sys
+ >>> read_bytes8(io.BytesIO(b"\x00\x00\x00\x00\x00\x00\x00\x00abc"))
+ b''
+ >>> read_bytes8(io.BytesIO(b"\x03\x00\x00\x00\x00\x00\x00\x00abcdef"))
+ b'abc'
+ >>> bigsize8 = struct.pack("<Q", sys.maxsize//3)
+ >>> read_bytes8(io.BytesIO(bigsize8 + b"abcdef")) #doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ValueError: expected ... bytes in a bytes8, but only 6 remain
+ """
+
+ n = read_uint8(f)
+ assert n >= 0
+ if n > sys.maxsize:
+ raise ValueError("bytes8 byte count > sys.maxsize: %d" % n)
+ data = f.read(n)
+ if len(data) == n:
+ return data
+ raise ValueError("expected %d bytes in a bytes8, but only %d remain" %
+ (n, len(data)))
+
+bytes8 = ArgumentDescriptor(
+ name="bytes8",
+ n=TAKEN_FROM_ARGUMENT8U,
+ reader=read_bytes8,
+ doc="""A counted bytes string.
+
+ The first argument is a 8-byte little-endian unsigned int giving
+ the number of bytes, and the second argument is that many bytes.
+ """)
+
def read_unicodestringnl(f):
r"""
>>> import io
@@ -531,6 +619,46 @@ unicodestringnl = ArgumentDescriptor(
escape sequences.
""")
+
+def read_unicodestring1(f):
+ r"""
+ >>> import io
+ >>> s = 'abcd\uabcd'
+ >>> enc = s.encode('utf-8')
+ >>> enc
+ b'abcd\xea\xaf\x8d'
+ >>> n = bytes([len(enc)]) # little-endian 1-byte length
+ >>> t = read_unicodestring1(io.BytesIO(n + enc + b'junk'))
+ >>> s == t
+ True
+
+ >>> read_unicodestring1(io.BytesIO(n + enc[:-1]))
+ Traceback (most recent call last):
+ ...
+ ValueError: expected 7 bytes in a unicodestring1, but only 6 remain
+ """
+
+ n = read_uint1(f)
+ assert n >= 0
+ data = f.read(n)
+ if len(data) == n:
+ return str(data, 'utf-8', 'surrogatepass')
+ raise ValueError("expected %d bytes in a unicodestring1, but only %d "
+ "remain" % (n, len(data)))
+
+unicodestring1 = ArgumentDescriptor(
+ name="unicodestring1",
+ n=TAKEN_FROM_ARGUMENT1,
+ reader=read_unicodestring1,
+ doc="""A counted Unicode string.
+
+ The first argument is a 1-byte little-endian signed int
+ giving the number of bytes in the string, and the second
+ argument-- the UTF-8 encoding of the Unicode string --
+ contains that many bytes.
+ """)
+
+
def read_unicodestring4(f):
r"""
>>> import io
@@ -550,6 +678,7 @@ def read_unicodestring4(f):
"""
n = read_uint4(f)
+ assert n >= 0
if n > sys.maxsize:
raise ValueError("unicodestring4 byte count > sys.maxsize: %d" % n)
data = f.read(n)
@@ -571,6 +700,47 @@ unicodestring4 = ArgumentDescriptor(
""")
+def read_unicodestring8(f):
+ r"""
+ >>> import io
+ >>> s = 'abcd\uabcd'
+ >>> enc = s.encode('utf-8')
+ >>> enc
+ b'abcd\xea\xaf\x8d'
+ >>> n = bytes([len(enc)]) + bytes(7) # little-endian 8-byte length
+ >>> t = read_unicodestring8(io.BytesIO(n + enc + b'junk'))
+ >>> s == t
+ True
+
+ >>> read_unicodestring8(io.BytesIO(n + enc[:-1]))
+ Traceback (most recent call last):
+ ...
+ ValueError: expected 7 bytes in a unicodestring8, but only 6 remain
+ """
+
+ n = read_uint8(f)
+ assert n >= 0
+ if n > sys.maxsize:
+ raise ValueError("unicodestring8 byte count > sys.maxsize: %d" % n)
+ data = f.read(n)
+ if len(data) == n:
+ return str(data, 'utf-8', 'surrogatepass')
+ raise ValueError("expected %d bytes in a unicodestring8, but only %d "
+ "remain" % (n, len(data)))
+
+unicodestring8 = ArgumentDescriptor(
+ name="unicodestring8",
+ n=TAKEN_FROM_ARGUMENT8U,
+ reader=read_unicodestring8,
+ doc="""A counted Unicode string.
+
+ The first argument is a 8-byte little-endian signed int
+ giving the number of bytes in the string, and the second
+ argument-- the UTF-8 encoding of the Unicode string --
+ contains that many bytes.
+ """)
+
+
def read_decimalnl_short(f):
r"""
>>> import io
@@ -799,103 +969,107 @@ class StackObject(object):
return self.name
-pyint = StackObject(
- name='int',
- obtype=int,
- doc="A short (as opposed to long) Python integer object.")
-
-pylong = StackObject(
- name='long',
- obtype=int,
- doc="A long (as opposed to short) Python integer object.")
+pyint = pylong = StackObject(
+ name='int',
+ obtype=int,
+ doc="A Python integer object.")
pyinteger_or_bool = StackObject(
- name='int_or_bool',
- obtype=(int, bool),
- doc="A Python integer object (short or long), or "
- "a Python bool.")
+ name='int_or_bool',
+ obtype=(int, bool),
+ doc="A Python integer or boolean object.")
pybool = StackObject(
- name='bool',
- obtype=(bool,),
- doc="A Python bool object.")
+ name='bool',
+ obtype=bool,
+ doc="A Python boolean object.")
pyfloat = StackObject(
- name='float',
- obtype=float,
- doc="A Python float object.")
+ name='float',
+ obtype=float,
+ doc="A Python float object.")
-pystring = StackObject(
- name='string',
- obtype=bytes,
- doc="A Python (8-bit) string object.")
+pybytes_or_str = pystring = StackObject(
+ name='bytes_or_str',
+ obtype=(bytes, str),
+ doc="A Python bytes or (Unicode) string object.")
pybytes = StackObject(
- name='bytes',
- obtype=bytes,
- doc="A Python bytes object.")
+ name='bytes',
+ obtype=bytes,
+ doc="A Python bytes object.")
pyunicode = StackObject(
- name='str',
- obtype=str,
- doc="A Python (Unicode) string object.")
+ name='str',
+ obtype=str,
+ doc="A Python (Unicode) string object.")
pynone = StackObject(
- name="None",
- obtype=type(None),
- doc="The Python None object.")
+ name="None",
+ obtype=type(None),
+ doc="The Python None object.")
pytuple = StackObject(
- name="tuple",
- obtype=tuple,
- doc="A Python tuple object.")
+ name="tuple",
+ obtype=tuple,
+ doc="A Python tuple object.")
pylist = StackObject(
- name="list",
- obtype=list,
- doc="A Python list object.")
+ name="list",
+ obtype=list,
+ doc="A Python list object.")
pydict = StackObject(
- name="dict",
- obtype=dict,
- doc="A Python dict object.")
+ name="dict",
+ obtype=dict,
+ doc="A Python dict object.")
+
+pyset = StackObject(
+ name="set",
+ obtype=set,
+ doc="A Python set object.")
+
+pyfrozenset = StackObject(
+ name="frozenset",
+ obtype=set,
+ doc="A Python frozenset object.")
anyobject = StackObject(
- name='any',
- obtype=object,
- doc="Any kind of object whatsoever.")
+ name='any',
+ obtype=object,
+ doc="Any kind of object whatsoever.")
markobject = StackObject(
- name="mark",
- obtype=StackObject,
- doc="""'The mark' is a unique object.
-
- Opcodes that operate on a variable number of objects
- generally don't embed the count of objects in the opcode,
- or pull it off the stack. Instead the MARK opcode is used
- to push a special marker object on the stack, and then
- some other opcodes grab all the objects from the top of
- the stack down to (but not including) the topmost marker
- object.
- """)
+ name="mark",
+ obtype=StackObject,
+ doc="""'The mark' is a unique object.
+
+Opcodes that operate on a variable number of objects
+generally don't embed the count of objects in the opcode,
+or pull it off the stack. Instead the MARK opcode is used
+to push a special marker object on the stack, and then
+some other opcodes grab all the objects from the top of
+the stack down to (but not including) the topmost marker
+object.
+""")
stackslice = StackObject(
- name="stackslice",
- obtype=StackObject,
- doc="""An object representing a contiguous slice of the stack.
+ name="stackslice",
+ obtype=StackObject,
+ doc="""An object representing a contiguous slice of the stack.
- This is used in conjunction with markobject, to represent all
- of the stack following the topmost markobject. For example,
- the POP_MARK opcode changes the stack from
+This is used in conjunction with markobject, to represent all
+of the stack following the topmost markobject. For example,
+the POP_MARK opcode changes the stack from
- [..., markobject, stackslice]
- to
- [...]
+ [..., markobject, stackslice]
+to
+ [...]
- No matter how many object are on the stack after the topmost
- markobject, POP_MARK gets rid of all of them (including the
- topmost markobject too).
- """)
+No matter how many object are on the stack after the topmost
+markobject, POP_MARK gets rid of all of them (including the
+topmost markobject too).
+""")
##############################################################################
# Descriptors for pickle opcodes.
@@ -1032,7 +1206,7 @@ opcodes = [
code='L',
arg=decimalnl_long,
stack_before=[],
- stack_after=[pylong],
+ stack_after=[pyint],
proto=0,
doc="""Push a long integer.
@@ -1050,7 +1224,7 @@ opcodes = [
code='\x8a',
arg=long1,
stack_before=[],
- stack_after=[pylong],
+ stack_after=[pyint],
proto=2,
doc="""Long integer using one-byte length.
@@ -1061,7 +1235,7 @@ opcodes = [
code='\x8b',
arg=long4,
stack_before=[],
- stack_after=[pylong],
+ stack_after=[pyint],
proto=2,
doc="""Long integer using found-byte length.
@@ -1074,45 +1248,50 @@ opcodes = [
code='S',
arg=stringnl,
stack_before=[],
- stack_after=[pystring],
+ stack_after=[pybytes_or_str],
proto=0,
doc="""Push a Python string object.
The argument is a repr-style string, with bracketing quote characters,
and perhaps embedded escapes. The argument extends until the next
- newline character. (Actually, they are decoded into a str instance
+ newline character. These are usually decoded into a str instance
using the encoding given to the Unpickler constructor. or the default,
- 'ASCII'.)
+ 'ASCII'. If the encoding given was 'bytes' however, they will be
+ decoded as bytes object instead.
"""),
I(name='BINSTRING',
code='T',
arg=string4,
stack_before=[],
- stack_after=[pystring],
+ stack_after=[pybytes_or_str],
proto=1,
doc="""Push a Python string object.
- There are two arguments: the first is a 4-byte little-endian signed int
- giving the number of bytes in the string, and the second is that many
- bytes, which are taken literally as the string content. (Actually,
- they are decoded into a str instance using the encoding given to the
- Unpickler constructor. or the default, 'ASCII'.)
+ There are two arguments: the first is a 4-byte little-endian
+ signed int giving the number of bytes in the string, and the
+ second is that many bytes, which are taken literally as the string
+ content. These are usually decoded into a str instance using the
+ encoding given to the Unpickler constructor. or the default,
+ 'ASCII'. If the encoding given was 'bytes' however, they will be
+ decoded as bytes object instead.
"""),
I(name='SHORT_BINSTRING',
code='U',
arg=string1,
stack_before=[],
- stack_after=[pystring],
+ stack_after=[pybytes_or_str],
proto=1,
doc="""Push a Python string object.
- There are two arguments: the first is a 1-byte unsigned int giving
- the number of bytes in the string, and the second is that many bytes,
- which are taken literally as the string content. (Actually, they
- are decoded into a str instance using the encoding given to the
- Unpickler constructor. or the default, 'ASCII'.)
+ There are two arguments: the first is a 1-byte unsigned int giving
+ the number of bytes in the string, and the second is that many
+ bytes, which are taken literally as the string content. These are
+ usually decoded into a str instance using the encoding given to
+ the Unpickler constructor. or the default, 'ASCII'. If the
+ encoding given was 'bytes' however, they will be decoded as bytes
+ object instead.
"""),
# Bytes (protocol 3 only; older protocols don't support bytes at all)
@@ -1143,6 +1322,19 @@ opcodes = [
literally as the string content.
"""),
+ I(name='BINBYTES8',
+ code='\x8e',
+ arg=bytes8,
+ stack_before=[],
+ stack_after=[pybytes],
+ proto=4,
+ doc="""Push a Python bytes object.
+
+ There are two arguments: the first is a 8-byte unsigned int giving
+ the number of bytes in the string, and the second is that many bytes,
+ which are taken literally as the string content.
+ """),
+
# Ways to spell None.
I(name='NONE',
@@ -1191,6 +1383,19 @@ opcodes = [
until the next newline character.
"""),
+ I(name='SHORT_BINUNICODE',
+ code='\x8c',
+ arg=unicodestring1,
+ stack_before=[],
+ stack_after=[pyunicode],
+ proto=4,
+ doc="""Push a Python Unicode string object.
+
+ There are two arguments: the first is a 1-byte little-endian signed int
+ giving the number of bytes in the string. The second is that many
+ bytes, and is the UTF-8 encoding of the Unicode string.
+ """),
+
I(name='BINUNICODE',
code='X',
arg=unicodestring4,
@@ -1204,6 +1409,19 @@ opcodes = [
bytes, and is the UTF-8 encoding of the Unicode string.
"""),
+ I(name='BINUNICODE8',
+ code='\x8d',
+ arg=unicodestring8,
+ stack_before=[],
+ stack_after=[pyunicode],
+ proto=4,
+ doc="""Push a Python Unicode string object.
+
+ There are two arguments: the first is a 8-byte little-endian signed int
+ giving the number of bytes in the string. The second is that many
+ bytes, and is the UTF-8 encoding of the Unicode string.
+ """),
+
# Ways to spell floats.
I(name='FLOAT',
@@ -1429,6 +1647,54 @@ opcodes = [
1, 2, ..., n, and in that order.
"""),
+ # Ways to build sets
+
+ I(name='EMPTY_SET',
+ code='\x8f',
+ arg=None,
+ stack_before=[],
+ stack_after=[pyset],
+ proto=4,
+ doc="Push an empty set."),
+
+ I(name='ADDITEMS',
+ code='\x90',
+ arg=None,
+ stack_before=[pyset, markobject, stackslice],
+ stack_after=[pyset],
+ proto=4,
+ doc="""Add an arbitrary number of items to an existing set.
+
+ The slice of the stack following the topmost markobject is taken as
+ a sequence of items, added to the set immediately under the topmost
+ markobject. Everything at and after the topmost markobject is popped,
+ leaving the mutated set at the top of the stack.
+
+ Stack before: ... pyset markobject item_1 ... item_n
+ Stack after: ... pyset
+
+ where pyset has been modified via pyset.add(item_i) = item_i for i in
+ 1, 2, ..., n, and in that order.
+ """),
+
+ # Way to build frozensets
+
+ I(name='FROZENSET',
+ code='\x91',
+ arg=None,
+ stack_before=[markobject, stackslice],
+ stack_after=[pyfrozenset],
+ proto=4,
+ doc="""Build a frozenset out of the topmost slice, after markobject.
+
+ All the stack entries following the topmost markobject are placed into
+ a single Python frozenset, which single frozenset object replaces all
+ of the stack from the topmost markobject onward. For example,
+
+ Stack before: ... markobject 1 2 3
+ Stack after: ... frozenset({1, 2, 3})
+ """),
+
# Stack manipulation.
I(name='POP',
@@ -1550,6 +1816,18 @@ opcodes = [
unsigned little-endian integer following.
"""),
+ I(name='MEMOIZE',
+ code='\x94',
+ arg=None,
+ stack_before=[anyobject],
+ stack_after=[anyobject],
+ proto=4,
+ doc="""Store the stack top into the memo. The stack is not popped.
+
+ The index of the memo location to write is the number of
+ elements currently present in the memo.
+ """),
+
# Access the extension registry (predefined objects). Akin to the GET
# family.
@@ -1615,6 +1893,15 @@ opcodes = [
stack, so unpickling subclasses can override this form of lookup.
"""),
+ I(name='STACK_GLOBAL',
+ code='\x93',
+ arg=None,
+ stack_before=[pyunicode, pyunicode],
+ stack_after=[anyobject],
+ proto=0,
+ doc="""Push a global object (module.attr) on the stack.
+ """),
+
# Ways to build objects of classes pickle doesn't know about directly
# (user-defined classes). I despair of documenting this accurately
# and comprehensibly -- you really have to read the pickle code to
@@ -1771,6 +2058,21 @@ opcodes = [
onto the stack.
"""),
+ I(name='NEWOBJ_EX',
+ code='\x92',
+ arg=None,
+ stack_before=[anyobject, anyobject, anyobject],
+ stack_after=[anyobject],
+ proto=4,
+ doc="""Build an object instance.
+
+ The stack before should be thought of as containing a class
+ object followed by an argument tuple and by a keyword argument dict
+ (the dict being the stack top). Call these cls and args. They are
+ popped off the stack, and the value returned by
+ cls.__new__(cls, *args, *kwargs) is pushed back onto the stack.
+ """),
+
# Machine control.
I(name='PROTO',
@@ -1798,6 +2100,20 @@ opcodes = [
empty then.
"""),
+ # Framing support.
+
+ I(name='FRAME',
+ code='\x95',
+ arg=uint8,
+ stack_before=[],
+ stack_after=[],
+ proto=4,
+ doc="""Indicate the beginning of a new frame.
+
+ The unpickler may use this opcode to safely prefetch data from its
+ underlying stream.
+ """),
+
# Ways to deal with persistent IDs.
I(name='PERSID',
@@ -1904,6 +2220,38 @@ del assure_pickle_consistency
##############################################################################
# A pickle opcode generator.
+def _genops(data, yield_end_pos=False):
+ if isinstance(data, bytes_types):
+ data = io.BytesIO(data)
+
+ if hasattr(data, "tell"):
+ getpos = data.tell
+ else:
+ getpos = lambda: None
+
+ while True:
+ pos = getpos()
+ code = data.read(1)
+ opcode = code2op.get(code.decode("latin-1"))
+ if opcode is None:
+ if code == b"":
+ raise ValueError("pickle exhausted before seeing STOP")
+ else:
+ raise ValueError("at position %s, opcode %r unknown" % (
+ "<unknown>" if pos is None else pos,
+ code))
+ if opcode.arg is None:
+ arg = None
+ else:
+ arg = opcode.arg.reader(data)
+ if yield_end_pos:
+ yield opcode, arg, pos, getpos()
+ else:
+ yield opcode, arg, pos
+ if code == b'.':
+ assert opcode.name == 'STOP'
+ break
+
def genops(pickle):
"""Generate all the opcodes in a pickle.
@@ -1927,62 +2275,48 @@ def genops(pickle):
used. Else (the pickle doesn't have a tell(), and it's not obvious how
to query its current position) pos is None.
"""
-
- if isinstance(pickle, bytes_types):
- import io
- pickle = io.BytesIO(pickle)
-
- if hasattr(pickle, "tell"):
- getpos = pickle.tell
- else:
- getpos = lambda: None
-
- while True:
- pos = getpos()
- code = pickle.read(1)
- opcode = code2op.get(code.decode("latin-1"))
- if opcode is None:
- if code == b"":
- raise ValueError("pickle exhausted before seeing STOP")
- else:
- raise ValueError("at position %s, opcode %r unknown" % (
- pos is None and "<unknown>" or pos,
- code))
- if opcode.arg is None:
- arg = None
- else:
- arg = opcode.arg.reader(pickle)
- yield opcode, arg, pos
- if code == b'.':
- assert opcode.name == 'STOP'
- break
+ return _genops(pickle)
##############################################################################
# A pickle optimizer.
def optimize(p):
'Optimize a pickle string by removing unused PUT opcodes'
- gets = set() # set of args used by a GET opcode
- puts = [] # (arg, startpos, stoppos) for the PUT opcodes
- prevpos = None # set to pos if previous opcode was a PUT
- for opcode, arg, pos in genops(p):
- if prevpos is not None:
- puts.append((prevarg, prevpos, pos))
- prevpos = None
+ not_a_put = object()
+ gets = { not_a_put } # set of args used by a GET opcode
+ opcodes = [] # (startpos, stoppos, putid)
+ proto = 0
+ for opcode, arg, pos, end_pos in _genops(p, yield_end_pos=True):
if 'PUT' in opcode.name:
- prevarg, prevpos = arg, pos
- elif 'GET' in opcode.name:
- gets.add(arg)
-
- # Copy the pickle string except for PUTS without a corresponding GET
- s = []
- i = 0
- for arg, start, stop in puts:
- j = stop if (arg in gets) else start
- s.append(p[i:j])
- i = stop
- s.append(p[i:])
- return b''.join(s)
+ opcodes.append((pos, end_pos, arg))
+ elif 'FRAME' in opcode.name:
+ pass
+ else:
+ if 'GET' in opcode.name:
+ gets.add(arg)
+ elif opcode.name == 'PROTO':
+ assert pos == 0, pos
+ proto = arg
+ opcodes.append((pos, end_pos, not_a_put))
+ prevpos, prevarg = pos, None
+
+ # Copy the opcodes except for PUTS without a corresponding GET
+ out = io.BytesIO()
+ opcodes = iter(opcodes)
+ if proto >= 2:
+ # Write the PROTO header before any framing
+ start, stop, _ = next(opcodes)
+ out.write(p[start:stop])
+ buf = pickle._Framer(out.write)
+ if proto >= 4:
+ buf.start_framing()
+ for start, stop, putid in opcodes:
+ if putid in gets:
+ buf.commit_frame()
+ buf.write(p[start:stop])
+ if proto >= 4:
+ buf.end_framing()
+ return out.getvalue()
##############################################################################
# A symbolic pickle disassembler.
@@ -2082,17 +2416,20 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0):
errormsg = markmsg = "no MARK exists on stack"
# Check for correct memo usage.
- if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT"):
- assert arg is not None
- if arg in memo:
+ if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"):
+ if opcode.name == "MEMOIZE":
+ memo_idx = len(memo)
+ else:
+ assert arg is not None
+ memo_idx = arg
+ if memo_idx in memo:
errormsg = "memo key %r already defined" % arg
elif not stack:
errormsg = "stack is empty -- can't store into memo"
elif stack[-1] is markobject:
errormsg = "can't store markobject in the memo"
else:
- memo[arg] = stack[-1]
-
+ memo[memo_idx] = stack[-1]
elif opcode.name in ("GET", "BINGET", "LONG_BINGET"):
if arg in memo:
assert len(after) == 1
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index 20e6498..58cccdc 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -1,12 +1,14 @@
"""Utilities to support packages."""
-import os
-import sys
+from functools import singledispatch as simplegeneric
import importlib
-import imp
+import importlib.util
+import importlib.machinery
+import os
import os.path
-from warnings import warn
+import sys
from types import ModuleType
+import warnings
__all__ = [
'get_importer', 'iter_importers', 'get_loader', 'find_loader',
@@ -14,59 +16,34 @@ __all__ = [
'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]
+
+def _get_spec(finder, name):
+ """Return the finder-specific module spec."""
+ # Works with legacy finders.
+ try:
+ find_spec = finder.find_spec
+ except AttributeError:
+ loader = finder.find_module(name)
+ if loader is None:
+ return None
+ return importlib.util.spec_from_loader(name, loader)
+ else:
+ return find_spec(name)
+
+
def read_code(stream):
# This helper is needed in order for the PEP 302 emulation to
# correctly handle compiled files
import marshal
magic = stream.read(4)
- if magic != imp.get_magic():
+ if magic != importlib.util.MAGIC_NUMBER:
return None
stream.read(8) # Skip timestamp and size
return marshal.load(stream)
-def simplegeneric(func):
- """Make a trivial single-dispatch generic function"""
- registry = {}
- def wrapper(*args, **kw):
- ob = args[0]
- try:
- cls = ob.__class__
- except AttributeError:
- cls = type(ob)
- try:
- mro = cls.__mro__
- except AttributeError:
- try:
- class cls(cls, object):
- pass
- mro = cls.__mro__[1:]
- except TypeError:
- mro = object, # must be an ExtensionClass or some such :(
- for t in mro:
- if t in registry:
- return registry[t](*args, **kw)
- else:
- return func(*args, **kw)
- try:
- wrapper.__name__ = func.__name__
- except (TypeError, AttributeError):
- pass # Python 2.3 doesn't allow functions to be renamed
-
- def register(typ, func=None):
- if func is None:
- return lambda f: register(typ, f)
- registry[typ] = func
- return func
-
- wrapper.__dict__ = func.__dict__
- wrapper.__doc__ = func.__doc__
- wrapper.register = register
- return wrapper
-
-
def walk_packages(path=None, prefix='', onerror=None):
"""Yields (module_loader, name, ispkg) for all modules recursively
on path, or, if path is None, all accessible modules.
@@ -121,8 +98,7 @@ def walk_packages(path=None, prefix='', onerror=None):
# don't traverse path items we've seen before
path = [p for p in path if not seen(p)]
- for item in walk_packages(path, name+'.', onerror):
- yield item
+ yield from walk_packages(path, name+'.', onerror)
def iter_modules(path=None, prefix=''):
@@ -149,13 +125,12 @@ def iter_modules(path=None, prefix=''):
yield i, name, ispkg
-#@simplegeneric
+@simplegeneric
def iter_importer_modules(importer, prefix=''):
if not hasattr(importer, 'iter_modules'):
return []
return importer.iter_modules(prefix)
-iter_importer_modules = simplegeneric(iter_importer_modules)
# Implement a file walker for the normal importlib path hook
def _iter_file_finder_modules(importer, prefix=''):
@@ -201,6 +176,13 @@ def _iter_file_finder_modules(importer, prefix=''):
iter_importer_modules.register(
importlib.machinery.FileFinder, _iter_file_finder_modules)
+
+def _import_imp():
+ global imp
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', PendingDeprecationWarning)
+ imp = importlib.import_module('imp')
+
class ImpImporter:
"""PEP 302 Importer that wraps Python's "classic" import algorithm
@@ -213,8 +195,10 @@ class ImpImporter:
"""
def __init__(self, path=None):
- warn("This emulation is deprecated, use 'importlib' instead",
+ global imp
+ warnings.warn("This emulation is deprecated, use 'importlib' instead",
DeprecationWarning)
+ _import_imp()
self.path = path
def find_module(self, fullname, path=None):
@@ -279,8 +263,9 @@ class ImpLoader:
code = source = None
def __init__(self, fullname, file, filename, etc):
- warn("This emulation is deprecated, use 'importlib' instead",
- DeprecationWarning)
+ warnings.warn("This emulation is deprecated, use 'importlib' instead",
+ DeprecationWarning)
+ _import_imp()
self.file = file
self.filename = filename
self.fullname = fullname
@@ -350,16 +335,16 @@ class ImpLoader:
self.file.close()
elif mod_type==imp.PY_COMPILED:
if os.path.exists(self.filename[:-1]):
- f = open(self.filename[:-1], 'r')
- self.source = f.read()
- f.close()
+ with open(self.filename[:-1], 'r') as f:
+ self.source = f.read()
elif mod_type==imp.PKG_DIRECTORY:
self.source = self._get_delegate().get_source()
return self.source
-
def _get_delegate(self):
- return ImpImporter(self.filename).find_module('__init__')
+ finder = ImpImporter(self.filename)
+ spec = _get_spec(finder, '__init__')
+ return spec.loader
def get_filename(self, fullname=None):
fullname = self._fix_name(fullname)
@@ -456,12 +441,12 @@ def iter_importers(fullname=""):
if path is None:
return
else:
- for importer in sys.meta_path:
- yield importer
+ yield from sys.meta_path
path = sys.path
for item in path:
yield get_importer(item)
+
def get_loader(module_or_name):
"""Get a PEP 302 "loader" object for module_or_name
@@ -485,29 +470,22 @@ def get_loader(module_or_name):
def find_loader(fullname):
"""Find a PEP 302 "loader" object for fullname
- This is s convenience wrapper around :func:`importlib.find_loader` that
- sets the *path* argument correctly when searching for submodules, and
- also ensures parent packages (if any) are imported before searching for
- submodules.
+ This is a backwards compatibility wrapper around
+ importlib.util.find_spec that converts most failures to ImportError
+ and only returns the loader rather than the full spec
"""
if fullname.startswith('.'):
msg = "Relative module name {!r} not supported".format(fullname)
raise ImportError(msg)
- path = None
- pkg_name = fullname.rpartition(".")[0]
- if pkg_name:
- pkg = importlib.import_module(pkg_name)
- path = getattr(pkg, "__path__", None)
- if path is None:
- return None
try:
- return importlib.find_loader(fullname, path)
+ spec = importlib.util.find_spec(fullname)
except (ImportError, AttributeError, TypeError, ValueError) as ex:
# This hack fixes an impedance mismatch between pkgutil and
# importlib, where the latter raises other errors for cases where
# pkgutil previously raised ImportError
msg = "Error while finding loader for {!r} ({}: {})"
raise ImportError(msg.format(fullname, type(ex), ex)) from ex
+ return spec.loader
def extend_path(path, name):
@@ -569,13 +547,14 @@ def extend_path(path, name):
finder = get_importer(dir)
if finder is not None:
+ portions = []
+ if hasattr(finder, 'find_spec'):
+ spec = finder.find_spec(final_name)
+ if spec is not None:
+ portions = spec.submodule_search_locations or []
# Is this finder PEP 420 compliant?
- if hasattr(finder, 'find_loader'):
- loader, portions = finder.find_loader(final_name)
- else:
- # No, no need to call it
- loader = None
- portions = []
+ elif hasattr(finder, 'find_loader'):
+ _, portions = finder.find_loader(final_name)
for portion in portions:
# XXX This may still add duplicate entries to path on
@@ -589,19 +568,20 @@ def extend_path(path, name):
if os.path.isfile(pkgfile):
try:
f = open(pkgfile)
- except IOError as msg:
+ except OSError as msg:
sys.stderr.write("Can't open %s: %s\n" %
(pkgfile, msg))
else:
- for line in f:
- line = line.rstrip('\n')
- if not line or line.startswith('#'):
- continue
- path.append(line) # Don't check for existence!
- f.close()
+ with f:
+ for line in f:
+ line = line.rstrip('\n')
+ if not line or line.startswith('#'):
+ continue
+ path.append(line) # Don't check for existence!
return path
+
def get_data(package, resource):
"""Get a resource from a package.
@@ -624,10 +604,15 @@ def get_data(package, resource):
which does not support get_data(), then None is returned.
"""
- loader = get_loader(package)
+ spec = importlib.util.find_spec(package)
+ if spec is None:
+ return None
+ loader = spec.loader
if loader is None or not hasattr(loader, 'get_data'):
return None
- mod = sys.modules.get(package) or loader.load_module(package)
+ # XXX needs test
+ mod = (sys.modules.get(package) or
+ importlib._bootstrap._SpecMethods(spec).load())
if mod is None or not hasattr(mod, '__file__'):
return None
diff --git a/Lib/plat-os2emx/IN.py b/Lib/plat-os2emx/IN.py
deleted file mode 100644
index 753ae24..0000000
--- a/Lib/plat-os2emx/IN.py
+++ /dev/null
@@ -1,82 +0,0 @@
-# Generated by h2py from f:/emx/include/netinet/in.h
-
-# Included from sys/param.h
-PAGE_SIZE = 0x1000
-HZ = 100
-MAXNAMLEN = 260
-MAXPATHLEN = 260
-def htonl(X): return _swapl(X)
-
-def ntohl(X): return _swapl(X)
-
-def htons(X): return _swaps(X)
-
-def ntohs(X): return _swaps(X)
-
-IPPROTO_IP = 0
-IPPROTO_ICMP = 1
-IPPROTO_IGMP = 2
-IPPROTO_GGP = 3
-IPPROTO_TCP = 6
-IPPROTO_EGP = 8
-IPPROTO_PUP = 12
-IPPROTO_UDP = 17
-IPPROTO_IDP = 22
-IPPROTO_TP = 29
-IPPROTO_EON = 80
-IPPROTO_RAW = 255
-IPPROTO_MAX = 256
-IPPORT_RESERVED = 1024
-IPPORT_USERRESERVED = 5000
-def IN_CLASSA(i): return (((int)(i) & 0x80000000) == 0)
-
-IN_CLASSA_NET = 0xff000000
-IN_CLASSA_NSHIFT = 24
-IN_CLASSA_HOST = 0x00ffffff
-IN_CLASSA_MAX = 128
-def IN_CLASSB(i): return (((int)(i) & 0xc0000000) == 0x80000000)
-
-IN_CLASSB_NET = 0xffff0000
-IN_CLASSB_NSHIFT = 16
-IN_CLASSB_HOST = 0x0000ffff
-IN_CLASSB_MAX = 65536
-def IN_CLASSC(i): return (((int)(i) & 0xe0000000) == 0xc0000000)
-
-IN_CLASSC_NET = 0xffffff00
-IN_CLASSC_NSHIFT = 8
-IN_CLASSC_HOST = 0x000000ff
-def IN_CLASSD(i): return (((int)(i) & 0xf0000000) == 0xe0000000)
-
-IN_CLASSD_NET = 0xf0000000
-IN_CLASSD_NSHIFT = 28
-IN_CLASSD_HOST = 0x0fffffff
-def IN_MULTICAST(i): return IN_CLASSD(i)
-
-def IN_EXPERIMENTAL(i): return (((int)(i) & 0xe0000000) == 0xe0000000)
-
-def IN_BADCLASS(i): return (((int)(i) & 0xf0000000) == 0xf0000000)
-
-INADDR_ANY = 0x00000000
-INADDR_LOOPBACK = 0x7f000001
-INADDR_BROADCAST = 0xffffffff
-INADDR_NONE = 0xffffffff
-INADDR_UNSPEC_GROUP = 0xe0000000
-INADDR_ALLHOSTS_GROUP = 0xe0000001
-INADDR_MAX_LOCAL_GROUP = 0xe00000ff
-IN_LOOPBACKNET = 127
-IP_OPTIONS = 1
-IP_MULTICAST_IF = 2
-IP_MULTICAST_TTL = 3
-IP_MULTICAST_LOOP = 4
-IP_ADD_MEMBERSHIP = 5
-IP_DROP_MEMBERSHIP = 6
-IP_HDRINCL = 2
-IP_TOS = 3
-IP_TTL = 4
-IP_RECVOPTS = 5
-IP_RECVRETOPTS = 6
-IP_RECVDSTADDR = 7
-IP_RETOPTS = 8
-IP_DEFAULT_MULTICAST_TTL = 1
-IP_DEFAULT_MULTICAST_LOOP = 1
-IP_MAX_MEMBERSHIPS = 20
diff --git a/Lib/plat-os2emx/SOCKET.py b/Lib/plat-os2emx/SOCKET.py
deleted file mode 100644
index dac594a..0000000
--- a/Lib/plat-os2emx/SOCKET.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Generated by h2py from f:/emx/include/sys/socket.h
-
-# Included from sys/types.h
-FD_SETSIZE = 256
-
-# Included from sys/uio.h
-FREAD = 1
-FWRITE = 2
-SOCK_STREAM = 1
-SOCK_DGRAM = 2
-SOCK_RAW = 3
-SOCK_RDM = 4
-SOCK_SEQPACKET = 5
-SO_DEBUG = 0x0001
-SO_ACCEPTCONN = 0x0002
-SO_REUSEADDR = 0x0004
-SO_KEEPALIVE = 0x0008
-SO_DONTROUTE = 0x0010
-SO_BROADCAST = 0x0020
-SO_USELOOPBACK = 0x0040
-SO_LINGER = 0x0080
-SO_OOBINLINE = 0x0100
-SO_L_BROADCAST = 0x0200
-SO_RCV_SHUTDOWN = 0x0400
-SO_SND_SHUTDOWN = 0x0800
-SO_SNDBUF = 0x1001
-SO_RCVBUF = 0x1002
-SO_SNDLOWAT = 0x1003
-SO_RCVLOWAT = 0x1004
-SO_SNDTIMEO = 0x1005
-SO_RCVTIMEO = 0x1006
-SO_ERROR = 0x1007
-SO_TYPE = 0x1008
-SO_OPTIONS = 0x1010
-SOL_SOCKET = 0xffff
-AF_UNSPEC = 0
-AF_UNIX = 1
-AF_INET = 2
-AF_IMPLINK = 3
-AF_PUP = 4
-AF_CHAOS = 5
-AF_NS = 6
-AF_NBS = 7
-AF_ISO = 7
-AF_OSI = AF_ISO
-AF_ECMA = 8
-AF_DATAKIT = 9
-AF_CCITT = 10
-AF_SNA = 11
-AF_DECnet = 12
-AF_DLI = 13
-AF_LAT = 14
-AF_HYLINK = 15
-AF_APPLETALK = 16
-AF_NB = 17
-AF_NETBIOS = AF_NB
-AF_OS2 = AF_UNIX
-AF_MAX = 18
-PF_UNSPEC = AF_UNSPEC
-PF_UNIX = AF_UNIX
-PF_INET = AF_INET
-PF_IMPLINK = AF_IMPLINK
-PF_PUP = AF_PUP
-PF_CHAOS = AF_CHAOS
-PF_NS = AF_NS
-PF_NBS = AF_NBS
-PF_ISO = AF_ISO
-PF_OSI = AF_ISO
-PF_ECMA = AF_ECMA
-PF_DATAKIT = AF_DATAKIT
-PF_CCITT = AF_CCITT
-PF_SNA = AF_SNA
-PF_DECnet = AF_DECnet
-PF_DLI = AF_DLI
-PF_LAT = AF_LAT
-PF_HYLINK = AF_HYLINK
-PF_APPLETALK = AF_APPLETALK
-PF_NB = AF_NB
-PF_NETBIOS = AF_NB
-PF_OS2 = AF_UNIX
-PF_MAX = AF_MAX
-SOMAXCONN = 5
-MSG_OOB = 0x1
-MSG_PEEK = 0x2
-MSG_DONTROUTE = 0x4
-MSG_EOR = 0x8
-MSG_TRUNC = 0x10
-MSG_CTRUNC = 0x20
-MSG_WAITALL = 0x40
-MSG_MAXIOVLEN = 16
-SCM_RIGHTS = 0x01
-MT_FREE = 0
-MT_DATA = 1
-MT_HEADER = 2
-MT_SOCKET = 3
-MT_PCB = 4
-MT_RTABLE = 5
-MT_HTABLE = 6
-MT_ATABLE = 7
-MT_SONAME = 8
-MT_ZOMBIE = 9
-MT_SOOPTS = 10
-MT_FTABLE = 11
-MT_RIGHTS = 12
-MT_IFADDR = 13
-MAXSOCKETS = 2048
diff --git a/Lib/plat-os2emx/_emx_link.py b/Lib/plat-os2emx/_emx_link.py
deleted file mode 100644
index 01e6b54..0000000
--- a/Lib/plat-os2emx/_emx_link.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# _emx_link.py
-
-# Written by Andrew I MacIntyre, December 2002.
-
-"""_emx_link.py is a simplistic emulation of the Unix link(2) library routine
-for creating so-called hard links. It is intended to be imported into
-the os module in place of the unimplemented (on OS/2) Posix link()
-function (os.link()).
-
-We do this on OS/2 by implementing a file copy, with link(2) semantics:-
- - the target cannot already exist;
- - we hope that the actual file open (if successful) is actually
- atomic...
-
-Limitations of this approach/implementation include:-
- - no support for correct link counts (EMX stat(target).st_nlink
- is always 1);
- - thread safety undefined;
- - default file permissions (r+w) used, can't be over-ridden;
- - implemented in Python so comparatively slow, especially for large
- source files;
- - need sufficient free disk space to store the copy.
-
-Behaviour:-
- - any exception should propagate to the caller;
- - want target to be an exact copy of the source, so use binary mode;
- - returns None, same as os.link() which is implemented in posixmodule.c;
- - target removed in the event of a failure where possible;
- - given the motivation to write this emulation came from trying to
- support a Unix resource lock implementation, where minimal overhead
- during creation of the target is desirable and the files are small,
- we read a source block before attempting to create the target so that
- we're ready to immediately write some data into it.
-"""
-
-import os
-import errno
-
-__all__ = ['link']
-
-def link(source, target):
- """link(source, target) -> None
-
- Attempt to hard link the source file to the target file name.
- On OS/2, this creates a complete copy of the source file.
- """
-
- s = os.open(source, os.O_RDONLY | os.O_BINARY)
- if os.isatty(s):
- raise OSError(errno.EXDEV, 'Cross-device link')
- data = os.read(s, 1024)
-
- try:
- t = os.open(target, os.O_WRONLY | os.O_BINARY | os.O_CREAT | os.O_EXCL)
- except OSError:
- os.close(s)
- raise
-
- try:
- while data:
- os.write(t, data)
- data = os.read(s, 1024)
- except OSError:
- os.close(s)
- os.close(t)
- os.unlink(target)
- raise
-
- os.close(s)
- os.close(t)
-
-if __name__ == '__main__':
- import sys
- try:
- link(sys.argv[1], sys.argv[2])
- except IndexError:
- print('Usage: emx_link <source> <target>')
- except OSError:
- print('emx_link: %s' % str(sys.exc_info()[1]))
diff --git a/Lib/plat-os2emx/grp.py b/Lib/plat-os2emx/grp.py
deleted file mode 100644
index ee63ef8..0000000
--- a/Lib/plat-os2emx/grp.py
+++ /dev/null
@@ -1,182 +0,0 @@
-# this module is an OS/2 oriented replacement for the grp standard
-# extension module.
-
-# written by Andrew MacIntyre, April 2001.
-# updated July 2003, adding field accessor support
-
-# note that this implementation checks whether ":" or ";" as used as
-# the field separator character.
-
-"""Replacement for grp standard extension module, intended for use on
-OS/2 and similar systems which don't normally have an /etc/group file.
-
-The standard Unix group database is an ASCII text file with 4 fields per
-record (line), separated by a colon:
- - group name (string)
- - group password (optional encrypted string)
- - group id (integer)
- - group members (comma delimited list of userids, with no spaces)
-
-Note that members are only included in the group file for groups that
-aren't their primary groups.
-(see the section 8.2 of the Python Library Reference)
-
-This implementation differs from the standard Unix implementation by
-allowing use of the platform's native path separator character - ';' on OS/2,
-DOS and MS-Windows - as the field separator in addition to the Unix
-standard ":".
-
-The module looks for the group database at the following locations
-(in order first to last):
- - ${ETC_GROUP} (or %ETC_GROUP%)
- - ${ETC}/group (or %ETC%/group)
- - ${PYTHONHOME}/Etc/group (or %PYTHONHOME%/Etc/group)
-
-Classes
--------
-
-None
-
-Functions
----------
-
-getgrgid(gid) - return the record for group-id gid as a 4-tuple
-
-getgrnam(name) - return the record for group 'name' as a 4-tuple
-
-getgrall() - return a list of 4-tuples, each tuple being one record
- (NOTE: the order is arbitrary)
-
-Attributes
-----------
-
-group_file - the path of the group database file
-
-"""
-
-import os
-
-# try and find the group file
-__group_path = []
-if 'ETC_GROUP' in os.environ:
- __group_path.append(os.environ['ETC_GROUP'])
-if 'ETC' in os.environ:
- __group_path.append('%s/group' % os.environ['ETC'])
-if 'PYTHONHOME' in os.environ:
- __group_path.append('%s/Etc/group' % os.environ['PYTHONHOME'])
-
-group_file = None
-for __i in __group_path:
- try:
- __f = open(__i, 'r')
- __f.close()
- group_file = __i
- break
- except:
- pass
-
-# decide what field separator we can try to use - Unix standard, with
-# the platform's path separator as an option. No special field conversion
-# handlers are required for the group file.
-__field_sep = [':']
-if os.pathsep:
- if os.pathsep != ':':
- __field_sep.append(os.pathsep)
-
-# helper routine to identify which separator character is in use
-def __get_field_sep(record):
- fs = None
- for c in __field_sep:
- # there should be 3 delimiter characters (for 4 fields)
- if record.count(c) == 3:
- fs = c
- break
- if fs:
- return fs
- else:
- raise KeyError('>> group database fields not delimited <<')
-
-# class to match the new record field name accessors.
-# the resulting object is intended to behave like a read-only tuple,
-# with each member also accessible by a field name.
-class Group:
- def __init__(self, name, passwd, gid, mem):
- self.__dict__['gr_name'] = name
- self.__dict__['gr_passwd'] = passwd
- self.__dict__['gr_gid'] = gid
- self.__dict__['gr_mem'] = mem
- self.__dict__['_record'] = (self.gr_name, self.gr_passwd,
- self.gr_gid, self.gr_mem)
-
- def __len__(self):
- return 4
-
- def __getitem__(self, key):
- return self._record[key]
-
- def __setattr__(self, name, value):
- raise AttributeError('attribute read-only: %s' % name)
-
- def __repr__(self):
- return str(self._record)
-
- def __cmp__(self, other):
- this = str(self._record)
- if this == other:
- return 0
- elif this < other:
- return -1
- else:
- return 1
-
-
-# read the whole file, parsing each entry into tuple form
-# with dictionaries to speed recall by GID or group name
-def __read_group_file():
- if group_file:
- group = open(group_file, 'r')
- else:
- raise KeyError('>> no group database <<')
- gidx = {}
- namx = {}
- sep = None
- while 1:
- entry = group.readline().strip()
- if len(entry) > 3:
- if sep is None:
- sep = __get_field_sep(entry)
- fields = entry.split(sep)
- fields[2] = int(fields[2])
- fields[3] = [f.strip() for f in fields[3].split(',')]
- record = Group(*fields)
- if fields[2] not in gidx:
- gidx[fields[2]] = record
- if fields[0] not in namx:
- namx[fields[0]] = record
- elif len(entry) > 0:
- pass # skip empty or malformed records
- else:
- break
- group.close()
- if len(gidx) == 0:
- raise KeyError
- return (gidx, namx)
-
-# return the group database entry by GID
-def getgrgid(gid):
- g, n = __read_group_file()
- return g[gid]
-
-# return the group database entry by group name
-def getgrnam(name):
- g, n = __read_group_file()
- return n[name]
-
-# return all the group database entries
-def getgrall():
- g, n = __read_group_file()
- return g.values()
-
-# test harness
-if __name__ == '__main__':
- getgrall()
diff --git a/Lib/plat-os2emx/pwd.py b/Lib/plat-os2emx/pwd.py
deleted file mode 100644
index 2cb077f..0000000
--- a/Lib/plat-os2emx/pwd.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# this module is an OS/2 oriented replacement for the pwd standard
-# extension module.
-
-# written by Andrew MacIntyre, April 2001.
-# updated July 2003, adding field accessor support
-
-# note that this implementation checks whether ":" or ";" as used as
-# the field separator character. Path conversions are are applied when
-# the database uses ":" as the field separator character.
-
-"""Replacement for pwd standard extension module, intended for use on
-OS/2 and similar systems which don't normally have an /etc/passwd file.
-
-The standard Unix password database is an ASCII text file with 7 fields
-per record (line), separated by a colon:
- - user name (string)
- - password (encrypted string, or "*" or "")
- - user id (integer)
- - group id (integer)
- - description (usually user's name)
- - home directory (path to user's home directory)
- - shell (path to the user's login shell)
-
-(see the section 8.1 of the Python Library Reference)
-
-This implementation differs from the standard Unix implementation by
-allowing use of the platform's native path separator character - ';' on OS/2,
-DOS and MS-Windows - as the field separator in addition to the Unix
-standard ":". Additionally, when ":" is the separator path conversions
-are applied to deal with any munging of the drive letter reference.
-
-The module looks for the password database at the following locations
-(in order first to last):
- - ${ETC_PASSWD} (or %ETC_PASSWD%)
- - ${ETC}/passwd (or %ETC%/passwd)
- - ${PYTHONHOME}/Etc/passwd (or %PYTHONHOME%/Etc/passwd)
-
-Classes
--------
-
-None
-
-Functions
----------
-
-getpwuid(uid) - return the record for user-id uid as a 7-tuple
-
-getpwnam(name) - return the record for user 'name' as a 7-tuple
-
-getpwall() - return a list of 7-tuples, each tuple being one record
- (NOTE: the order is arbitrary)
-
-Attributes
-----------
-
-passwd_file - the path of the password database file
-
-"""
-
-import os
-
-# try and find the passwd file
-__passwd_path = []
-if 'ETC_PASSWD' in os.environ:
- __passwd_path.append(os.environ['ETC_PASSWD'])
-if 'ETC' in os.environ:
- __passwd_path.append('%s/passwd' % os.environ['ETC'])
-if 'PYTHONHOME' in os.environ:
- __passwd_path.append('%s/Etc/passwd' % os.environ['PYTHONHOME'])
-
-passwd_file = None
-for __i in __passwd_path:
- try:
- __f = open(__i, 'r')
- __f.close()
- passwd_file = __i
- break
- except:
- pass
-
-# path conversion handlers
-def __nullpathconv(path):
- return path.replace(os.altsep, os.sep)
-
-def __unixpathconv(path):
- # two known drive letter variations: "x;" and "$x"
- if path[0] == '$':
- conv = path[1] + ':' + path[2:]
- elif path[1] == ';':
- conv = path[0] + ':' + path[2:]
- else:
- conv = path
- return conv.replace(os.altsep, os.sep)
-
-# decide what field separator we can try to use - Unix standard, with
-# the platform's path separator as an option. No special field conversion
-# handler is required when using the platform's path separator as field
-# separator, but are required for the home directory and shell fields when
-# using the standard Unix (":") field separator.
-__field_sep = {':': __unixpathconv}
-if os.pathsep:
- if os.pathsep != ':':
- __field_sep[os.pathsep] = __nullpathconv
-
-# helper routine to identify which separator character is in use
-def __get_field_sep(record):
- fs = None
- for c in __field_sep.keys():
- # there should be 6 delimiter characters (for 7 fields)
- if record.count(c) == 6:
- fs = c
- break
- if fs:
- return fs
- else:
- raise KeyError('>> passwd database fields not delimited <<')
-
-# class to match the new record field name accessors.
-# the resulting object is intended to behave like a read-only tuple,
-# with each member also accessible by a field name.
-class Passwd:
- def __init__(self, name, passwd, uid, gid, gecos, dir, shell):
- self.__dict__['pw_name'] = name
- self.__dict__['pw_passwd'] = passwd
- self.__dict__['pw_uid'] = uid
- self.__dict__['pw_gid'] = gid
- self.__dict__['pw_gecos'] = gecos
- self.__dict__['pw_dir'] = dir
- self.__dict__['pw_shell'] = shell
- self.__dict__['_record'] = (self.pw_name, self.pw_passwd,
- self.pw_uid, self.pw_gid,
- self.pw_gecos, self.pw_dir,
- self.pw_shell)
-
- def __len__(self):
- return 7
-
- def __getitem__(self, key):
- return self._record[key]
-
- def __setattr__(self, name, value):
- raise AttributeError('attribute read-only: %s' % name)
-
- def __repr__(self):
- return str(self._record)
-
- def __cmp__(self, other):
- this = str(self._record)
- if this == other:
- return 0
- elif this < other:
- return -1
- else:
- return 1
-
-
-# read the whole file, parsing each entry into tuple form
-# with dictionaries to speed recall by UID or passwd name
-def __read_passwd_file():
- if passwd_file:
- passwd = open(passwd_file, 'r')
- else:
- raise KeyError('>> no password database <<')
- uidx = {}
- namx = {}
- sep = None
- while True:
- entry = passwd.readline().strip()
- if len(entry) > 6:
- if sep is None:
- sep = __get_field_sep(entry)
- fields = entry.split(sep)
- for i in (2, 3):
- fields[i] = int(fields[i])
- for i in (5, 6):
- fields[i] = __field_sep[sep](fields[i])
- record = Passwd(*fields)
- if fields[2] not in uidx:
- uidx[fields[2]] = record
- if fields[0] not in namx:
- namx[fields[0]] = record
- elif len(entry) > 0:
- pass # skip empty or malformed records
- else:
- break
- passwd.close()
- if len(uidx) == 0:
- raise KeyError
- return (uidx, namx)
-
-# return the passwd database entry by UID
-def getpwuid(uid):
- u, n = __read_passwd_file()
- return u[uid]
-
-# return the passwd database entry by passwd name
-def getpwnam(name):
- u, n = __read_passwd_file()
- return n[name]
-
-# return all the passwd database entries
-def getpwall():
- u, n = __read_passwd_file()
- return n.values()
-
-# test harness
-if __name__ == '__main__':
- getpwall()
diff --git a/Lib/plat-os2emx/regen b/Lib/plat-os2emx/regen
deleted file mode 100755
index 3ecd2a8..0000000
--- a/Lib/plat-os2emx/regen
+++ /dev/null
@@ -1,7 +0,0 @@
-#! /bin/sh
-export INCLUDE=$C_INCLUDE_PATH
-set -v
-python.exe ../../Tools/scripts/h2py.py $C_INCLUDE_PATH/fcntl.h
-python.exe ../../Tools/scripts/h2py.py $C_INCLUDE_PATH/sys/socket.h
-python.exe ../../Tools/scripts/h2py.py -i '(u_long)' $C_INCLUDE_PATH/netinet/in.h
-#python.exe ../../Tools/scripts/h2py.py $C_INCLUDE_PATH/termios.h
diff --git a/Lib/platform.py b/Lib/platform.py
index 030ef2a..c4ffe95 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -122,7 +122,7 @@ try:
except AttributeError:
# os.devnull was added in Python 2.4, so emulate it for earlier
# Python versions
- if sys.platform in ('dos','win32','win16','os2'):
+ if sys.platform in ('dos', 'win32', 'win16'):
# Use the old CP/M NUL as device name
DEV_NULL = 'NUL'
else:
@@ -141,7 +141,7 @@ _libc_search = re.compile(b'(__libc_init)'
b'|'
br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII)
-def libc_ver(executable=sys.executable,lib='',version='',
+def libc_ver(executable=sys.executable, lib='', version='',
chunksize=16384):
@@ -163,12 +163,12 @@ def libc_ver(executable=sys.executable,lib='',version='',
# here to work around problems with Cygwin not being
# able to open symlinks for reading
executable = os.path.realpath(executable)
- f = open(executable,'rb')
+ f = open(executable, 'rb')
binary = f.read(chunksize)
pos = 0
while 1:
if b'libc' in binary or b'GLIBC' in binary:
- m = _libc_search.search(binary,pos)
+ m = _libc_search.search(binary, pos)
else:
m = None
if not m:
@@ -177,7 +177,7 @@ def libc_ver(executable=sys.executable,lib='',version='',
break
pos = 0
continue
- libcinit,glibc,glibcversion,so,threads,soversion = [
+ libcinit, glibc, glibcversion, so, threads, soversion = [
s.decode('latin1') if s is not None else s
for s in m.groups()]
if libcinit and not lib:
@@ -197,9 +197,9 @@ def libc_ver(executable=sys.executable,lib='',version='',
version = version + threads
pos = m.end()
f.close()
- return lib,version
+ return lib, version
-def _dist_try_harder(distname,version,id):
+def _dist_try_harder(distname, version, id):
""" Tries some special tricks to get the distribution
information in case the default method fails.
@@ -214,7 +214,7 @@ def _dist_try_harder(distname,version,id):
for line in open('/var/adm/inst-log/info'):
tv = line.split()
if len(tv) == 2:
- tag,value = tv
+ tag, value = tv
else:
continue
if tag == 'MIN_DIST_VERSION':
@@ -222,7 +222,7 @@ def _dist_try_harder(distname,version,id):
elif tag == 'DIST_IDENT':
values = value.split('-')
id = values[2]
- return distname,version,id
+ return distname, version, id
if os.path.exists('/etc/.installed'):
# Caldera OpenLinux has some infos in that file (thanks to Colin Kong)
@@ -231,7 +231,7 @@ def _dist_try_harder(distname,version,id):
if len(pkg) >= 2 and pkg[0] == 'OpenLinux':
# XXX does Caldera support non Intel platforms ? If yes,
# where can we find the needed id ?
- return 'OpenLinux',pkg[1],id
+ return 'OpenLinux', pkg[1], id
if os.path.isdir('/usr/lib/setup'):
# Check for slackware version tag file (thanks to Greg Andruk)
@@ -243,9 +243,9 @@ def _dist_try_harder(distname,version,id):
verfiles.sort()
distname = 'slackware'
version = verfiles[-1][14:]
- return distname,version,id
+ return distname, version, id
- return distname,version,id
+ return distname, version, id
_release_filename = re.compile(r'(\w+)[-_](release|version)', re.ASCII)
_lsb_release_version = re.compile(r'(.+)'
@@ -314,25 +314,25 @@ def linux_distribution(distname='', version='', id='',
distribution read from the OS is returned. Otherwise the short
name taken from supported_dists is used.
- Returns a tuple (distname,version,id) which default to the
+ Returns a tuple (distname, version, id) which default to the
args given as parameters.
"""
try:
etc = os.listdir(_UNIXCONFDIR)
- except os.error:
+ except OSError:
# Probably not a Unix system
- return distname,version,id
+ return distname, version, id
etc.sort()
for file in etc:
m = _release_filename.match(file)
if m is not None:
- _distname,dummy = m.groups()
+ _distname, dummy = m.groups()
if _distname in supported_dists:
distname = _distname
break
else:
- return _dist_try_harder(distname,version,id)
+ return _dist_try_harder(distname, version, id)
# Read the first line
with open(os.path.join(_UNIXCONFDIR, file), 'r',
@@ -350,7 +350,7 @@ def linux_distribution(distname='', version='', id='',
# To maintain backwards compatibility:
-def dist(distname='',version='',id='',
+def dist(distname='', version='', id='',
supported_dists=_supported_dists):
@@ -360,7 +360,7 @@ def dist(distname='',version='',id='',
/etc and then reverts to _dist_try_harder() in case no
suitable files are found.
- Returns a tuple (distname,version,id) which default to the
+ Returns a tuple (distname, version, id) which default to the
args given as parameters.
"""
@@ -385,11 +385,11 @@ def _norm_version(version, build=''):
if build:
l.append(build)
try:
- ints = map(int,l)
+ ints = map(int, l)
except ValueError:
strings = l
else:
- strings = list(map(str,ints))
+ strings = list(map(str, ints))
version = '.'.join(strings[:3])
return version
@@ -408,46 +408,43 @@ _ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) '
def _syscmd_ver(system='', release='', version='',
- supported_platforms=('win32','win16','dos','os2')):
+ supported_platforms=('win32', 'win16', 'dos')):
""" Tries to figure out the OS version used and returns
- a tuple (system,release,version).
+ a tuple (system, release, version).
It uses the "ver" shell command for this which is known
- to exists on Windows, DOS and OS/2. XXX Others too ?
+ to exists on Windows, DOS. XXX Others too ?
In case this fails, the given parameters are used as
defaults.
"""
if sys.platform not in supported_platforms:
- return system,release,version
+ return system, release, version
# Try some common cmd strings
- for cmd in ('ver','command /c ver','cmd /c ver'):
+ for cmd in ('ver', 'command /c ver', 'cmd /c ver'):
try:
pipe = popen(cmd)
info = pipe.read()
if pipe.close():
- raise os.error('command failed')
+ raise OSError('command failed')
# XXX How can I suppress shell errors from being written
# to stderr ?
- except os.error as why:
- #print 'Command %s failed: %s' % (cmd,why)
- continue
- except IOError as why:
- #print 'Command %s failed: %s' % (cmd,why)
+ except OSError as why:
+ #print 'Command %s failed: %s' % (cmd, why)
continue
else:
break
else:
- return system,release,version
+ return system, release, version
# Parse the output
info = info.strip()
m = _ver_output.match(info)
if m is not None:
- system,release,version = m.groups()
+ system, release, version = m.groups()
# Strip trailing dots from version and release
if release[-1] == '.':
release = release[:-1]
@@ -456,9 +453,9 @@ def _syscmd_ver(system='', release='', version='',
# Normalize the version and build strings (eliminating additional
# zeros)
version = _norm_version(version)
- return system,release,version
+ return system, release, version
-def _win32_getvalue(key,name,default=''):
+def _win32_getvalue(key, name, default=''):
""" Read a value for name from the registry key.
@@ -473,14 +470,14 @@ def _win32_getvalue(key,name,default=''):
import winreg
RegQueryValueEx = winreg.QueryValueEx
try:
- return RegQueryValueEx(key,name)
+ return RegQueryValueEx(key, name)
except:
return default
-def win32_ver(release='',version='',csd='',ptype=''):
+def win32_ver(release='', version='', csd='', ptype=''):
""" Get additional version information from the Windows Registry
- and return a tuple (version,csd,ptype) referring to version
+ and return a tuple (version, csd, ptype) referring to version
number, CSD level (service pack), and OS type (multi/single
processor).
@@ -506,7 +503,6 @@ def win32_ver(release='',version='',csd='',ptype=''):
# Import the needed APIs
try:
- import win32api
from win32api import RegQueryValueEx, RegOpenKeyEx, \
RegCloseKey, GetVersionEx
from win32con import HKEY_LOCAL_MACHINE, VER_PLATFORM_WIN32_NT, \
@@ -517,7 +513,7 @@ def win32_ver(release='',version='',csd='',ptype=''):
sys.getwindowsversion
except AttributeError:
# No emulation possible, so return the defaults...
- return release,version,csd,ptype
+ return release, version, csd, ptype
else:
# Emulation using winreg (added in Python 2.0) and
# sys.getwindowsversion() (added in Python 2.3)
@@ -535,8 +531,8 @@ def win32_ver(release='',version='',csd='',ptype=''):
# Find out the registry key and some general version infos
winver = GetVersionEx()
- maj,min,buildno,plat,csd = winver
- version = '%i.%i.%i' % (maj,min,buildno & 0xFFFF)
+ maj, min, buildno, plat, csd = winver
+ version = '%i.%i.%i' % (maj, min, buildno & 0xFFFF)
if hasattr(winver, "service_pack"):
if winver.service_pack != "":
csd = 'SP%s' % winver.service_pack_major
@@ -586,7 +582,7 @@ def win32_ver(release='',version='',csd='',ptype=''):
# Discard any type that isn't REG_SZ
if type == REG_SZ and name.find("Server") != -1:
product_type = VER_NT_SERVER
- except WindowsError:
+ except OSError:
# Use default of VER_NT_WORKSTATION
pass
@@ -611,8 +607,8 @@ def win32_ver(release='',version='',csd='',ptype=''):
else:
if not release:
# E.g. Win3.1 with win32s
- release = '%i.%i' % (maj,min)
- return release,version,csd,ptype
+ release = '%i.%i' % (maj, min)
+ return release, version, csd, ptype
# Open the registry key
try:
@@ -620,7 +616,7 @@ def win32_ver(release='',version='',csd='',ptype=''):
# Get a value to make sure the key exists...
RegQueryValueEx(keyCurVer, 'SystemRoot')
except:
- return release,version,csd,ptype
+ return release, version, csd, ptype
# Parse values
#subversion = _win32_getvalue(keyCurVer,
@@ -630,73 +626,17 @@ def win32_ver(release='',version='',csd='',ptype=''):
# release = release + subversion # 95a, 95b, etc.
build = _win32_getvalue(keyCurVer,
'CurrentBuildNumber',
- ('',1))[0]
+ ('', 1))[0]
ptype = _win32_getvalue(keyCurVer,
'CurrentType',
- (ptype,1))[0]
+ (ptype, 1))[0]
# Normalize version
- version = _norm_version(version,build)
+ version = _norm_version(version, build)
# Close key
RegCloseKey(keyCurVer)
- return release,version,csd,ptype
-
-def _mac_ver_lookup(selectors,default=None):
-
- from _gestalt import gestalt
- l = []
- append = l.append
- for selector in selectors:
- try:
- append(gestalt(selector))
- except (RuntimeError, OSError):
- append(default)
- return l
-
-def _bcd2str(bcd):
-
- return hex(bcd)[2:]
-
-def _mac_ver_gestalt():
- """
- Thanks to Mark R. Levinson for mailing documentation links and
- code examples for this function. Documentation for the
- gestalt() API is available online at:
-
- http://www.rgaros.nl/gestalt/
- """
- # Check whether the version info module is available
- try:
- import _gestalt
- except ImportError:
- return None
- # Get the infos
- sysv, sysa = _mac_ver_lookup(('sysv','sysa'))
- # Decode the infos
- if sysv:
- major = (sysv & 0xFF00) >> 8
- minor = (sysv & 0x00F0) >> 4
- patch = (sysv & 0x000F)
-
- if (major, minor) >= (10, 4):
- # the 'sysv' gestald cannot return patchlevels
- # higher than 9. Apple introduced 3 new
- # gestalt codes in 10.4 to deal with this
- # issue (needed because patch levels can
- # run higher than 9, such as 10.4.11)
- major,minor,patch = _mac_ver_lookup(('sys1','sys2','sys3'))
- release = '%i.%i.%i' %(major, minor, patch)
- else:
- release = '%s.%i.%i' % (_bcd2str(major),minor,patch)
-
- if sysa:
- machine = {0x1: '68k',
- 0x2: 'PowerPC',
- 0xa: 'i386'}.get(sysa,'')
-
- versioninfo=('', '', '')
- return release,versioninfo,machine
+ return release, version, csd, ptype
def _mac_ver_xml():
fn = '/System/Library/CoreServices/SystemVersion.plist'
@@ -708,18 +648,19 @@ def _mac_ver_xml():
except ImportError:
return None
- pl = plistlib.readPlist(fn)
+ with open(fn, 'rb') as f:
+ pl = plistlib.load(f)
release = pl['ProductVersion']
- versioninfo=('', '', '')
+ versioninfo = ('', '', '')
machine = os.uname().machine
if machine in ('ppc', 'Power Macintosh'):
- # for compatibility with the gestalt based code
+ # Canonical name
machine = 'PowerPC'
- return release,versioninfo,machine
+ return release, versioninfo, machine
-def mac_ver(release='',versioninfo=('','',''),machine=''):
+def mac_ver(release='', versioninfo=('', '', ''), machine=''):
""" Get MacOS version information and return it as tuple (release,
versioninfo, machine) with versioninfo being a tuple (version,
@@ -735,16 +676,10 @@ def mac_ver(release='',versioninfo=('','',''),machine=''):
if info is not None:
return info
- # If that doesn't work for some reason fall back to reading the
- # information using gestalt calls.
- info = _mac_ver_gestalt()
- if info is not None:
- return info
-
# If that also doesn't work return the default values
- return release,versioninfo,machine
+ return release, versioninfo, machine
-def _java_getprop(name,default):
+def _java_getprop(name, default):
from java.lang import System
try:
@@ -755,13 +690,13 @@ def _java_getprop(name,default):
except AttributeError:
return default
-def java_ver(release='',vendor='',vminfo=('','',''),osinfo=('','','')):
+def java_ver(release='', vendor='', vminfo=('', '', ''), osinfo=('', '', '')):
""" Version interface for Jython.
- Returns a tuple (release,vendor,vminfo,osinfo) with vminfo being
- a tuple (vm_name,vm_release,vm_vendor) and osinfo being a
- tuple (os_name,os_version,os_arch).
+ Returns a tuple (release, vendor, vminfo, osinfo) with vminfo being
+ a tuple (vm_name, vm_release, vm_vendor) and osinfo being a
+ tuple (os_name, os_version, os_arch).
Values which cannot be determined are set to the defaults
given as parameters (which all default to '').
@@ -771,7 +706,7 @@ def java_ver(release='',vendor='',vminfo=('','',''),osinfo=('','','')):
try:
import java.lang
except ImportError:
- return release,vendor,vminfo,osinfo
+ return release, vendor, vminfo, osinfo
vendor = _java_getprop('java.vendor', vendor)
release = _java_getprop('java.version', release)
@@ -790,9 +725,9 @@ def java_ver(release='',vendor='',vminfo=('','',''),osinfo=('','','')):
### System name aliasing
-def system_alias(system,release,version):
+def system_alias(system, release, version):
- """ Returns (system,release,version) aliased to common
+ """ Returns (system, release, version) aliased to common
marketing names used for some systems.
It also does some reordering of the information in some cases
@@ -802,13 +737,13 @@ def system_alias(system,release,version):
if system == 'Rhapsody':
# Apple's BSD derivative
# XXX How can we determine the marketing release number ?
- return 'MacOS X Server',system+release,version
+ return 'MacOS X Server', system+release, version
elif system == 'SunOS':
# Sun's OS
if release < '5':
# These releases use the old name SunOS
- return system,release,version
+ return system, release, version
# Modify release (marketing release = SunOS release - 3)
l = release.split('.')
if l:
@@ -836,11 +771,11 @@ def system_alias(system,release,version):
else:
version = '64bit'
- elif system in ('win32','win16'):
+ elif system in ('win32', 'win16'):
# In case one of the other tricks
system = 'Windows'
- return system,release,version
+ return system, release, version
### Various internal helpers
@@ -853,21 +788,21 @@ def _platform(*args):
platform = '-'.join(x.strip() for x in filter(len, args))
# Cleanup some possible filename obstacles...
- platform = platform.replace(' ','_')
- platform = platform.replace('/','-')
- platform = platform.replace('\\','-')
- platform = platform.replace(':','-')
- platform = platform.replace(';','-')
- platform = platform.replace('"','-')
- platform = platform.replace('(','-')
- platform = platform.replace(')','-')
+ platform = platform.replace(' ', '_')
+ platform = platform.replace('/', '-')
+ platform = platform.replace('\\', '-')
+ platform = platform.replace(':', '-')
+ platform = platform.replace(';', '-')
+ platform = platform.replace('"', '-')
+ platform = platform.replace('(', '-')
+ platform = platform.replace(')', '-')
# No need to report 'unknown' information...
- platform = platform.replace('unknown','')
+ platform = platform.replace('unknown', '')
# Fold '--'s and remove trailing '-'
while 1:
- cleaned = platform.replace('--','-')
+ cleaned = platform.replace('--', '-')
if cleaned == platform:
break
platform = cleaned
@@ -887,7 +822,7 @@ def _node(default=''):
return default
try:
return socket.gethostname()
- except socket.error:
+ except OSError:
# Still not working...
return default
@@ -899,19 +834,19 @@ def _follow_symlinks(filepath):
filepath = os.path.abspath(filepath)
while os.path.islink(filepath):
filepath = os.path.normpath(
- os.path.join(os.path.dirname(filepath),os.readlink(filepath)))
+ os.path.join(os.path.dirname(filepath), os.readlink(filepath)))
return filepath
-def _syscmd_uname(option,default=''):
+def _syscmd_uname(option, default=''):
""" Interface to the system's uname command.
"""
- if sys.platform in ('dos','win32','win16','os2'):
+ if sys.platform in ('dos', 'win32', 'win16'):
# XXX Others too ?
return default
try:
f = os.popen('uname %s 2> %s' % (option, DEV_NULL))
- except (AttributeError,os.error):
+ except (AttributeError, OSError):
return default
output = f.read().strip()
rc = f.close()
@@ -920,7 +855,7 @@ def _syscmd_uname(option,default=''):
else:
return output
-def _syscmd_file(target,default=''):
+def _syscmd_file(target, default=''):
""" Interface to the system's file command.
@@ -929,7 +864,7 @@ def _syscmd_file(target,default=''):
default in case the command should fail.
"""
- if sys.platform in ('dos','win32','win16','os2'):
+ if sys.platform in ('dos', 'win32', 'win16'):
# XXX Others too ?
return default
target = _follow_symlinks(target)
@@ -937,7 +872,7 @@ def _syscmd_file(target,default=''):
proc = subprocess.Popen(['file', target],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- except (AttributeError,os.error):
+ except (AttributeError, OSError):
return default
output = proc.communicate()[0].decode('latin-1')
rc = proc.wait()
@@ -951,17 +886,17 @@ def _syscmd_file(target,default=''):
# Default values for architecture; non-empty strings override the
# defaults given as parameters
_default_architecture = {
- 'win32': ('','WindowsPE'),
- 'win16': ('','Windows'),
- 'dos': ('','MSDOS'),
+ 'win32': ('', 'WindowsPE'),
+ 'win16': ('', 'Windows'),
+ 'dos': ('', 'MSDOS'),
}
-def architecture(executable=sys.executable,bits='',linkage=''):
+def architecture(executable=sys.executable, bits='', linkage=''):
""" Queries the given executable (defaults to the Python interpreter
binary) for various architecture information.
- Returns a tuple (bits,linkage) which contains information about
+ Returns a tuple (bits, linkage) which contains information about
the bit architecture and the linkage format used for the
executable. Both values are returned as strings.
@@ -999,16 +934,16 @@ def architecture(executable=sys.executable,bits='',linkage=''):
# "file" command did not return anything; we'll try to provide
# some sensible defaults then...
if sys.platform in _default_architecture:
- b,l = _default_architecture[sys.platform]
+ b, l = _default_architecture[sys.platform]
if b:
bits = b
if l:
linkage = l
- return bits,linkage
+ return bits, linkage
if 'executable' not in fileout:
# Format not supported
- return bits,linkage
+ return bits, linkage
# Bits
if '32-bit' in fileout:
@@ -1036,7 +971,7 @@ def architecture(executable=sys.executable,bits='',linkage=''):
# XXX the A.OUT format also falls under this class...
pass
- return bits,linkage
+ return bits, linkage
### Portable uname() interface
@@ -1048,7 +983,7 @@ _uname_cache = None
def uname():
""" Fairly portable uname interface. Returns a tuple
- of strings (system,node,release,version,machine,processor)
+ of strings (system, node, release, version, machine, processor)
identifying the underlying platform.
Note that unlike the os.uname function this also returns
@@ -1067,7 +1002,7 @@ def uname():
# Get some infos from the builtin os.uname API...
try:
- system,node,release,version,machine = os.uname()
+ system, node, release, version, machine = os.uname()
except AttributeError:
no_os_uname = 1
@@ -1085,7 +1020,7 @@ def uname():
# Try win32_ver() on win32 platforms
if system == 'win32':
- release,version,csd,ptype = win32_ver()
+ release, version, csd, ptype = win32_ver()
if release and version:
use_syscmd_ver = 0
# Try to use the PROCESSOR_* environment variables
@@ -1104,7 +1039,7 @@ def uname():
# Try the 'ver' system command available on some
# platforms
if use_syscmd_ver:
- system,release,version = _syscmd_ver(system)
+ system, release, version = _syscmd_ver(system)
# Normalize system to what win32_ver() normally returns
# (_syscmd_ver() tends to return the vendor name as well)
if system == 'Microsoft Windows':
@@ -1122,7 +1057,7 @@ def uname():
# In case we still don't know anything useful, we'll try to
# help ourselves
- if system in ('win32','win16'):
+ if system in ('win32', 'win16'):
if not version:
if system == 'win32':
version = '32bit'
@@ -1131,7 +1066,7 @@ def uname():
system = 'Windows'
elif system[:4] == 'java':
- release,vendor,vminfo,osinfo = java_ver()
+ release, vendor, vminfo, osinfo = java_ver()
system = 'Java'
version = ', '.join(vminfo)
if not version:
@@ -1149,14 +1084,14 @@ def uname():
except ImportError:
pass
else:
- csid, cpu_number = vms_lib.getsyi('SYI$_CPU',0)
+ csid, cpu_number = vms_lib.getsyi('SYI$_CPU', 0)
if (cpu_number >= 128):
processor = 'Alpha'
else:
processor = 'VAX'
if not processor:
# Get processor information from the uname system command
- processor = _syscmd_uname('-p','')
+ processor = _syscmd_uname('-p', '')
#If any unknowns still exist, replace them with ''s, which are more portable
if system == 'unknown':
@@ -1177,7 +1112,8 @@ def uname():
system = 'Windows'
release = 'Vista'
- _uname_cache = uname_result(system,node,release,version,machine,processor)
+ _uname_cache = uname_result(system, node, release, version,
+ machine, processor)
return _uname_cache
### Direct interfaces to some of the uname() return values
@@ -1474,57 +1410,58 @@ def platform(aliased=0, terse=0):
# Get uname information and then apply platform specific cosmetics
# to it...
- system,node,release,version,machine,processor = uname()
+ system, node, release, version, machine, processor = uname()
if machine == processor:
processor = ''
if aliased:
- system,release,version = system_alias(system,release,version)
+ system, release, version = system_alias(system, release, version)
if system == 'Windows':
# MS platforms
- rel,vers,csd,ptype = win32_ver(version)
+ rel, vers, csd, ptype = win32_ver(version)
if terse:
- platform = _platform(system,release)
+ platform = _platform(system, release)
else:
- platform = _platform(system,release,version,csd)
+ platform = _platform(system, release, version, csd)
elif system in ('Linux',):
# Linux based systems
- distname,distversion,distid = dist('')
+ distname, distversion, distid = dist('')
if distname and not terse:
- platform = _platform(system,release,machine,processor,
+ platform = _platform(system, release, machine, processor,
'with',
- distname,distversion,distid)
+ distname, distversion, distid)
else:
# If the distribution name is unknown check for libc vs. glibc
- libcname,libcversion = libc_ver(sys.executable)
- platform = _platform(system,release,machine,processor,
+ libcname, libcversion = libc_ver(sys.executable)
+ platform = _platform(system, release, machine, processor,
'with',
libcname+libcversion)
elif system == 'Java':
# Java platforms
- r,v,vminfo,(os_name,os_version,os_arch) = java_ver()
+ r, v, vminfo, (os_name, os_version, os_arch) = java_ver()
if terse or not os_name:
- platform = _platform(system,release,version)
+ platform = _platform(system, release, version)
else:
- platform = _platform(system,release,version,
+ platform = _platform(system, release, version,
'on',
- os_name,os_version,os_arch)
+ os_name, os_version, os_arch)
elif system == 'MacOS':
# MacOS platforms
if terse:
- platform = _platform(system,release)
+ platform = _platform(system, release)
else:
- platform = _platform(system,release,machine)
+ platform = _platform(system, release, machine)
else:
# Generic handler
if terse:
- platform = _platform(system,release)
+ platform = _platform(system, release)
else:
- bits,linkage = architecture(sys.executable)
- platform = _platform(system,release,machine,processor,bits,linkage)
+ bits, linkage = architecture(sys.executable)
+ platform = _platform(system, release, machine,
+ processor, bits, linkage)
_platform_cache[(aliased, terse)] = platform
return platform
@@ -1535,5 +1472,5 @@ if __name__ == '__main__':
# Default is to print the aliased verbose platform string
terse = ('terse' in sys.argv or '--terse' in sys.argv)
aliased = (not 'nonaliased' in sys.argv and not '--nonaliased' in sys.argv)
- print(platform(aliased,terse))
+ print(platform(aliased, terse))
sys.exit(0)
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
index 2b0b634..dcb0f9c 100644
--- a/Lib/plistlib.py
+++ b/Lib/plistlib.py
@@ -4,25 +4,20 @@ The property list (.plist) file format is a simple XML pickle supporting
basic object types, like dictionaries, lists, numbers and strings.
Usually the top level object is a dictionary.
-To write out a plist file, use the writePlist(rootObject, pathOrFile)
-function. 'rootObject' is the top level object, 'pathOrFile' is a
-filename or a (writable) file object.
+To write out a plist file, use the dump(value, file)
+function. 'value' is the top level object, 'file' is
+a (writable) file object.
-To parse a plist from a file, use the readPlist(pathOrFile) function,
-with a file name or a (readable) file object as the only argument. It
+To parse a plist from a file, use the load(file) function,
+with a (readable) file object as the only argument. It
returns the top level object (again, usually a dictionary).
-To work with plist data in bytes objects, you can use readPlistFromBytes()
-and writePlistToBytes().
+To work with plist data in bytes objects, you can use loads()
+and dumps().
Values can be strings, integers, floats, booleans, tuples, lists,
-dictionaries (but only with string keys), Data or datetime.datetime objects.
-String values (including dictionary keys) have to be unicode strings -- they
-will be written out as UTF-8.
-
-The <data> plist type is supported through the Data class. This is a
-thin wrapper around a Python bytes object. Use 'Data' if your strings
-contain control characters.
+dictionaries (but only with string keys), Data, bytes, bytearray, or
+datetime.datetime objects.
Generate Plist example:
@@ -37,226 +32,48 @@ Generate Plist example:
aTrueValue = True,
aFalseValue = False,
),
- someData = Data(b"<binary gunk>"),
- someMoreData = Data(b"<lots of binary gunk>" * 10),
+ someData = b"<binary gunk>",
+ someMoreData = b"<lots of binary gunk>" * 10,
aDate = datetime.datetime.fromtimestamp(time.mktime(time.gmtime())),
)
- writePlist(pl, fileName)
+ with open(fileName, 'wb') as fp:
+ dump(pl, fp)
Parse Plist example:
- pl = readPlist(pathOrFile)
- print pl["aKey"]
+ with open(fileName, 'rb') as fp:
+ pl = load(fp)
+ print(pl["aKey"])
"""
-
-
__all__ = [
"readPlist", "writePlist", "readPlistFromBytes", "writePlistToBytes",
- "Plist", "Data", "Dict"
+ "Plist", "Data", "Dict", "FMT_XML", "FMT_BINARY",
+ "load", "dump", "loads", "dumps"
]
-# Note: the Plist and Dict classes have been deprecated.
import binascii
+import codecs
+import contextlib
import datetime
+import enum
from io import BytesIO
+import itertools
+import os
import re
+import struct
+from warnings import warn
+from xml.parsers.expat import ParserCreate
-def readPlist(pathOrFile):
- """Read a .plist file. 'pathOrFile' may either be a file name or a
- (readable) file object. Return the unpacked root object (which
- usually is a dictionary).
- """
- didOpen = False
- try:
- if isinstance(pathOrFile, str):
- pathOrFile = open(pathOrFile, 'rb')
- didOpen = True
- p = PlistParser()
- rootObject = p.parse(pathOrFile)
- finally:
- if didOpen:
- pathOrFile.close()
- return rootObject
-
-
-def writePlist(rootObject, pathOrFile):
- """Write 'rootObject' to a .plist file. 'pathOrFile' may either be a
- file name or a (writable) file object.
- """
- didOpen = False
- try:
- if isinstance(pathOrFile, str):
- pathOrFile = open(pathOrFile, 'wb')
- didOpen = True
- writer = PlistWriter(pathOrFile)
- writer.writeln("<plist version=\"1.0\">")
- writer.writeValue(rootObject)
- writer.writeln("</plist>")
- finally:
- if didOpen:
- pathOrFile.close()
-
-
-def readPlistFromBytes(data):
- """Read a plist data from a bytes object. Return the root object.
- """
- return readPlist(BytesIO(data))
-
-
-def writePlistToBytes(rootObject):
- """Return 'rootObject' as a plist-formatted bytes object.
- """
- f = BytesIO()
- writePlist(rootObject, f)
- return f.getvalue()
-
-
-class DumbXMLWriter:
- def __init__(self, file, indentLevel=0, indent="\t"):
- self.file = file
- self.stack = []
- self.indentLevel = indentLevel
- self.indent = indent
-
- def beginElement(self, element):
- self.stack.append(element)
- self.writeln("<%s>" % element)
- self.indentLevel += 1
-
- def endElement(self, element):
- assert self.indentLevel > 0
- assert self.stack.pop() == element
- self.indentLevel -= 1
- self.writeln("</%s>" % element)
-
- def simpleElement(self, element, value=None):
- if value is not None:
- value = _escape(value)
- self.writeln("<%s>%s</%s>" % (element, value, element))
- else:
- self.writeln("<%s/>" % element)
-
- def writeln(self, line):
- if line:
- # plist has fixed encoding of utf-8
- if isinstance(line, str):
- line = line.encode('utf-8')
- self.file.write(self.indentLevel * self.indent)
- self.file.write(line)
- self.file.write(b'\n')
-
-
-# Contents should conform to a subset of ISO 8601
-# (in particular, YYYY '-' MM '-' DD 'T' HH ':' MM ':' SS 'Z'. Smaller units may be omitted with
-# a loss of precision)
-_dateParser = re.compile(r"(?P<year>\d\d\d\d)(?:-(?P<month>\d\d)(?:-(?P<day>\d\d)(?:T(?P<hour>\d\d)(?::(?P<minute>\d\d)(?::(?P<second>\d\d))?)?)?)?)?Z", re.ASCII)
-
-def _dateFromString(s):
- order = ('year', 'month', 'day', 'hour', 'minute', 'second')
- gd = _dateParser.match(s).groupdict()
- lst = []
- for key in order:
- val = gd[key]
- if val is None:
- break
- lst.append(int(val))
- return datetime.datetime(*lst)
-
-def _dateToString(d):
- return '%04d-%02d-%02dT%02d:%02d:%02dZ' % (
- d.year, d.month, d.day,
- d.hour, d.minute, d.second
- )
-
-
-# Regex to find any control chars, except for \t \n and \r
-_controlCharPat = re.compile(
- r"[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f"
- r"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]")
-
-def _escape(text):
- m = _controlCharPat.search(text)
- if m is not None:
- raise ValueError("strings can't contains control characters; "
- "use plistlib.Data instead")
- text = text.replace("\r\n", "\n") # convert DOS line endings
- text = text.replace("\r", "\n") # convert Mac line endings
- text = text.replace("&", "&amp;") # escape '&'
- text = text.replace("<", "&lt;") # escape '<'
- text = text.replace(">", "&gt;") # escape '>'
- return text
-
-
-PLISTHEADER = b"""\
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-"""
-
-class PlistWriter(DumbXMLWriter):
-
- def __init__(self, file, indentLevel=0, indent=b"\t", writeHeader=1):
- if writeHeader:
- file.write(PLISTHEADER)
- DumbXMLWriter.__init__(self, file, indentLevel, indent)
-
- def writeValue(self, value):
- if isinstance(value, str):
- self.simpleElement("string", value)
- elif isinstance(value, bool):
- # must switch for bool before int, as bool is a
- # subclass of int...
- if value:
- self.simpleElement("true")
- else:
- self.simpleElement("false")
- elif isinstance(value, int):
- self.simpleElement("integer", "%d" % value)
- elif isinstance(value, float):
- self.simpleElement("real", repr(value))
- elif isinstance(value, dict):
- self.writeDict(value)
- elif isinstance(value, Data):
- self.writeData(value)
- elif isinstance(value, datetime.datetime):
- self.simpleElement("date", _dateToString(value))
- elif isinstance(value, (tuple, list)):
- self.writeArray(value)
- else:
- raise TypeError("unsupported type: %s" % type(value))
-
- def writeData(self, data):
- self.beginElement("data")
- self.indentLevel -= 1
- maxlinelength = max(16, 76 - len(self.indent.replace(b"\t", b" " * 8) *
- self.indentLevel))
- for line in data.asBase64(maxlinelength).split(b"\n"):
- if line:
- self.writeln(line)
- self.indentLevel += 1
- self.endElement("data")
+PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__)
+globals().update(PlistFormat.__members__)
- def writeDict(self, d):
- if d:
- self.beginElement("dict")
- items = sorted(d.items())
- for key, value in items:
- if not isinstance(key, str):
- raise TypeError("keys must be strings")
- self.simpleElement("key", key)
- self.writeValue(value)
- self.endElement("dict")
- else:
- self.simpleElement("dict")
- def writeArray(self, array):
- if array:
- self.beginElement("array")
- for value in array:
- self.writeValue(value)
- self.endElement("array")
- else:
- self.simpleElement("array")
+#
+#
+# Deprecated functionality
+#
+#
class _InternalDict(dict):
@@ -264,19 +81,18 @@ class _InternalDict(dict):
# This class is needed while Dict is scheduled for deprecation:
# we only need to warn when a *user* instantiates Dict or when
# the "attribute notation for dict keys" is used.
+ __slots__ = ()
def __getattr__(self, attr):
try:
value = self[attr]
except KeyError:
raise AttributeError(attr)
- from warnings import warn
warn("Attribute access from plist dicts is deprecated, use d[key] "
"notation instead", DeprecationWarning, 2)
return value
def __setattr__(self, attr, value):
- from warnings import warn
warn("Attribute access from plist dicts is deprecated, use d[key] "
"notation instead", DeprecationWarning, 2)
self[attr] = value
@@ -286,56 +102,111 @@ class _InternalDict(dict):
del self[attr]
except KeyError:
raise AttributeError(attr)
- from warnings import warn
warn("Attribute access from plist dicts is deprecated, use d[key] "
"notation instead", DeprecationWarning, 2)
+
class Dict(_InternalDict):
def __init__(self, **kwargs):
- from warnings import warn
warn("The plistlib.Dict class is deprecated, use builtin dict instead",
DeprecationWarning, 2)
super().__init__(**kwargs)
-class Plist(_InternalDict):
+@contextlib.contextmanager
+def _maybe_open(pathOrFile, mode):
+ if isinstance(pathOrFile, str):
+ with open(pathOrFile, mode) as fp:
+ yield fp
+
+ else:
+ yield pathOrFile
+
- """This class has been deprecated. Use readPlist() and writePlist()
+class Plist(_InternalDict):
+ """This class has been deprecated. Use dump() and load()
functions instead, together with regular dict objects.
"""
def __init__(self, **kwargs):
- from warnings import warn
- warn("The Plist class is deprecated, use the readPlist() and "
- "writePlist() functions instead", DeprecationWarning, 2)
+ warn("The Plist class is deprecated, use the load() and "
+ "dump() functions instead", DeprecationWarning, 2)
super().__init__(**kwargs)
+ @classmethod
def fromFile(cls, pathOrFile):
- """Deprecated. Use the readPlist() function instead."""
- rootObject = readPlist(pathOrFile)
+ """Deprecated. Use the load() function instead."""
+ with _maybe_open(pathOrFile, 'rb') as fp:
+ value = load(fp)
plist = cls()
- plist.update(rootObject)
+ plist.update(value)
return plist
- fromFile = classmethod(fromFile)
def write(self, pathOrFile):
- """Deprecated. Use the writePlist() function instead."""
- writePlist(self, pathOrFile)
+ """Deprecated. Use the dump() function instead."""
+ with _maybe_open(pathOrFile, 'wb') as fp:
+ dump(self, fp)
-def _encodeBase64(s, maxlinelength=76):
- # copied from base64.encodebytes(), with added maxlinelength argument
- maxbinsize = (maxlinelength//4)*3
- pieces = []
- for i in range(0, len(s), maxbinsize):
- chunk = s[i : i + maxbinsize]
- pieces.append(binascii.b2a_base64(chunk))
- return b''.join(pieces)
+def readPlist(pathOrFile):
+ """
+ Read a .plist from a path or file. pathOrFile should either
+ be a file name, or a readable binary file object.
+
+ This function is deprecated, use load instead.
+ """
+ warn("The readPlist function is deprecated, use load() instead",
+ DeprecationWarning, 2)
+
+ with _maybe_open(pathOrFile, 'rb') as fp:
+ return load(fp, fmt=None, use_builtin_types=False,
+ dict_type=_InternalDict)
+
+def writePlist(value, pathOrFile):
+ """
+ Write 'value' to a .plist file. 'pathOrFile' may either be a
+ file name or a (writable) file object.
+
+ This function is deprecated, use dump instead.
+ """
+ warn("The writePlist function is deprecated, use dump() instead",
+ DeprecationWarning, 2)
+ with _maybe_open(pathOrFile, 'wb') as fp:
+ dump(value, fp, fmt=FMT_XML, sort_keys=True, skipkeys=False)
+
+
+def readPlistFromBytes(data):
+ """
+ Read a plist data from a bytes object. Return the root object.
+
+ This function is deprecated, use loads instead.
+ """
+ warn("The readPlistFromBytes function is deprecated, use loads() instead",
+ DeprecationWarning, 2)
+ return load(BytesIO(data), fmt=None, use_builtin_types=False,
+ dict_type=_InternalDict)
+
+
+def writePlistToBytes(value):
+ """
+ Return 'value' as a plist-formatted bytes object.
+
+ This function is deprecated, use dumps instead.
+ """
+ warn("The writePlistToBytes function is deprecated, use dumps() instead",
+ DeprecationWarning, 2)
+ f = BytesIO()
+ dump(value, f, fmt=FMT_XML, sort_keys=True, skipkeys=False)
+ return f.getvalue()
+
class Data:
+ """
+ Wrapper for binary data.
- """Wrapper for binary data."""
+ This class is deprecated, use a bytes object instead.
+ """
def __init__(self, data):
if not isinstance(data, bytes):
@@ -346,10 +217,10 @@ class Data:
def fromBase64(cls, data):
# base64.decodebytes just calls binascii.a2b_base64;
# it seems overkill to use both base64 and binascii.
- return cls(binascii.a2b_base64(data))
+ return cls(_decode_base64(data))
def asBase64(self, maxlinelength=76):
- return _encodeBase64(self.data, maxlinelength)
+ return _encode_base64(self.data, maxlinelength)
def __eq__(self, other):
if isinstance(other, self.__class__):
@@ -362,43 +233,119 @@ class Data:
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, repr(self.data))
-class PlistParser:
+#
+#
+# End of deprecated functionality
+#
+#
+
+
+#
+# XML support
+#
+
+
+# XML 'header'
+PLISTHEADER = b"""\
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+"""
+
+
+# Regex to find any control chars, except for \t \n and \r
+_controlCharPat = re.compile(
+ r"[\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0b\x0c\x0e\x0f"
+ r"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f]")
+
+def _encode_base64(s, maxlinelength=76):
+ # copied from base64.encodebytes(), with added maxlinelength argument
+ maxbinsize = (maxlinelength//4)*3
+ pieces = []
+ for i in range(0, len(s), maxbinsize):
+ chunk = s[i : i + maxbinsize]
+ pieces.append(binascii.b2a_base64(chunk))
+ return b''.join(pieces)
+
+def _decode_base64(s):
+ if isinstance(s, str):
+ return binascii.a2b_base64(s.encode("utf-8"))
+
+ else:
+ return binascii.a2b_base64(s)
+
+# Contents should conform to a subset of ISO 8601
+# (in particular, YYYY '-' MM '-' DD 'T' HH ':' MM ':' SS 'Z'. Smaller units
+# may be omitted with # a loss of precision)
+_dateParser = re.compile(r"(?P<year>\d\d\d\d)(?:-(?P<month>\d\d)(?:-(?P<day>\d\d)(?:T(?P<hour>\d\d)(?::(?P<minute>\d\d)(?::(?P<second>\d\d))?)?)?)?)?Z", re.ASCII)
+
+
+def _date_from_string(s):
+ order = ('year', 'month', 'day', 'hour', 'minute', 'second')
+ gd = _dateParser.match(s).groupdict()
+ lst = []
+ for key in order:
+ val = gd[key]
+ if val is None:
+ break
+ lst.append(int(val))
+ return datetime.datetime(*lst)
+
+
+def _date_to_string(d):
+ return '%04d-%02d-%02dT%02d:%02d:%02dZ' % (
+ d.year, d.month, d.day,
+ d.hour, d.minute, d.second
+ )
- def __init__(self):
+def _escape(text):
+ m = _controlCharPat.search(text)
+ if m is not None:
+ raise ValueError("strings can't contains control characters; "
+ "use bytes instead")
+ text = text.replace("\r\n", "\n") # convert DOS line endings
+ text = text.replace("\r", "\n") # convert Mac line endings
+ text = text.replace("&", "&amp;") # escape '&'
+ text = text.replace("<", "&lt;") # escape '<'
+ text = text.replace(">", "&gt;") # escape '>'
+ return text
+
+class _PlistParser:
+ def __init__(self, use_builtin_types, dict_type):
self.stack = []
- self.currentKey = None
+ self.current_key = None
self.root = None
+ self._use_builtin_types = use_builtin_types
+ self._dict_type = dict_type
def parse(self, fileobj):
- from xml.parsers.expat import ParserCreate
self.parser = ParserCreate()
- self.parser.StartElementHandler = self.handleBeginElement
- self.parser.EndElementHandler = self.handleEndElement
- self.parser.CharacterDataHandler = self.handleData
+ self.parser.StartElementHandler = self.handle_begin_element
+ self.parser.EndElementHandler = self.handle_end_element
+ self.parser.CharacterDataHandler = self.handle_data
self.parser.ParseFile(fileobj)
return self.root
- def handleBeginElement(self, element, attrs):
+ def handle_begin_element(self, element, attrs):
self.data = []
handler = getattr(self, "begin_" + element, None)
if handler is not None:
handler(attrs)
- def handleEndElement(self, element):
+ def handle_end_element(self, element):
handler = getattr(self, "end_" + element, None)
if handler is not None:
handler()
- def handleData(self, data):
+ def handle_data(self, data):
self.data.append(data)
- def addObject(self, value):
- if self.currentKey is not None:
+ def add_object(self, value):
+ if self.current_key is not None:
if not isinstance(self.stack[-1], type({})):
raise ValueError("unexpected element at line %d" %
self.parser.CurrentLineNumber)
- self.stack[-1][self.currentKey] = value
- self.currentKey = None
+ self.stack[-1][self.current_key] = value
+ self.current_key = None
elif not self.stack:
# this is the root object
self.root = value
@@ -408,7 +355,7 @@ class PlistParser:
self.parser.CurrentLineNumber)
self.stack[-1].append(value)
- def getData(self):
+ def get_data(self):
data = ''.join(self.data)
self.data = []
return data
@@ -416,39 +363,661 @@ class PlistParser:
# element handlers
def begin_dict(self, attrs):
- d = _InternalDict()
- self.addObject(d)
+ d = self._dict_type()
+ self.add_object(d)
self.stack.append(d)
+
def end_dict(self):
- if self.currentKey:
+ if self.current_key:
raise ValueError("missing value for key '%s' at line %d" %
- (self.currentKey,self.parser.CurrentLineNumber))
+ (self.current_key,self.parser.CurrentLineNumber))
self.stack.pop()
def end_key(self):
- if self.currentKey or not isinstance(self.stack[-1], type({})):
+ if self.current_key or not isinstance(self.stack[-1], type({})):
raise ValueError("unexpected key at line %d" %
self.parser.CurrentLineNumber)
- self.currentKey = self.getData()
+ self.current_key = self.get_data()
def begin_array(self, attrs):
a = []
- self.addObject(a)
+ self.add_object(a)
self.stack.append(a)
+
def end_array(self):
self.stack.pop()
def end_true(self):
- self.addObject(True)
+ self.add_object(True)
+
def end_false(self):
- self.addObject(False)
+ self.add_object(False)
+
def end_integer(self):
- self.addObject(int(self.getData()))
+ self.add_object(int(self.get_data()))
+
def end_real(self):
- self.addObject(float(self.getData()))
+ self.add_object(float(self.get_data()))
+
def end_string(self):
- self.addObject(self.getData())
+ self.add_object(self.get_data())
+
def end_data(self):
- self.addObject(Data.fromBase64(self.getData().encode("utf-8")))
+ if self._use_builtin_types:
+ self.add_object(_decode_base64(self.get_data()))
+
+ else:
+ self.add_object(Data.fromBase64(self.get_data()))
+
def end_date(self):
- self.addObject(_dateFromString(self.getData()))
+ self.add_object(_date_from_string(self.get_data()))
+
+
+class _DumbXMLWriter:
+ def __init__(self, file, indent_level=0, indent="\t"):
+ self.file = file
+ self.stack = []
+ self._indent_level = indent_level
+ self.indent = indent
+
+ def begin_element(self, element):
+ self.stack.append(element)
+ self.writeln("<%s>" % element)
+ self._indent_level += 1
+
+ def end_element(self, element):
+ assert self._indent_level > 0
+ assert self.stack.pop() == element
+ self._indent_level -= 1
+ self.writeln("</%s>" % element)
+
+ def simple_element(self, element, value=None):
+ if value is not None:
+ value = _escape(value)
+ self.writeln("<%s>%s</%s>" % (element, value, element))
+
+ else:
+ self.writeln("<%s/>" % element)
+
+ def writeln(self, line):
+ if line:
+ # plist has fixed encoding of utf-8
+
+ # XXX: is this test needed?
+ if isinstance(line, str):
+ line = line.encode('utf-8')
+ self.file.write(self._indent_level * self.indent)
+ self.file.write(line)
+ self.file.write(b'\n')
+
+
+class _PlistWriter(_DumbXMLWriter):
+ def __init__(
+ self, file, indent_level=0, indent=b"\t", writeHeader=1,
+ sort_keys=True, skipkeys=False):
+
+ if writeHeader:
+ file.write(PLISTHEADER)
+ _DumbXMLWriter.__init__(self, file, indent_level, indent)
+ self._sort_keys = sort_keys
+ self._skipkeys = skipkeys
+
+ def write(self, value):
+ self.writeln("<plist version=\"1.0\">")
+ self.write_value(value)
+ self.writeln("</plist>")
+
+ def write_value(self, value):
+ if isinstance(value, str):
+ self.simple_element("string", value)
+
+ elif value is True:
+ self.simple_element("true")
+
+ elif value is False:
+ self.simple_element("false")
+
+ elif isinstance(value, int):
+ if -1 << 63 <= value < 1 << 64:
+ self.simple_element("integer", "%d" % value)
+ else:
+ raise OverflowError(value)
+
+ elif isinstance(value, float):
+ self.simple_element("real", repr(value))
+
+ elif isinstance(value, dict):
+ self.write_dict(value)
+
+ elif isinstance(value, Data):
+ self.write_data(value)
+
+ elif isinstance(value, (bytes, bytearray)):
+ self.write_bytes(value)
+
+ elif isinstance(value, datetime.datetime):
+ self.simple_element("date", _date_to_string(value))
+
+ elif isinstance(value, (tuple, list)):
+ self.write_array(value)
+
+ else:
+ raise TypeError("unsupported type: %s" % type(value))
+
+ def write_data(self, data):
+ self.write_bytes(data.data)
+
+ def write_bytes(self, data):
+ self.begin_element("data")
+ self._indent_level -= 1
+ maxlinelength = max(
+ 16,
+ 76 - len(self.indent.replace(b"\t", b" " * 8) * self._indent_level))
+
+ for line in _encode_base64(data, maxlinelength).split(b"\n"):
+ if line:
+ self.writeln(line)
+ self._indent_level += 1
+ self.end_element("data")
+
+ def write_dict(self, d):
+ if d:
+ self.begin_element("dict")
+ if self._sort_keys:
+ items = sorted(d.items())
+ else:
+ items = d.items()
+
+ for key, value in items:
+ if not isinstance(key, str):
+ if self._skipkeys:
+ continue
+ raise TypeError("keys must be strings")
+ self.simple_element("key", key)
+ self.write_value(value)
+ self.end_element("dict")
+
+ else:
+ self.simple_element("dict")
+
+ def write_array(self, array):
+ if array:
+ self.begin_element("array")
+ for value in array:
+ self.write_value(value)
+ self.end_element("array")
+
+ else:
+ self.simple_element("array")
+
+
+def _is_fmt_xml(header):
+ prefixes = (b'<?xml', b'<plist')
+
+ for pfx in prefixes:
+ if header.startswith(pfx):
+ return True
+
+ # Also check for alternative XML encodings, this is slightly
+ # overkill because the Apple tools (and plistlib) will not
+ # generate files with these encodings.
+ for bom, encoding in (
+ (codecs.BOM_UTF8, "utf-8"),
+ (codecs.BOM_UTF16_BE, "utf-16-be"),
+ (codecs.BOM_UTF16_LE, "utf-16-le"),
+ # expat does not support utf-32
+ #(codecs.BOM_UTF32_BE, "utf-32-be"),
+ #(codecs.BOM_UTF32_LE, "utf-32-le"),
+ ):
+ if not header.startswith(bom):
+ continue
+
+ for start in prefixes:
+ prefix = bom + start.decode('ascii').encode(encoding)
+ if header[:len(prefix)] == prefix:
+ return True
+
+ return False
+
+#
+# Binary Plist
+#
+
+
+class InvalidFileException (ValueError):
+ def __init__(self, message="Invalid file"):
+ ValueError.__init__(self, message)
+
+_BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'}
+
+class _BinaryPlistParser:
+ """
+ Read or write a binary plist file, following the description of the binary
+ format. Raise InvalidFileException in case of error, otherwise return the
+ root object.
+
+ see also: http://opensource.apple.com/source/CF/CF-744.18/CFBinaryPList.c
+ """
+ def __init__(self, use_builtin_types, dict_type):
+ self._use_builtin_types = use_builtin_types
+ self._dict_type = dict_type
+
+ def parse(self, fp):
+ try:
+ # The basic file format:
+ # HEADER
+ # object...
+ # refid->offset...
+ # TRAILER
+ self._fp = fp
+ self._fp.seek(-32, os.SEEK_END)
+ trailer = self._fp.read(32)
+ if len(trailer) != 32:
+ raise InvalidFileException()
+ (
+ offset_size, self._ref_size, num_objects, top_object,
+ offset_table_offset
+ ) = struct.unpack('>6xBBQQQ', trailer)
+ self._fp.seek(offset_table_offset)
+ offset_format = '>' + _BINARY_FORMAT[offset_size] * num_objects
+ self._ref_format = _BINARY_FORMAT[self._ref_size]
+ self._object_offsets = struct.unpack(
+ offset_format, self._fp.read(offset_size * num_objects))
+ return self._read_object(self._object_offsets[top_object])
+
+ except (OSError, IndexError, struct.error):
+ raise InvalidFileException()
+
+ def _get_size(self, tokenL):
+ """ return the size of the next object."""
+ if tokenL == 0xF:
+ m = self._fp.read(1)[0] & 0x3
+ s = 1 << m
+ f = '>' + _BINARY_FORMAT[s]
+ return struct.unpack(f, self._fp.read(s))[0]
+
+ return tokenL
+
+ def _read_refs(self, n):
+ return struct.unpack(
+ '>' + self._ref_format * n, self._fp.read(n * self._ref_size))
+
+ def _read_object(self, offset):
+ """
+ read the object at offset.
+
+ May recursively read sub-objects (content of an array/dict/set)
+ """
+ self._fp.seek(offset)
+ token = self._fp.read(1)[0]
+ tokenH, tokenL = token & 0xF0, token & 0x0F
+
+ if token == 0x00:
+ return None
+
+ elif token == 0x08:
+ return False
+
+ elif token == 0x09:
+ return True
+
+ # The referenced source code also mentions URL (0x0c, 0x0d) and
+ # UUID (0x0e), but neither can be generated using the Cocoa libraries.
+
+ elif token == 0x0f:
+ return b''
+
+ elif tokenH == 0x10: # int
+ return int.from_bytes(self._fp.read(1 << tokenL),
+ 'big', signed=tokenL >= 3)
+
+ elif token == 0x22: # real
+ return struct.unpack('>f', self._fp.read(4))[0]
+
+ elif token == 0x23: # real
+ return struct.unpack('>d', self._fp.read(8))[0]
+
+ elif token == 0x33: # date
+ f = struct.unpack('>d', self._fp.read(8))[0]
+ # timestamp 0 of binary plists corresponds to 1/1/2001
+ # (year of Mac OS X 10.0), instead of 1/1/1970.
+ return datetime.datetime.utcfromtimestamp(f + (31 * 365 + 8) * 86400)
+
+ elif tokenH == 0x40: # data
+ s = self._get_size(tokenL)
+ if self._use_builtin_types:
+ return self._fp.read(s)
+ else:
+ return Data(self._fp.read(s))
+
+ elif tokenH == 0x50: # ascii string
+ s = self._get_size(tokenL)
+ result = self._fp.read(s).decode('ascii')
+ return result
+
+ elif tokenH == 0x60: # unicode string
+ s = self._get_size(tokenL)
+ return self._fp.read(s * 2).decode('utf-16be')
+
+ # tokenH == 0x80 is documented as 'UID' and appears to be used for
+ # keyed-archiving, not in plists.
+
+ elif tokenH == 0xA0: # array
+ s = self._get_size(tokenL)
+ obj_refs = self._read_refs(s)
+ return [self._read_object(self._object_offsets[x])
+ for x in obj_refs]
+
+ # tokenH == 0xB0 is documented as 'ordset', but is not actually
+ # implemented in the Apple reference code.
+
+ # tokenH == 0xC0 is documented as 'set', but sets cannot be used in
+ # plists.
+
+ elif tokenH == 0xD0: # dict
+ s = self._get_size(tokenL)
+ key_refs = self._read_refs(s)
+ obj_refs = self._read_refs(s)
+ result = self._dict_type()
+ for k, o in zip(key_refs, obj_refs):
+ result[self._read_object(self._object_offsets[k])
+ ] = self._read_object(self._object_offsets[o])
+ return result
+
+ raise InvalidFileException()
+
+def _count_to_size(count):
+ if count < 1 << 8:
+ return 1
+
+ elif count < 1 << 16:
+ return 2
+
+ elif count << 1 << 32:
+ return 4
+
+ else:
+ return 8
+
+class _BinaryPlistWriter (object):
+ def __init__(self, fp, sort_keys, skipkeys):
+ self._fp = fp
+ self._sort_keys = sort_keys
+ self._skipkeys = skipkeys
+
+ def write(self, value):
+
+ # Flattened object list:
+ self._objlist = []
+
+ # Mappings from object->objectid
+ # First dict has (type(object), object) as the key,
+ # second dict is used when object is not hashable and
+ # has id(object) as the key.
+ self._objtable = {}
+ self._objidtable = {}
+
+ # Create list of all objects in the plist
+ self._flatten(value)
+
+ # Size of object references in serialized containers
+ # depends on the number of objects in the plist.
+ num_objects = len(self._objlist)
+ self._object_offsets = [0]*num_objects
+ self._ref_size = _count_to_size(num_objects)
+
+ self._ref_format = _BINARY_FORMAT[self._ref_size]
+
+ # Write file header
+ self._fp.write(b'bplist00')
+
+ # Write object list
+ for obj in self._objlist:
+ self._write_object(obj)
+
+ # Write refnum->object offset table
+ top_object = self._getrefnum(value)
+ offset_table_offset = self._fp.tell()
+ offset_size = _count_to_size(offset_table_offset)
+ offset_format = '>' + _BINARY_FORMAT[offset_size] * num_objects
+ self._fp.write(struct.pack(offset_format, *self._object_offsets))
+
+ # Write trailer
+ sort_version = 0
+ trailer = (
+ sort_version, offset_size, self._ref_size, num_objects,
+ top_object, offset_table_offset
+ )
+ self._fp.write(struct.pack('>5xBBBQQQ', *trailer))
+
+ def _flatten(self, value):
+ # First check if the object is in the object table, not used for
+ # containers to ensure that two subcontainers with the same contents
+ # will be serialized as distinct values.
+ if isinstance(value, (
+ str, int, float, datetime.datetime, bytes, bytearray)):
+ if (type(value), value) in self._objtable:
+ return
+
+ elif isinstance(value, Data):
+ if (type(value.data), value.data) in self._objtable:
+ return
+
+ # Add to objectreference map
+ refnum = len(self._objlist)
+ self._objlist.append(value)
+ try:
+ if isinstance(value, Data):
+ self._objtable[(type(value.data), value.data)] = refnum
+ else:
+ self._objtable[(type(value), value)] = refnum
+ except TypeError:
+ self._objidtable[id(value)] = refnum
+
+ # And finally recurse into containers
+ if isinstance(value, dict):
+ keys = []
+ values = []
+ items = value.items()
+ if self._sort_keys:
+ items = sorted(items)
+
+ for k, v in items:
+ if not isinstance(k, str):
+ if self._skipkeys:
+ continue
+ raise TypeError("keys must be strings")
+ keys.append(k)
+ values.append(v)
+
+ for o in itertools.chain(keys, values):
+ self._flatten(o)
+
+ elif isinstance(value, (list, tuple)):
+ for o in value:
+ self._flatten(o)
+
+ def _getrefnum(self, value):
+ try:
+ if isinstance(value, Data):
+ return self._objtable[(type(value.data), value.data)]
+ else:
+ return self._objtable[(type(value), value)]
+ except TypeError:
+ return self._objidtable[id(value)]
+
+ def _write_size(self, token, size):
+ if size < 15:
+ self._fp.write(struct.pack('>B', token | size))
+
+ elif size < 1 << 8:
+ self._fp.write(struct.pack('>BBB', token | 0xF, 0x10, size))
+
+ elif size < 1 << 16:
+ self._fp.write(struct.pack('>BBH', token | 0xF, 0x11, size))
+
+ elif size < 1 << 32:
+ self._fp.write(struct.pack('>BBL', token | 0xF, 0x12, size))
+
+ else:
+ self._fp.write(struct.pack('>BBQ', token | 0xF, 0x13, size))
+
+ def _write_object(self, value):
+ ref = self._getrefnum(value)
+ self._object_offsets[ref] = self._fp.tell()
+ if value is None:
+ self._fp.write(b'\x00')
+
+ elif value is False:
+ self._fp.write(b'\x08')
+
+ elif value is True:
+ self._fp.write(b'\x09')
+
+ elif isinstance(value, int):
+ if value < 0:
+ try:
+ self._fp.write(struct.pack('>Bq', 0x13, value))
+ except struct.error:
+ raise OverflowError(value) from None
+ elif value < 1 << 8:
+ self._fp.write(struct.pack('>BB', 0x10, value))
+ elif value < 1 << 16:
+ self._fp.write(struct.pack('>BH', 0x11, value))
+ elif value < 1 << 32:
+ self._fp.write(struct.pack('>BL', 0x12, value))
+ elif value < 1 << 63:
+ self._fp.write(struct.pack('>BQ', 0x13, value))
+ elif value < 1 << 64:
+ self._fp.write(b'\x14' + value.to_bytes(16, 'big', signed=True))
+ else:
+ raise OverflowError(value)
+
+ elif isinstance(value, float):
+ self._fp.write(struct.pack('>Bd', 0x23, value))
+
+ elif isinstance(value, datetime.datetime):
+ f = (value - datetime.datetime(2001, 1, 1)).total_seconds()
+ self._fp.write(struct.pack('>Bd', 0x33, f))
+
+ elif isinstance(value, Data):
+ self._write_size(0x40, len(value.data))
+ self._fp.write(value.data)
+
+ elif isinstance(value, (bytes, bytearray)):
+ self._write_size(0x40, len(value))
+ self._fp.write(value)
+
+ elif isinstance(value, str):
+ try:
+ t = value.encode('ascii')
+ self._write_size(0x50, len(value))
+ except UnicodeEncodeError:
+ t = value.encode('utf-16be')
+ self._write_size(0x60, len(value))
+
+ self._fp.write(t)
+
+ elif isinstance(value, (list, tuple)):
+ refs = [self._getrefnum(o) for o in value]
+ s = len(refs)
+ self._write_size(0xA0, s)
+ self._fp.write(struct.pack('>' + self._ref_format * s, *refs))
+
+ elif isinstance(value, dict):
+ keyRefs, valRefs = [], []
+
+ if self._sort_keys:
+ rootItems = sorted(value.items())
+ else:
+ rootItems = value.items()
+
+ for k, v in rootItems:
+ if not isinstance(k, str):
+ if self._skipkeys:
+ continue
+ raise TypeError("keys must be strings")
+ keyRefs.append(self._getrefnum(k))
+ valRefs.append(self._getrefnum(v))
+
+ s = len(keyRefs)
+ self._write_size(0xD0, s)
+ self._fp.write(struct.pack('>' + self._ref_format * s, *keyRefs))
+ self._fp.write(struct.pack('>' + self._ref_format * s, *valRefs))
+
+ else:
+ raise TypeError(value)
+
+
+def _is_fmt_binary(header):
+ return header[:8] == b'bplist00'
+
+
+#
+# Generic bits
+#
+
+_FORMATS={
+ FMT_XML: dict(
+ detect=_is_fmt_xml,
+ parser=_PlistParser,
+ writer=_PlistWriter,
+ ),
+ FMT_BINARY: dict(
+ detect=_is_fmt_binary,
+ parser=_BinaryPlistParser,
+ writer=_BinaryPlistWriter,
+ )
+}
+
+
+def load(fp, *, fmt=None, use_builtin_types=True, dict_type=dict):
+ """Read a .plist file. 'fp' should be (readable) file object.
+ Return the unpacked root object (which usually is a dictionary).
+ """
+ if fmt is None:
+ header = fp.read(32)
+ fp.seek(0)
+ for info in _FORMATS.values():
+ if info['detect'](header):
+ p = info['parser'](
+ use_builtin_types=use_builtin_types,
+ dict_type=dict_type,
+ )
+ break
+
+ else:
+ raise InvalidFileException()
+
+ else:
+ p = _FORMATS[fmt]['parser'](use_builtin_types=use_builtin_types)
+
+ return p.parse(fp)
+
+
+def loads(value, *, fmt=None, use_builtin_types=True, dict_type=dict):
+ """Read a .plist file from a bytes object.
+ Return the unpacked root object (which usually is a dictionary).
+ """
+ fp = BytesIO(value)
+ return load(
+ fp, fmt=fmt, use_builtin_types=use_builtin_types, dict_type=dict_type)
+
+
+def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False):
+ """Write 'value' to a .plist file. 'fp' should be a (writable)
+ file object.
+ """
+ if fmt not in _FORMATS:
+ raise ValueError("Unsupported format: %r"%(fmt,))
+
+ writer = _FORMATS[fmt]["writer"](fp, sort_keys=sort_keys, skipkeys=skipkeys)
+ writer.write(value)
+
+
+def dumps(value, *, fmt=FMT_XML, skipkeys=False, sort_keys=True):
+ """Return a bytes object with the contents for a .plist file.
+ """
+ fp = BytesIO()
+ dump(value, fp, fmt=fmt, skipkeys=skipkeys, sort_keys=sort_keys)
+ return fp.getvalue()
diff --git a/Lib/poplib.py b/Lib/poplib.py
index 43f8305..23a3517 100644
--- a/Lib/poplib.py
+++ b/Lib/poplib.py
@@ -13,7 +13,15 @@ Based on the J. Myers POP3 draft, Jan. 96
# Imports
-import re, socket
+import errno
+import re
+import socket
+
+try:
+ import ssl
+ HAVE_SSL = True
+except ImportError:
+ HAVE_SSL = False
__all__ = ["POP3","error_proto"]
@@ -61,6 +69,8 @@ class POP3:
APOP name digest apop(name, digest)
TOP msg n top(msg, n)
UIDL [msg] uidl(msg = None)
+ CAPA capa()
+ STLS stls()
Raises one exception: 'error_proto'.
@@ -87,6 +97,7 @@ class POP3:
timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
self.host = host
self.port = port
+ self._tls_established = False
self.sock = self._create_socket(timeout)
self.file = self.sock.makefile('rb')
self._debugging = 0
@@ -268,7 +279,14 @@ class POP3:
if self.file is not None:
self.file.close()
if self.sock is not None:
- self.sock.close()
+ try:
+ self.sock.shutdown(socket.SHUT_RDWR)
+ except OSError as e:
+ # The server might already have closed the connection
+ if e.errno != errno.ENOTCONN:
+ raise
+ finally:
+ self.sock.close()
self.file = self.sock = None
#__del__ = quit
@@ -324,21 +342,72 @@ class POP3:
return self._shortcmd('UIDL %s' % which)
return self._longcmd('UIDL')
-try:
- import ssl
-except ImportError:
- pass
-else:
+
+ def capa(self):
+ """Return server capabilities (RFC 2449) as a dictionary
+ >>> c=poplib.POP3('localhost')
+ >>> c.capa()
+ {'IMPLEMENTATION': ['Cyrus', 'POP3', 'server', 'v2.2.12'],
+ 'TOP': [], 'LOGIN-DELAY': ['0'], 'AUTH-RESP-CODE': [],
+ 'EXPIRE': ['NEVER'], 'USER': [], 'STLS': [], 'PIPELINING': [],
+ 'UIDL': [], 'RESP-CODES': []}
+ >>>
+
+ Really, according to RFC 2449, the cyrus folks should avoid
+ having the implementation split into multiple arguments...
+ """
+ def _parsecap(line):
+ lst = line.decode('ascii').split()
+ return lst[0], lst[1:]
+
+ caps = {}
+ try:
+ resp = self._longcmd('CAPA')
+ rawcaps = resp[1]
+ for capline in rawcaps:
+ capnm, capargs = _parsecap(capline)
+ caps[capnm] = capargs
+ except error_proto as _err:
+ raise error_proto('-ERR CAPA not supported by server')
+ return caps
+
+
+ def stls(self, context=None):
+ """Start a TLS session on the active connection as specified in RFC 2595.
+
+ context - a ssl.SSLContext
+ """
+ if not HAVE_SSL:
+ raise error_proto('-ERR TLS support missing')
+ if self._tls_established:
+ raise error_proto('-ERR TLS session already established')
+ caps = self.capa()
+ if not 'STLS' in caps:
+ raise error_proto('-ERR STLS not supported by server')
+ if context is None:
+ context = ssl._create_stdlib_context()
+ resp = self._shortcmd('STLS')
+ server_hostname = self.host if ssl.HAS_SNI else None
+ self.sock = context.wrap_socket(self.sock,
+ server_hostname=server_hostname)
+ self.file = self.sock.makefile('rb')
+ self._tls_established = True
+ return resp
+
+
+if HAVE_SSL:
class POP3_SSL(POP3):
"""POP3 client class over SSL connection
- Instantiate with: POP3_SSL(hostname, port=995, keyfile=None, certfile=None)
+ Instantiate with: POP3_SSL(hostname, port=995, keyfile=None, certfile=None,
+ context=None)
hostname - the hostname of the pop3 over ssl server
port - port number
keyfile - PEM formatted file that contains your private key
certfile - PEM formatted certificate chain file
+ context - a ssl.SSLContext
See the methods of the parent class POP3 for more documentation.
"""
@@ -353,17 +422,26 @@ else:
"exclusive")
self.keyfile = keyfile
self.certfile = certfile
+ if context is None:
+ context = ssl._create_stdlib_context(certfile=certfile,
+ keyfile=keyfile)
self.context = context
POP3.__init__(self, host, port, timeout)
def _create_socket(self, timeout):
sock = POP3._create_socket(self, timeout)
- if self.context is not None:
- sock = self.context.wrap_socket(sock)
- else:
- sock = ssl.wrap_socket(sock, self.keyfile, self.certfile)
+ server_hostname = self.host if ssl.HAS_SNI else None
+ sock = self.context.wrap_socket(sock,
+ server_hostname=server_hostname)
return sock
+ def stls(self, keyfile=None, certfile=None, context=None):
+ """The method unconditionally raises an exception since the
+ STLS command doesn't make any sense on an already established
+ SSL/TLS session.
+ """
+ raise error_proto('-ERR TLS session already established')
+
__all__.append("POP3_SSL")
if __name__ == "__main__":
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
index fd63f97..3e13239 100644
--- a/Lib/posixpath.py
+++ b/Lib/posixpath.py
@@ -162,7 +162,7 @@ def islink(path):
"""Test whether a path is a symbolic link"""
try:
st = os.lstat(path)
- except (os.error, AttributeError):
+ except (OSError, AttributeError):
return False
return stat.S_ISLNK(st.st_mode)
@@ -172,56 +172,35 @@ def lexists(path):
"""Test whether a path exists. Returns True for broken symbolic links"""
try:
os.lstat(path)
- except os.error:
+ except OSError:
return False
return True
-# Are two filenames really pointing to the same file?
-
-def samefile(f1, f2):
- """Test whether two pathnames reference the same actual file"""
- s1 = os.stat(f1)
- s2 = os.stat(f2)
- return samestat(s1, s2)
-
-
-# Are two open files really referencing the same file?
-# (Not necessarily the same file descriptor!)
-
-def sameopenfile(fp1, fp2):
- """Test whether two open file objects reference the same file"""
- s1 = os.fstat(fp1)
- s2 = os.fstat(fp2)
- return samestat(s1, s2)
-
-
-# Are two stat buffers (obtained from stat, fstat or lstat)
-# describing the same file?
-
-def samestat(s1, s2):
- """Test whether two stat buffers reference the same file"""
- return s1.st_ino == s2.st_ino and \
- s1.st_dev == s2.st_dev
-
-
# Is a path a mount point?
# (Does this work for all UNIXes? Is it even guaranteed to work by Posix?)
def ismount(path):
"""Test whether a path is a mount point"""
- if islink(path):
- # A symlink can never be a mount point
- return False
try:
s1 = os.lstat(path)
- if isinstance(path, bytes):
- parent = join(path, b'..')
- else:
- parent = join(path, '..')
+ except OSError:
+ # It doesn't exist -- so not a mount point. :-)
+ return False
+ else:
+ # A symlink can never be a mount point
+ if stat.S_ISLNK(s1.st_mode):
+ return False
+
+ if isinstance(path, bytes):
+ parent = join(path, b'..')
+ else:
+ parent = join(path, '..')
+ try:
s2 = os.lstat(parent)
- except os.error:
- return False # It doesn't exist -- so not a mount point :-)
+ except OSError:
+ return False
+
dev1 = s1.st_dev
dev2 = s2.st_dev
if dev1 != dev2:
diff --git a/Lib/pprint.py b/Lib/pprint.py
index 22be0b4b..3be9c36 100644
--- a/Lib/pprint.py
+++ b/Lib/pprint.py
@@ -34,6 +34,7 @@ saferepr()
"""
+import re
import sys as _sys
from collections import OrderedDict as _OrderedDict
from io import StringIO as _StringIO
@@ -41,22 +42,19 @@ from io import StringIO as _StringIO
__all__ = ["pprint","pformat","isreadable","isrecursive","saferepr",
"PrettyPrinter"]
-# cache these for faster access:
-_commajoin = ", ".join
-_id = id
-_len = len
-_type = type
-
-def pprint(object, stream=None, indent=1, width=80, depth=None):
+def pprint(object, stream=None, indent=1, width=80, depth=None, *,
+ compact=False):
"""Pretty-print a Python object to a stream [default is sys.stdout]."""
printer = PrettyPrinter(
- stream=stream, indent=indent, width=width, depth=depth)
+ stream=stream, indent=indent, width=width, depth=depth,
+ compact=compact)
printer.pprint(object)
-def pformat(object, indent=1, width=80, depth=None):
+def pformat(object, indent=1, width=80, depth=None, *, compact=False):
"""Format a Python object into a pretty-printed representation."""
- return PrettyPrinter(indent=indent, width=width, depth=depth).pformat(object)
+ return PrettyPrinter(indent=indent, width=width, depth=depth,
+ compact=compact).pformat(object)
def saferepr(object):
"""Version of repr() which can handle recursive data structures."""
@@ -101,7 +99,8 @@ def _safe_tuple(t):
return _safe_key(t[0]), _safe_key(t[1])
class PrettyPrinter:
- def __init__(self, indent=1, width=80, depth=None, stream=None):
+ def __init__(self, indent=1, width=80, depth=None, stream=None, *,
+ compact=False):
"""Handle pretty printing operations onto a stream using a set of
configured parameters.
@@ -118,6 +117,9 @@ class PrettyPrinter:
The desired output stream. If omitted (or false), the standard
output stream available at construction will be used.
+ compact
+ If true, several items will be combined in one line.
+
"""
indent = int(indent)
width = int(width)
@@ -131,6 +133,7 @@ class PrettyPrinter:
self._stream = stream
else:
self._stream = _sys.stdout
+ self._compact = bool(compact)
def pprint(self, object):
self._format(object, self._stream, 0, 0, {}, 0)
@@ -150,28 +153,25 @@ class PrettyPrinter:
def _format(self, object, stream, indent, allowance, context, level):
level = level + 1
- objid = _id(object)
+ objid = id(object)
if objid in context:
stream.write(_recursion(object))
self._recursive = True
self._readable = False
return
rep = self._repr(object, context, level - 1)
- typ = _type(object)
- sepLines = _len(rep) > (self._width - 1 - indent - allowance)
+ typ = type(object)
+ max_width = self._width - 1 - indent - allowance
+ sepLines = len(rep) > max_width
write = stream.write
- if self._depth and level > self._depth:
- write(rep)
- return
-
if sepLines:
r = getattr(typ, "__repr__", None)
if issubclass(typ, dict):
write('{')
if self._indent_per_level > 1:
write((self._indent_per_level - 1) * ' ')
- length = _len(object)
+ length = len(object)
if length:
context[objid] = 1
indent = indent + self._indent_per_level
@@ -183,13 +183,13 @@ class PrettyPrinter:
rep = self._repr(key, context, level)
write(rep)
write(': ')
- self._format(ent, stream, indent + _len(rep) + 2,
+ self._format(ent, stream, indent + len(rep) + 2,
allowance + 1, context, level)
if length > 1:
for key, ent in items[1:]:
rep = self._repr(key, context, level)
write(',\n%s%s: ' % (' '*indent, rep))
- self._format(ent, stream, indent + _len(rep) + 2,
+ self._format(ent, stream, indent + len(rep) + 2,
allowance + 1, context, level)
indent = indent - self._indent_per_level
del context[objid]
@@ -201,7 +201,7 @@ class PrettyPrinter:
(issubclass(typ, set) and r is set.__repr__) or
(issubclass(typ, frozenset) and r is frozenset.__repr__)
):
- length = _len(object)
+ length = len(object)
if issubclass(typ, list):
write('[')
endchar = ']'
@@ -225,23 +225,71 @@ class PrettyPrinter:
write((self._indent_per_level - 1) * ' ')
if length:
context[objid] = 1
- indent = indent + self._indent_per_level
- self._format(object[0], stream, indent, allowance + 1,
- context, level)
- if length > 1:
- for ent in object[1:]:
- write(',\n' + ' '*indent)
- self._format(ent, stream, indent,
- allowance + 1, context, level)
- indent = indent - self._indent_per_level
+ self._format_items(object, stream,
+ indent + self._indent_per_level,
+ allowance + 1, context, level)
del context[objid]
if issubclass(typ, tuple) and length == 1:
write(',')
write(endchar)
return
+ if issubclass(typ, str) and len(object) > 0 and r is str.__repr__:
+ def _str_parts(s):
+ """
+ Return a list of string literals comprising the repr()
+ of the given string using literal concatenation.
+ """
+ lines = s.splitlines(True)
+ for i, line in enumerate(lines):
+ rep = repr(line)
+ if len(rep) <= max_width:
+ yield rep
+ else:
+ # A list of alternating (non-space, space) strings
+ parts = re.split(r'(\s+)', line) + ['']
+ current = ''
+ for i in range(0, len(parts), 2):
+ part = parts[i] + parts[i+1]
+ candidate = current + part
+ if len(repr(candidate)) > max_width:
+ if current:
+ yield repr(current)
+ current = part
+ else:
+ current = candidate
+ if current:
+ yield repr(current)
+ for i, rep in enumerate(_str_parts(object)):
+ if i > 0:
+ write('\n' + ' '*indent)
+ write(rep)
+ return
write(rep)
+ def _format_items(self, items, stream, indent, allowance, context, level):
+ write = stream.write
+ delimnl = ',\n' + ' ' * indent
+ delim = ''
+ width = max_width = self._width - indent - allowance + 2
+ for ent in items:
+ if self._compact:
+ rep = self._repr(ent, context, level)
+ w = len(rep) + 2
+ if width < w:
+ width = max_width
+ if delim:
+ delim = delimnl
+ if width >= w:
+ width -= w
+ write(delim)
+ delim = ', '
+ write(rep)
+ continue
+ write(delim)
+ delim = delimnl
+ self._format(ent, stream, indent, allowance, context, level)
+
def _repr(self, object, context, level):
repr, readable, recursive = self.format(object, context.copy(),
self._depth, level)
@@ -262,7 +310,7 @@ class PrettyPrinter:
# Return triple (repr_string, isreadable, isrecursive).
def _safe_repr(object, context, maxlevels, level):
- typ = _type(object)
+ typ = type(object)
if typ is str:
if 'locale' not in _sys.modules:
return repr(object), True, False
@@ -286,7 +334,7 @@ def _safe_repr(object, context, maxlevels, level):
if issubclass(typ, dict) and r is dict.__repr__:
if not object:
return "{}", True, False
- objid = _id(object)
+ objid = id(object)
if maxlevels and level >= maxlevels:
return "{...}", False, objid in context
if objid in context:
@@ -307,7 +355,7 @@ def _safe_repr(object, context, maxlevels, level):
if krecur or vrecur:
recursive = True
del context[objid]
- return "{%s}" % _commajoin(components), readable, recursive
+ return "{%s}" % ", ".join(components), readable, recursive
if (issubclass(typ, list) and r is list.__repr__) or \
(issubclass(typ, tuple) and r is tuple.__repr__):
@@ -315,13 +363,13 @@ def _safe_repr(object, context, maxlevels, level):
if not object:
return "[]", True, False
format = "[%s]"
- elif _len(object) == 1:
+ elif len(object) == 1:
format = "(%s,)"
else:
if not object:
return "()", True, False
format = "(%s)"
- objid = _id(object)
+ objid = id(object)
if maxlevels and level >= maxlevels:
return format % "...", False, objid in context
if objid in context:
@@ -340,7 +388,7 @@ def _safe_repr(object, context, maxlevels, level):
if orecur:
recursive = True
del context[objid]
- return format % _commajoin(components), readable, recursive
+ return format % ", ".join(components), readable, recursive
rep = repr(object)
return rep, (rep and not rep.startswith('<')), False
@@ -348,7 +396,7 @@ def _safe_repr(object, context, maxlevels, level):
def _recursion(object):
return ("<Recursion on %s with id=%s>"
- % (_type(object).__name__, _id(object)))
+ % (type(object).__name__, id(object)))
def _perfcheck(object=None):
diff --git a/Lib/profile.py b/Lib/profile.py
index 743e77d..5d0e968 100755
--- a/Lib/profile.py
+++ b/Lib/profile.py
@@ -40,6 +40,40 @@ __all__ = ["run", "runctx", "Profile"]
# return i_count
#itimes = integer_timer # replace with C coded timer returning integers
+class _Utils:
+ """Support class for utility functions which are shared by
+ profile.py and cProfile.py modules.
+ Not supposed to be used directly.
+ """
+
+ def __init__(self, profiler):
+ self.profiler = profiler
+
+ def run(self, statement, filename, sort):
+ prof = self.profiler()
+ try:
+ prof.run(statement)
+ except SystemExit:
+ pass
+ finally:
+ self._show(prof, filename, sort)
+
+ def runctx(self, statement, globals, locals, filename, sort):
+ prof = self.profiler()
+ try:
+ prof.runctx(statement, globals, locals)
+ except SystemExit:
+ pass
+ finally:
+ self._show(prof, filename, sort)
+
+ def _show(self, prof, filename, sort):
+ if filename is not None:
+ prof.dump_stats(filename)
+ else:
+ prof.print_stats(sort)
+
+
#**************************************************************************
# The following are the static member functions for the profiler class
# Note that an instance of Profile() is *not* needed to call them.
@@ -56,15 +90,7 @@ def run(statement, filename=None, sort=-1):
standard name string (file/line/function-name) that is presented in
each line.
"""
- prof = Profile()
- try:
- prof = prof.run(statement)
- except SystemExit:
- pass
- if filename is not None:
- prof.dump_stats(filename)
- else:
- return prof.print_stats(sort)
+ return _Utils(Profile).run(statement, filename, sort)
def runctx(statement, globals, locals, filename=None, sort=-1):
"""Run statement under profiler, supplying your own globals and locals,
@@ -72,16 +98,8 @@ def runctx(statement, globals, locals, filename=None, sort=-1):
statement and filename have the same semantics as profile.run
"""
- prof = Profile()
- try:
- prof = prof.runctx(statement, globals, locals)
- except SystemExit:
- pass
-
- if filename is not None:
- prof.dump_stats(filename)
- else:
- return prof.print_stats(sort)
+ return _Utils(Profile).runctx(statement, globals, locals, filename, sort)
+
class Profile:
"""Profiler class.
@@ -373,10 +391,9 @@ class Profile:
print_stats()
def dump_stats(self, file):
- f = open(file, 'wb')
- self.create_stats()
- marshal.dump(self.stats, f)
- f.close()
+ with open(file, 'wb') as f:
+ self.create_stats()
+ marshal.dump(self.stats, f)
def create_stats(self):
self.simulate_cmd_complete()
diff --git a/Lib/pstats.py b/Lib/pstats.py
index 6a77605..e1ec355 100644
--- a/Lib/pstats.py
+++ b/Lib/pstats.py
@@ -93,9 +93,8 @@ class Stats:
self.stats = {}
return
elif isinstance(arg, str):
- f = open(arg, 'rb')
- self.stats = marshal.load(f)
- f.close()
+ with open(arg, 'rb') as f:
+ self.stats = marshal.load(f)
try:
file_stats = os.stat(arg)
arg = time.ctime(file_stats.st_mtime) + " " + arg
@@ -149,11 +148,8 @@ class Stats:
def dump_stats(self, filename):
"""Write the profile data to a file we know how to load back."""
- f = open(filename, 'wb')
- try:
+ with open(filename, 'wb') as f:
marshal.dump(self.stats, f)
- finally:
- f.close()
# list the tuple indices and directions for sorting,
# along with some printable description
@@ -612,7 +608,7 @@ if __name__ == '__main__':
if line:
try:
self.stats = Stats(line)
- except IOError as err:
+ except OSError as err:
print(err.args[1], file=self.stream)
return
except Exception as err:
diff --git a/Lib/pty.py b/Lib/pty.py
index 3ccf619..e841f12 100644
--- a/Lib/pty.py
+++ b/Lib/pty.py
@@ -47,27 +47,16 @@ def master_open():
return _open_terminal()
def _open_terminal():
- """Open pty master and return (master_fd, tty_name).
- SGI and generic BSD version, for when openpty() fails."""
- try:
- import sgi
- except ImportError:
- pass
- else:
- try:
- tty_name, master_fd = sgi._getpty(os.O_RDWR, 0o666, 0)
- except IOError as msg:
- raise os.error(msg)
- return master_fd, tty_name
+ """Open pty master and return (master_fd, tty_name)."""
for x in 'pqrstuvwxyzPQRST':
for y in '0123456789abcdef':
pty_name = '/dev/pty' + x + y
try:
fd = os.open(pty_name, os.O_RDWR)
- except os.error:
+ except OSError:
continue
return (fd, '/dev/tty' + x + y)
- raise os.error('out of pty devices')
+ raise OSError('out of pty devices')
def slave_open(tty_name):
"""slave_open(tty_name) -> slave_fd
@@ -83,7 +72,7 @@ def slave_open(tty_name):
try:
ioctl(result, I_PUSH, "ptem")
ioctl(result, I_PUSH, "ldterm")
- except IOError:
+ except OSError:
pass
return result
@@ -173,8 +162,9 @@ def spawn(argv, master_read=_read, stdin_read=_read):
restore = 0
try:
_copy(master_fd, master_read, stdin_read)
- except (IOError, OSError):
+ except OSError:
if restore:
tty.tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
os.close(master_fd)
+ return os.waitpid(pid, 0)[1]
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index 62d69ad..1277b93 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -3,17 +3,14 @@
This module has intimate knowledge of the format of .pyc files.
"""
-import builtins
-import errno
-import imp
-import marshal
+import importlib._bootstrap
+import importlib.machinery
+import importlib.util
import os
+import os.path
import sys
-import tokenize
import traceback
-MAGIC = imp.get_magic()
-
__all__ = ["compile", "main", "PyCompileError"]
@@ -65,13 +62,6 @@ class PyCompileError(Exception):
return self.msg
-def wr_long(f, x):
- """Internal; write a 32-bit int to a file in little-endian order."""
- f.write(bytes([x & 0xff,
- (x >> 8) & 0xff,
- (x >> 16) & 0xff,
- (x >> 24) & 0xff]))
-
def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
"""Byte-compile one Python source file to Python bytecode.
@@ -107,18 +97,31 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
See compileall.py for a script/module that uses this module to
byte-compile all installed files (or all files in selected
directories).
+
+ Do note that FileExistsError is raised if cfile ends up pointing at a
+ non-regular file or symlink. Because the compilation uses a file renaming,
+ the resulting file would be regular and thus not the same type of file as
+ it was previously.
"""
- with tokenize.open(file) as f:
- try:
- st = os.fstat(f.fileno())
- except AttributeError:
- st = os.stat(file)
- timestamp = int(st.st_mtime)
- size = st.st_size & 0xFFFFFFFF
- codestring = f.read()
+ if cfile is None:
+ if optimize >= 0:
+ cfile = importlib.util.cache_from_source(file,
+ debug_override=not optimize)
+ else:
+ cfile = importlib.util.cache_from_source(file)
+ if os.path.islink(cfile):
+ msg = ('{} is a symlink and will be changed into a regular file if '
+ 'import writes a byte-compiled file to it')
+ raise FileExistsError(msg.format(cfile))
+ elif os.path.exists(cfile) and not os.path.isfile(cfile):
+ msg = ('{} is a non-regular file and will be changed into a regular '
+ 'one if import writes a byte-compiled file to it')
+ raise FileExistsError(msg.format(cfile))
+ loader = importlib.machinery.SourceFileLoader('<py_compile>', file)
+ source_bytes = loader.get_data(file)
try:
- codeobject = builtins.compile(codestring, dfile or file, 'exec',
- optimize=optimize)
+ code = loader.source_to_code(source_bytes, dfile or file,
+ _optimize=optimize)
except Exception as err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
@@ -126,28 +129,20 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
else:
sys.stderr.write(py_exc.msg + '\n')
return
- if cfile is None:
- if optimize >= 0:
- cfile = imp.cache_from_source(file, debug_override=not optimize)
- else:
- cfile = imp.cache_from_source(file)
try:
dirname = os.path.dirname(cfile)
if dirname:
os.makedirs(dirname)
- except OSError as error:
- if error.errno != errno.EEXIST:
- raise
- with open(cfile, 'wb') as fc:
- fc.write(b'\0\0\0\0')
- wr_long(fc, timestamp)
- wr_long(fc, size)
- marshal.dump(codeobject, fc)
- fc.flush()
- fc.seek(0, 0)
- fc.write(MAGIC)
+ except FileExistsError:
+ pass
+ source_stats = loader.path_stats(file)
+ bytecode = importlib._bootstrap._code_to_bytecode(
+ code, source_stats['mtime'], source_stats['size'])
+ mode = importlib._bootstrap._calc_mode(file)
+ importlib._bootstrap._write_atomic(cfile, bytecode, mode)
return cfile
+
def main(args=None):
"""Compile several source files.
@@ -173,7 +168,7 @@ def main(args=None):
except PyCompileError as error:
rv = 1
sys.stderr.write("%s\n" % error.msg)
- except IOError as error:
+ except OSError as error:
rv = 1
sys.stderr.write("%s\n" % error)
else:
diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py
index 9ec05ee..dd58ada 100644
--- a/Lib/pyclbr.py
+++ b/Lib/pyclbr.py
@@ -42,7 +42,7 @@ Instances of this class have the following instance variables:
import io
import os
import sys
-import importlib
+import importlib.util
import tokenize
from token import NAME, DEDENT, OP
from operator import itemgetter
@@ -140,13 +140,14 @@ def _readmodule(module, path, inpackage=None):
search_path = path
else:
search_path = path + sys.path
- loader = importlib.find_loader(fullmodule, search_path)
- fname = loader.get_filename(fullmodule)
+ # XXX This will change once issue19944 lands.
+ spec = importlib.util._find_spec_from_path(fullmodule, search_path)
+ fname = spec.loader.get_filename(fullmodule)
_modules[fullmodule] = dict
- if loader.is_package(fullmodule):
+ if spec.loader.is_package(fullmodule):
dict['__path__'] = [os.path.dirname(fname)]
try:
- source = loader.get_source(fullmodule)
+ source = spec.loader.get_source(fullmodule)
if source is None:
return dict
except (AttributeError, ImportError):
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 9dce079..006212b 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -44,16 +44,16 @@ Richard Chamberlain, for the first implementation of textdoc.
"""
# Known bugs that can't be fixed here:
-# - imp.load_module() cannot be prevented from clobbering existing
-# loaded modules, so calling synopsis() on a binary module file
-# changes the contents of any existing module with the same name.
+# - synopsis() cannot be prevented from clobbering existing
+# loaded modules.
# - If the __file__ attribute on a module is a relative path and
# the current directory is changed with os.chdir(), an incorrect
# path will be displayed.
import builtins
-import imp
+import importlib._bootstrap
import importlib.machinery
+import importlib.util
import inspect
import io
import os
@@ -66,7 +66,7 @@ import tokenize
import warnings
from collections import deque
from reprlib import Repr
-from traceback import extract_tb, format_exception_only
+from traceback import format_exception_only
# --------------------------------------------------------- common routines
@@ -137,6 +137,19 @@ def _is_some_method(obj):
inspect.isbuiltin(obj) or
inspect.ismethoddescriptor(obj))
+def _is_bound_method(fn):
+ """
+ Returns True if fn is a bound method, regardless of whether
+ fn was implemented in Python or in C.
+ """
+ if inspect.ismethod(fn):
+ return True
+ if inspect.isbuiltin(fn):
+ self = getattr(fn, '__self__', None)
+ return not (inspect.ismodule(self) or (self is None))
+ return False
+
+
def allmethods(cl):
methods = {}
for key, value in inspect.getmembers(cl, _is_some_method):
@@ -167,8 +180,9 @@ def _split_list(s, predicate):
def visiblename(name, all=None, obj=None):
"""Decide whether to show documentation on a variable."""
# Certain special names are redundant or internal.
+ # XXX Remove __initializing__?
if name in {'__author__', '__builtins__', '__cached__', '__credits__',
- '__date__', '__doc__', '__file__', '__initializing__',
+ '__date__', '__doc__', '__file__', '__spec__',
'__loader__', '__module__', '__name__', '__package__',
'__path__', '__qualname__', '__slots__', '__version__'}:
return 0
@@ -224,34 +238,38 @@ def synopsis(filename, cache={}):
mtime = os.stat(filename).st_mtime
lastupdate, result = cache.get(filename, (None, None))
if lastupdate is None or lastupdate < mtime:
- try:
- file = tokenize.open(filename)
- except IOError:
- # module can't be opened, so skip it
- return None
- binary_suffixes = importlib.machinery.BYTECODE_SUFFIXES[:]
- binary_suffixes += importlib.machinery.EXTENSION_SUFFIXES[:]
- if any(filename.endswith(x) for x in binary_suffixes):
- # binary modules have to be imported
- file.close()
- if any(filename.endswith(x) for x in
- importlib.machinery.BYTECODE_SUFFIXES):
- loader = importlib.machinery.SourcelessFileLoader('__temp__',
- filename)
- else:
- loader = importlib.machinery.ExtensionFileLoader('__temp__',
- filename)
+ # Look for binary suffixes first, falling back to source.
+ if filename.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)):
+ loader_cls = importlib.machinery.SourcelessFileLoader
+ elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)):
+ loader_cls = importlib.machinery.ExtensionFileLoader
+ else:
+ loader_cls = None
+ # Now handle the choice.
+ if loader_cls is None:
+ # Must be a source file.
try:
- module = loader.load_module('__temp__')
+ file = tokenize.open(filename)
+ except OSError:
+ # module can't be opened, so skip it
+ return None
+ # text modules can be directly examined
+ with file:
+ result = source_synopsis(file)
+ else:
+ # Must be a binary module, which has to be imported.
+ loader = loader_cls('__temp__', filename)
+ # XXX We probably don't need to pass in the loader here.
+ spec = importlib.util.spec_from_file_location('__temp__', filename,
+ loader=loader)
+ _spec = importlib._bootstrap._SpecMethods(spec)
+ try:
+ module = _spec.load()
except:
return None
- result = (module.__doc__ or '').splitlines()[0]
del sys.modules['__temp__']
- else:
- # text modules can be directly examined
- result = source_synopsis(file)
- file.close()
-
+ result = (module.__doc__ or '').splitlines()[0]
+ # Cache the result.
cache[filename] = (mtime, result)
return result
@@ -267,20 +285,22 @@ class ErrorDuringImport(Exception):
def importfile(path):
"""Import a Python source file or compiled file given its path."""
- magic = imp.get_magic()
+ magic = importlib.util.MAGIC_NUMBER
with open(path, 'rb') as file:
- if file.read(len(magic)) == magic:
- kind = imp.PY_COMPILED
- else:
- kind = imp.PY_SOURCE
- file.seek(0)
- filename = os.path.basename(path)
- name, ext = os.path.splitext(filename)
- try:
- module = imp.load_module(name, file, path, (ext, 'r', kind))
- except:
- raise ErrorDuringImport(path, sys.exc_info())
- return module
+ is_bytecode = magic == file.read(len(magic))
+ filename = os.path.basename(path)
+ name, ext = os.path.splitext(filename)
+ if is_bytecode:
+ loader = importlib._bootstrap.SourcelessFileLoader(name, path)
+ else:
+ loader = importlib._bootstrap.SourceFileLoader(name, path)
+ # XXX We probably don't need to pass in the loader here.
+ spec = importlib.util.spec_from_file_location(name, path, loader=loader)
+ _spec = importlib._bootstrap._SpecMethods(spec)
+ try:
+ return _spec.load()
+ except:
+ raise ErrorDuringImport(path, sys.exc_info())
def safeimport(path, forceload=0, cache={}):
"""Import a module; handle errors; return None if the module isn't found.
@@ -891,7 +911,7 @@ class HTMLDoc(Doc):
anchor = (cl and cl.__name__ or '') + '-' + name
note = ''
skipdocs = 0
- if inspect.ismethod(object):
+ if _is_bound_method(object):
imclass = object.__self__.__class__
if cl:
if imclass is not cl:
@@ -902,7 +922,6 @@ class HTMLDoc(Doc):
object.__self__.__class__, mod)
else:
note = ' unbound %s method' % self.classlink(imclass,mod)
- object = object.__func__
if name == realname:
title = '<a name="%s"><strong>%s</strong></a>' % (anchor, realname)
@@ -916,20 +935,21 @@ class HTMLDoc(Doc):
reallink = realname
title = '<a name="%s"><strong>%s</strong></a> = %s' % (
anchor, name, reallink)
- if inspect.isfunction(object):
- args, varargs, kwonlyargs, kwdefaults, varkw, defaults, ann = \
- inspect.getfullargspec(object)
- argspec = inspect.formatargspec(
- args, varargs, kwonlyargs, kwdefaults, varkw, defaults, ann,
- formatvalue=self.formatvalue,
- formatannotation=inspect.formatannotationrelativeto(object))
- if realname == '<lambda>':
- title = '<strong>%s</strong> <em>lambda</em> ' % name
- # XXX lambda's won't usually have func_annotations['return']
- # since the syntax doesn't support but it is possible.
- # So removing parentheses isn't truly safe.
- argspec = argspec[1:-1] # remove parentheses
- else:
+ argspec = None
+ if inspect.isroutine(object):
+ try:
+ signature = inspect.signature(object)
+ except (ValueError, TypeError):
+ signature = None
+ if signature:
+ argspec = str(signature)
+ if realname == '<lambda>':
+ title = '<strong>%s</strong> <em>lambda</em> ' % name
+ # XXX lambda's won't usually have func_annotations['return']
+ # since the syntax doesn't support but it is possible.
+ # So removing parentheses isn't truly safe.
+ argspec = argspec[1:-1] # remove parentheses
+ if not argspec:
argspec = '(...)'
decl = title + argspec + (note and self.grey(
@@ -1236,8 +1256,12 @@ location listed above.
doc = getdoc(value)
else:
doc = None
- push(self.docother(getattr(object, name),
- name, mod, maxlen=70, doc=doc) + '\n')
+ try:
+ obj = getattr(object, name)
+ except AttributeError:
+ obj = homecls.__dict__[name]
+ push(self.docother(obj, name, mod, maxlen=70, doc=doc) +
+ '\n')
return attrs
attrs = [(name, kind, cls, value)
@@ -1259,7 +1283,6 @@ location listed above.
else:
tag = "inherited from %s" % classname(thisclass,
object.__module__)
-
# Sort attrs by name.
attrs.sort()
@@ -1274,6 +1297,7 @@ location listed above.
lambda t: t[1] == 'data descriptor')
attrs = spilldata("Data and other attributes %s:\n" % tag, attrs,
lambda t: t[1] == 'data')
+
assert attrs == []
attrs = inherited
@@ -1292,7 +1316,7 @@ location listed above.
name = name or realname
note = ''
skipdocs = 0
- if inspect.ismethod(object):
+ if _is_bound_method(object):
imclass = object.__self__.__class__
if cl:
if imclass is not cl:
@@ -1303,7 +1327,6 @@ location listed above.
object.__self__.__class__, mod)
else:
note = ' unbound %s method' % classname(imclass,mod)
- object = object.__func__
if name == realname:
title = self.bold(realname)
@@ -1312,20 +1335,22 @@ location listed above.
cl.__dict__[realname] is object):
skipdocs = 1
title = self.bold(name) + ' = ' + realname
- if inspect.isfunction(object):
- args, varargs, varkw, defaults, kwonlyargs, kwdefaults, ann = \
- inspect.getfullargspec(object)
- argspec = inspect.formatargspec(
- args, varargs, varkw, defaults, kwonlyargs, kwdefaults, ann,
- formatvalue=self.formatvalue,
- formatannotation=inspect.formatannotationrelativeto(object))
- if realname == '<lambda>':
- title = self.bold(name) + ' lambda '
- # XXX lambda's won't usually have func_annotations['return']
- # since the syntax doesn't support but it is possible.
- # So removing parentheses isn't truly safe.
- argspec = argspec[1:-1] # remove parentheses
- else:
+ argspec = None
+
+ if inspect.isroutine(object):
+ try:
+ signature = inspect.signature(object)
+ except (ValueError, TypeError):
+ signature = None
+ if signature:
+ argspec = str(signature)
+ if realname == '<lambda>':
+ title = self.bold(name) + ' lambda '
+ # XXX lambda's won't usually have func_annotations['return']
+ # since the syntax doesn't support but it is possible.
+ # So removing parentheses isn't truly safe.
+ argspec = argspec[1:-1] # remove parentheses
+ if not argspec:
argspec = '(...)'
decl = title + argspec + note
@@ -1396,7 +1421,7 @@ def getpager():
return lambda text: pipepager(text, os.environ['PAGER'])
if os.environ.get('TERM') in ('dumb', 'emacs'):
return plainpager
- if sys.platform == 'win32' or sys.platform.startswith('os2'):
+ if sys.platform == 'win32':
return lambda text: tempfilepager(plain(text), 'more <')
if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
return lambda text: pipepager(text, 'less')
@@ -1422,16 +1447,15 @@ def pipepager(text, cmd):
try:
pipe.write(text)
pipe.close()
- except IOError:
+ except OSError:
pass # Ignore broken pipes caused by quitting the pager program.
def tempfilepager(text, cmd):
"""Page through text by invoking a program on a temporary file."""
import tempfile
filename = tempfile.mktemp()
- file = open(filename, 'w')
- file.write(text)
- file.close()
+ with open(filename, 'w') as file:
+ file.write(text)
try:
os.system(cmd + ' "' + filename + '"')
finally:
@@ -1850,10 +1874,10 @@ Enter the name of any module, keyword, or topic to get help on writing
Python programs and using Python modules. To quit this help utility and
return to the interpreter, just type "quit".
-To get a list of available modules, keywords, or topics, type "modules",
-"keywords", or "topics". Each module also comes with a one-line summary
-of what it does; to list the modules whose summaries contain a given word
-such as "spam", type "modules spam".
+To get a list of available modules, keywords, symbols, or topics, type
+"modules", "keywords", "symbols", or "topics". Each module also comes
+with a one-line summary of what it does; to list the modules whose name
+or summary contain a given string such as "spam", type "modules spam".
''' % tuple([sys.version[:3]]*2))
def list(self, items, columns=4, width=80):
@@ -1917,11 +1941,10 @@ module "pydoc_data.topics" could not be found.
if more_xrefs:
xrefs = (xrefs or '') + ' ' + more_xrefs
if xrefs:
- import formatter
- buffer = io.StringIO()
- formatter.DumbWriter(buffer).send_flowing_data(
- 'Related help topics: ' + ', '.join(xrefs.split()) + '\n')
- self.output.write('\n%s\n' % buffer.getvalue())
+ import textwrap
+ text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n'
+ wrapped_text = textwrap.wrap(text, 72)
+ self.output.write('\n%s\n' % ''.join(wrapped_text))
def _gettopic(self, topic, more_xrefs=''):
"""Return unbuffered tuple of (topic, xrefs).
@@ -1958,9 +1981,10 @@ module "pydoc_data.topics" could not be found.
def listmodules(self, key=''):
if key:
self.output.write('''
-Here is a list of matching modules. Enter any module name to get more help.
+Here is a list of modules whose name or summary contains '{}'.
+If there are any, enter a module name to get more help.
-''')
+'''.format(key))
apropos(key)
else:
self.output.write('''
@@ -1979,35 +2003,11 @@ Please wait a moment while I gather a list of all available modules...
self.list(modules.keys())
self.output.write('''
Enter any module name to get more help. Or, type "modules spam" to search
-for modules whose descriptions contain the word "spam".
+for modules whose name or summary contain the string "spam".
''')
help = Helper()
-class Scanner:
- """A generic tree iterator."""
- def __init__(self, roots, children, descendp):
- self.roots = roots[:]
- self.state = []
- self.children = children
- self.descendp = descendp
-
- def next(self):
- if not self.state:
- if not self.roots:
- return None
- root = self.roots.pop(0)
- self.state = [(root, self.children(root))]
- node, children = self.state[-1]
- if not children:
- self.state.pop()
- return self.next()
- child = children.pop(0)
- if self.descendp(child):
- self.state.append((child, self.children(child)))
- return child
-
-
class ModuleScanner:
"""An interruptible scanner that searches module synopses."""
@@ -2036,10 +2036,11 @@ class ModuleScanner:
callback(None, modname, '')
else:
try:
- loader = importer.find_module(modname)
+ spec = pkgutil._get_spec(importer, modname)
except SyntaxError:
# raised by tests for bad coding cookies or BOM
continue
+ loader = spec.loader
if hasattr(loader, 'get_source'):
try:
source = loader.get_source(modname)
@@ -2053,8 +2054,9 @@ class ModuleScanner:
else:
path = None
else:
+ _spec = importlib._bootstrap._SpecMethods(spec)
try:
- module = loader.load_module(modname)
+ module = _spec.load()
except ImportError:
if onerror:
onerror(modname)
diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py
index e2fd380..16c911e 100644
--- a/Lib/pydoc_data/topics.py
+++ b/Lib/pydoc_data/topics.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Sun Feb 23 08:13:25 2014
+# Autogenerated by Sphinx on Sat Mar 15 22:21:45 2014
topics = {'assert': '\nThe "assert" statement\n**********************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, "assert expression", is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, "assert expression1, expression2", is equivalent to\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that "__debug__" and "AssertionError" refer\nto the built-in variables with those names. In the current\nimplementation, the built-in variable "__debug__" is "True" under\nnormal circumstances, "False" when optimization is requested (command\nline option -O). The current code generator emits no code for an\nassert statement when optimization is requested at compile time. Note\nthat it is unnecessary to include the source code for the expression\nthat failed in the error message; it will be displayed as part of the\nstack trace.\n\nAssignments to "__debug__" are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n',
'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n | "*" target\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The\n object must be an iterable with the same number of items as there\n are targets in the target list, and the items are assigned, from\n left to right, to the corresponding targets.\n\n * If the target list contains one target prefixed with an\n asterisk, called a "starred" target: The object must be a sequence\n with at least as many items as there are targets in the target\n list, minus one. The first items of the sequence are assigned,\n from left to right, to the targets before the starred target. The\n final items of the sequence are assigned to the targets after the\n starred target. A list of the remaining items in the sequence is\n then assigned to the starred target (the list can be empty).\n\n * Else: The object must be a sequence with the same number of\n items as there are targets in the target list, and the items are\n assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a "global" or "nonlocal" statement\n in the current code block: the name is bound to the object in the\n current local namespace.\n\n * Otherwise: the name is bound to the object in the global\n namespace or the outer namespace determined by "nonlocal",\n respectively.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in\n square brackets: The object must be an iterable with the same number\n of items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, "TypeError" is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily "AttributeError").\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n "a.x" can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target "a.x" is always\n set as an instance attribute, creating it if necessary. Thus, the\n two occurrences of "a.x" do not necessarily refer to the same\n attribute: if the RHS expression refers to a class attribute, the\n LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with "property()".\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield an integer. If it is negative, the sequence\'s\n length is added to it. The resulting value must be a nonnegative\n integer less than the sequence\'s length, and the sequence is asked\n to assign the assigned object to its item with that index. If the\n index is out of range, "IndexError" is raised (assignment to a\n subscripted sequence cannot add new items to a list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n For user-defined objects, the "__setitem__()" method is called with\n appropriate arguments.\n\n* If the target is a slicing: The primary expression in the\n reference is evaluated. It should yield a mutable sequence object\n (such as a list). The assigned object should be a sequence object\n of the same type. Next, the lower and upper bound expressions are\n evaluated, insofar they are present; defaults are zero and the\n sequence\'s length. The bounds should evaluate to integers. If\n either bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample "a, b = b, a" swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints "[0, 2]":\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print(x)\n\nSee also: **PEP 3132** - Extended Iterable Unpacking\n\n The specification for the "*target" feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like "x += 1" can be rewritten as\n"x = x + 1" to achieve a similar, but not exactly equal effect. In the\naugmented version, "x" is only evaluated once. Also, when possible,\nthe actual operation is performed *in-place*, meaning that rather than\ncreating a new object and assigning that to the target, the old object\nis modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n',
'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a "NameError" exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name. For example, the identifier "__spam"\noccurring in a class named "Ham" will be transformed to "_Ham__spam".\nThis transformation is independent of the syntactical context in which\nthe identifier is used. If the transformed name is extremely long\n(longer than 255 characters), implementation defined truncation may\nhappen. If the class name consists only of underscores, no\ntransformation is done.\n',
@@ -19,12 +19,12 @@ topics = {'assert': '\nThe "assert" statement\n**********************\n\nAssert
'calls': '\nCalls\n*****\n\nA call calls a callable object (e.g., a *function*) with a possibly\nempty series of *arguments*:\n\n call ::= primary "(" [argument_list [","] | comprehension] ")"\n argument_list ::= positional_arguments ["," keyword_arguments]\n ["," "*" expression] ["," keyword_arguments]\n ["," "**" expression]\n | keyword_arguments ["," "*" expression]\n ["," keyword_arguments] ["," "**" expression]\n | "*" expression ["," keyword_arguments] ["," "**" expression]\n | "**" expression\n positional_arguments ::= expression ("," expression)*\n keyword_arguments ::= keyword_item ("," keyword_item)*\n keyword_item ::= identifier "=" expression\n\nA trailing comma may be present after the positional and keyword\narguments but does not affect the semantics.\n\nThe primary must evaluate to a callable object (user-defined\nfunctions, built-in functions, methods of built-in objects, class\nobjects, methods of class instances, and all objects having a\n"__call__()" method are callable). All argument expressions are\nevaluated before the call is attempted. Please refer to section\n*Function definitions* for the syntax of formal *parameter* lists.\n\nIf keyword arguments are present, they are first converted to\npositional arguments, as follows. First, a list of unfilled slots is\ncreated for the formal parameters. If there are N positional\narguments, they are placed in the first N slots. Next, for each\nkeyword argument, the identifier is used to determine the\ncorresponding slot (if the identifier is the same as the first formal\nparameter name, the first slot is used, and so on). If the slot is\nalready filled, a "TypeError" exception is raised. Otherwise, the\nvalue of the argument is placed in the slot, filling it (even if the\nexpression is "None", it fills the slot). When all arguments have\nbeen processed, the slots that are still unfilled are filled with the\ncorresponding default value from the function definition. (Default\nvalues are calculated, once, when the function is defined; thus, a\nmutable object such as a list or dictionary used as default value will\nbe shared by all calls that don\'t specify an argument value for the\ncorresponding slot; this should usually be avoided.) If there are any\nunfilled slots for which no default value is specified, a "TypeError"\nexception is raised. Otherwise, the list of filled slots is used as\nthe argument list for the call.\n\n**CPython implementation detail:** An implementation may provide\nbuilt-in functions whose positional parameters do not have names, even\nif they are \'named\' for the purpose of documentation, and which\ntherefore cannot be supplied by keyword. In CPython, this is the case\nfor functions implemented in C that use "PyArg_ParseTuple()" to parse\ntheir arguments.\n\nIf there are more positional arguments than there are formal parameter\nslots, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "*identifier" is present; in this case, that formal\nparameter receives a tuple containing the excess positional arguments\n(or an empty tuple if there were no excess positional arguments).\n\nIf any keyword argument does not correspond to a formal parameter\nname, a "TypeError" exception is raised, unless a formal parameter\nusing the syntax "**identifier" is present; in this case, that formal\nparameter receives a dictionary containing the excess keyword\narguments (using the keywords as keys and the argument values as\ncorresponding values), or a (new) empty dictionary if there were no\nexcess keyword arguments.\n\nIf the syntax "*expression" appears in the function call, "expression"\nmust evaluate to an iterable. Elements from this iterable are treated\nas if they were additional positional arguments; if there are\npositional arguments *x1*, ..., *xN*, and "expression" evaluates to a\nsequence *y1*, ..., *yM*, this is equivalent to a call with M+N\npositional arguments *x1*, ..., *xN*, *y1*, ..., *yM*.\n\nA consequence of this is that although the "*expression" syntax may\nappear *after* some keyword arguments, it is processed *before* the\nkeyword arguments (and the "**expression" argument, if any -- see\nbelow). So:\n\n >>> def f(a, b):\n ... print(a, b)\n ...\n >>> f(b=1, *(2,))\n 2 1\n >>> f(a=1, *(2,))\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n TypeError: f() got multiple values for keyword argument \'a\'\n >>> f(1, *(2,))\n 1 2\n\nIt is unusual for both keyword arguments and the "*expression" syntax\nto be used in the same call, so in practice this confusion does not\narise.\n\nIf the syntax "**expression" appears in the function call,\n"expression" must evaluate to a mapping, the contents of which are\ntreated as additional keyword arguments. In the case of a keyword\nappearing in both "expression" and as an explicit keyword argument, a\n"TypeError" exception is raised.\n\nFormal parameters using the syntax "*identifier" or "**identifier"\ncannot be used as positional argument slots or as keyword argument\nnames.\n\nA call always returns some value, possibly "None", unless it raises an\nexception. How this value is computed depends on the type of the\ncallable object.\n\nIf it is---\n\na user-defined function:\n The code block for the function is executed, passing it the\n argument list. The first thing the code block will do is bind the\n formal parameters to the arguments; this is described in section\n *Function definitions*. When the code block executes a "return"\n statement, this specifies the return value of the function call.\n\na built-in function or method:\n The result is up to the interpreter; see *Built-in Functions* for\n the descriptions of built-in functions and methods.\n\na class object:\n A new instance of that class is returned.\n\na class instance method:\n The corresponding user-defined function is called, with an argument\n list that is one longer than the argument list of the call: the\n instance becomes the first argument.\n\na class instance:\n The class must define a "__call__()" method; the effect is then the\n same as if that method was called.\n',
'class': '\nClass definitions\n*****************\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [parameter_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with "self.name = value". Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way. Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results. *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n Class Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless\n there is a "finally" clause which happens to raise another\n exception. That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of\n an exception or the execution of a "return", "continue", or\n "break" statement.\n\n[3] A string literal appearing as the first statement in the\n function body is transformed into the function\'s "__doc__"\n attribute and therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s "__doc__" item and\n therefore the class\'s *docstring*.\n',
'comparisons': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like "a < b < c" have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: "True" or "False".\n\nComparisons can be chained arbitrarily, e.g., "x < y <= z" is\nequivalent to "x < y and y <= z", except that "y" is evaluated only\nonce (but in both cases "z" is not evaluated at all when "x < y" is\nfound to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then "a op1 b op2 c ... y\nopN z" is equivalent to "a op1 b and b op2 c and ... y opN z", except\nthat each expression is evaluated at most once.\n\nNote that "a op1 b op2 c" doesn\'t imply any kind of comparison between\n*a* and *c*, so that, e.g., "x < y > z" is perfectly legal (though\nperhaps not pretty).\n\nThe operators "<", ">", "==", ">=", "<=", and "!=" compare the values\nof two objects. The objects need not have the same type. If both are\nnumbers, they are converted to a common type. Otherwise, the "==" and\n"!=" operators *always* consider objects of different types to be\nunequal, while the "<", ">", ">=" and "<=" operators raise a\n"TypeError" when comparing objects of different types that do not\nimplement these operators for the given pair of types. You can\ncontrol comparison behavior of objects of non-built-in types by\ndefining rich comparison methods like "__gt__()", described in section\n*Basic customization*.\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* The values "float(\'NaN\')" and "Decimal(\'NaN\')" are special. The\n are identical to themselves, "x is x" but are not equal to\n themselves, "x != x". Additionally, comparing any value to a\n not-a-number value will return "False". For example, both "3 <\n float(\'NaN\')" and "float(\'NaN\') < 3" will return "False".\n\n* Bytes objects are compared lexicographically using the numeric\n values of their elements.\n\n* Strings are compared lexicographically using the numeric\n equivalents (the result of the built-in function "ord()") of their\n characters. [3] String and bytes object can\'t be compared!\n\n* Tuples and lists are compared lexicographically using comparison\n of corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, "[1,2,x] <= [1,2,y]" has the same\n value as "x <= y". If the corresponding element does not exist, the\n shorter sequence is ordered first (for example, "[1,2] < [1,2,3]").\n\n* Mappings (dictionaries) compare equal if and only if they have the\n same "(key, value)" pairs. Order comparisons "(\'<\', \'<=\', \'>=\',\n \'>\')" raise "TypeError".\n\n* Sets and frozensets define comparison operators to mean subset and\n superset tests. Those relations do not define total orderings (the\n two sets "{1,2}" and {2,3} are not equal, nor subsets of one\n another, nor supersets of one another). Accordingly, sets are not\n appropriate arguments for functions which depend on total ordering.\n For example, "min()", "max()", and "sorted()" produce undefined\n results given a list of sets as inputs.\n\n* Most other objects of built-in types compare unequal unless they\n are the same object; the choice whether one object is considered\n smaller or larger than another one is made arbitrarily but\n consistently within one execution of a program.\n\nComparison of objects of the differing types depends on whether either\nof the types provide explicit support for the comparison. Most\nnumeric types can be compared with one another. When cross-type\ncomparison is not supported, the comparison method returns\n"NotImplemented".\n\nThe operators "in" and "not in" test for membership. "x in s"\nevaluates to true if *x* is a member of *s*, and false otherwise. "x\nnot in s" returns the negation of "x in s". All built-in sequences\nand set types support this as well as dictionary, for which "in" tests\nwhether a the dictionary has a given key. For container types such as\nlist, tuple, set, frozenset, dict, or collections.deque, the\nexpression "x in y" is equivalent to "any(x is e or x == e for e in\ny)".\n\nFor the string and bytes types, "x in y" is true if and only if *x* is\na substring of *y*. An equivalent test is "y.find(x) != -1". Empty\nstrings are always considered to be a substring of any other string,\nso """ in "abc"" will return "True".\n\nFor user-defined classes which define the "__contains__()" method, "x\nin y" is true if and only if "y.__contains__(x)" is true.\n\nFor user-defined classes which do not define "__contains__()" but do\ndefine "__iter__()", "x in y" is true if some value "z" with "x == z"\nis produced while iterating over "y". If an exception is raised\nduring the iteration, it is as if "in" raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n"__getitem__()", "x in y" is true if and only if there is a non-\nnegative integer index *i* such that "x == y[i]", and all lower\ninteger indices do not raise "IndexError" exception. (If any other\nexception is raised, it is as if "in" raised that exception).\n\nThe operator "not in" is defined to have the inverse true value of\n"in".\n\nThe operators "is" and "is not" test for object identity: "x is y" is\ntrue if and only if *x* and *y* are the same object. "x is not y"\nyields the inverse truth value. [4]\n',
- 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe "if", "while" and "for" statements implement traditional control\nflow constructs. "try" specifies exception handlers and/or cleanup\ncode for a group of statements, while the "with" statement allows the\nexecution of initialization and finalization code around a block of\ncode. Function and class definitions are also syntactically compound\nstatements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which "if" clause a following "else" clause would belong:\n\n if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n"print()" calls are executed:\n\n if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a "NEWLINE" possibly followed by a\n"DEDENT". Also note that optional continuation clauses always begin\nwith a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling "else"\' problem is solved in Python by\nrequiring nested "if" statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe "if" statement\n==================\n\nThe "if" statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n\n\nThe "while" statement\n=====================\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n\n\nThe "for" statement\n===================\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n"expression_list". The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there was no next\nitem.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop. Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n loop (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe "try" statement\n===================\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started. This search inspects the except clauses\nin turn until one is found that matches the exception. An expression-\nless except clause, if present, must be last; it matches any\nexception. For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception. An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be access via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred. "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler. The "try"\nclause is executed, including any "except" and "else" clauses. If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed. If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause. If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe "with" statement\n====================\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n value from "__enter__()" is assigned to it.\n\n Note: The "with" statement guarantees that if the "__enter__()"\n method returns without an error, then "__exit__()" will always be\n called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to "__exit__()". Otherwise, three\n "None" arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the "__exit__()" method was false, the exception is reraised.\n If the return value was true, the exception is suppressed, and\n execution continues with the statement following the "with"\n statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from "__exit__()" is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use "None" as the default,\nand explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [parameter_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with "self.name = value". Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way. Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results. *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n Class Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless\n there is a "finally" clause which happens to raise another\n exception. That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of\n an exception or the execution of a "return", "continue", or\n "break" statement.\n\n[3] A string literal appearing as the first statement in the\n function body is transformed into the function\'s "__doc__"\n attribute and therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s "__doc__" item and\n therefore the class\'s *docstring*.\n',
+ 'compound': '\nCompound statements\n*******************\n\nCompound statements contain (groups of) other statements; they affect\nor control the execution of those other statements in some way. In\ngeneral, compound statements span multiple lines, although in simple\nincarnations a whole compound statement may be contained in one line.\n\nThe "if", "while" and "for" statements implement traditional control\nflow constructs. "try" specifies exception handlers and/or cleanup\ncode for a group of statements, while the "with" statement allows the\nexecution of initialization and finalization code around a block of\ncode. Function and class definitions are also syntactically compound\nstatements.\n\nCompound statements consist of one or more \'clauses.\' A clause\nconsists of a header and a \'suite.\' The clause headers of a\nparticular compound statement are all at the same indentation level.\nEach clause header begins with a uniquely identifying keyword and ends\nwith a colon. A suite is a group of statements controlled by a\nclause. A suite can be one or more semicolon-separated simple\nstatements on the same line as the header, following the header\'s\ncolon, or it can be one or more indented statements on subsequent\nlines. Only the latter form of suite can contain nested compound\nstatements; the following is illegal, mostly because it wouldn\'t be\nclear to which "if" clause a following "else" clause would belong:\n\n if test1: if test2: print(x)\n\nAlso note that the semicolon binds tighter than the colon in this\ncontext, so that in the following example, either all or none of the\n"print()" calls are executed:\n\n if x < y < z: print(x); print(y); print(z)\n\nSummarizing:\n\n compound_stmt ::= if_stmt\n | while_stmt\n | for_stmt\n | try_stmt\n | with_stmt\n | funcdef\n | classdef\n suite ::= stmt_list NEWLINE | NEWLINE INDENT statement+ DEDENT\n statement ::= stmt_list NEWLINE | compound_stmt\n stmt_list ::= simple_stmt (";" simple_stmt)* [";"]\n\nNote that statements always end in a "NEWLINE" possibly followed by a\n"DEDENT". Also note that optional continuation clauses always begin\nwith a keyword that cannot start a statement, thus there are no\nambiguities (the \'dangling "else"\' problem is solved in Python by\nrequiring nested "if" statements to be indented).\n\nThe formatting of the grammar rules in the following sections places\neach clause on a separate line for clarity.\n\n\nThe "if" statement\n==================\n\nThe "if" statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the "if" statement is executed or evaluated).\nIf all expressions are false, the suite of the "else" clause, if\npresent, is executed.\n\n\nThe "while" statement\n=====================\n\nThe "while" statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the "else" clause, if present, is executed\nand the loop terminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and goes back\nto testing the expression.\n\n\nThe "for" statement\n===================\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n"expression_list". The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there was no next\nitem.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop. Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n loop (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n\n\nThe "try" statement\n===================\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started. This search inspects the except clauses\nin turn until one is found that matches the exception. An expression-\nless except clause, if present, must be last; it matches any\nexception. For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception. An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be access via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred. "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler. The "try"\nclause is executed, including any "except" and "else" clauses. If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed. If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause. If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n\n\nThe "with" statement\n====================\n\nThe "with" statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common "try"..."except"..."finally"\nusage patterns to be encapsulated for convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the "with" statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the "with_item")\n is evaluated to obtain a context manager.\n\n2. The context manager\'s "__exit__()" is loaded for later use.\n\n3. The context manager\'s "__enter__()" method is invoked.\n\n4. If a target was included in the "with" statement, the return\n value from "__enter__()" is assigned to it.\n\n Note: The "with" statement guarantees that if the "__enter__()"\n method returns without an error, then "__exit__()" will always be\n called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s "__exit__()" method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to "__exit__()". Otherwise, three\n "None" arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the "__exit__()" method was false, the exception is reraised.\n If the return value was true, the exception is suppressed, and\n execution continues with the statement following the "with"\n statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from "__exit__()" is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple "with" statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nChanged in version 3.1: Support for multiple context expressions.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nFunction definitions\n====================\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call. This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended. A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n\n\nClass definitions\n=================\n\nA class definition defines a class object (see section *The standard\ntype hierarchy*):\n\n classdef ::= [decorators] "class" classname [inheritance] ":" suite\n inheritance ::= "(" [parameter_list] ")"\n classname ::= identifier\n\nA class definition is an executable statement. The inheritance list\nusually gives a list of base classes (see *Customizing class creation*\nfor more advanced uses), so each item in the list should evaluate to a\nclass object which allows subclassing. Classes without an inheritance\nlist inherit, by default, from the base class "object"; hence,\n\n class Foo:\n pass\n\nis equivalent to\n\n class Foo(object):\n pass\n\nThe class\'s suite is then executed in a new execution frame (see\n*Naming and binding*), using a newly created local namespace and the\noriginal global namespace. (Usually, the suite contains mostly\nfunction definitions.) When the class\'s suite finishes execution, its\nexecution frame is discarded but its local namespace is saved. [4] A\nclass object is then created using the inheritance list for the base\nclasses and the saved local namespace for the attribute dictionary.\nThe class name is bound to this class object in the original local\nnamespace.\n\nClass creation can be customized heavily using *metaclasses*.\n\nClasses can also be decorated: just like when decorating functions,\n\n @f1(arg)\n @f2\n class Foo: pass\n\nis equivalent to\n\n class Foo: pass\n Foo = f1(arg)(f2(Foo))\n\nThe evaluation rules for the decorator expressions are the same as for\nfunction decorators. The result must be a class object, which is then\nbound to the class name.\n\n**Programmer\'s note:** Variables defined in the class definition are\nclass attributes; they are shared by instances. Instance attributes\ncan be set in a method with "self.name = value". Both class and\ninstance attributes are accessible through the notation ""self.name"",\nand an instance attribute hides a class attribute with the same name\nwhen accessed in this way. Class attributes can be used as defaults\nfor instance attributes, but using mutable values there can lead to\nunexpected results. *Descriptors* can be used to create instance\nvariables with different implementation details.\n\nSee also: **PEP 3115** - Metaclasses in Python 3 **PEP 3129** -\n Class Decorators\n\n-[ Footnotes ]-\n\n[1] The exception is propagated to the invocation stack unless\n there is a "finally" clause which happens to raise another\n exception. That new exception causes the old one to be lost.\n\n[2] Currently, control "flows off the end" except in the case of\n an exception or the execution of a "return", "continue", or\n "break" statement.\n\n[3] A string literal appearing as the first statement in the\n function body is transformed into the function\'s "__doc__"\n attribute and therefore the function\'s *docstring*.\n\n[4] A string literal appearing as the first statement in the class\n body is transformed into the namespace\'s "__doc__" item and\n therefore the class\'s *docstring*.\n',
'context-managers': '\nWith Statement Context Managers\n*******************************\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code. Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The "with"\n statement will bind this method\'s return value to the target(s)\n specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be "None".\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that "__exit__()" methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n',
'continue': '\nThe "continue" statement\n************************\n\n continue_stmt ::= "continue"\n\n"continue" may only occur syntactically nested in a "for" or "while"\nloop, but not nested in a function or class definition or "finally"\nclause within that loop. It continues with the next cycle of the\nnearest enclosing loop.\n\nWhen "continue" passes control out of a "try" statement with a\n"finally" clause, that "finally" clause is executed before really\nstarting the next loop cycle.\n',
'conversions': '\nArithmetic conversions\n**********************\n\nWhen a description of an arithmetic operator below uses the phrase\n"the numeric arguments are converted to a common type," this means\nthat the operator implementation for built-in types works that way:\n\n* If either argument is a complex number, the other is converted to\n complex;\n\n* otherwise, if either argument is a floating point number, the\n other is converted to floating point;\n\n* otherwise, both must be integers and no conversion is necessary.\n\nSome additional rules apply for certain operators (e.g., a string left\nargument to the \'%\' operator). Extensions must define their own\nconversion behavior.\n',
- 'customization': '\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])". As a special constraint on\n constructors, no value may be returned; doing so will cause a\n "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing "None" in\n "sys.last_traceback". Circular references which are garbage are\n detected when the option cycle detector is enabled (it\'s on by\n default), but can only be cleaned up if there are no Python-\n level "__del__()" methods involved. Refer to the documentation\n for the "gc" module for more information about how "__del__()"\n methods are handled by the cycle detector, particularly the\n description of the "garbage" value.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n <ParentClass>.__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n',
- 'debugger': '\n"pdb" --- The Python Debugger\n*****************************\n\nThe module "pdb" defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n"Pdb". This is currently undocumented but easily understood by reading\nthe source. The extension interface uses the modules "bdb" and "cmd".\n\nThe debugger\'s prompt is "(Pdb)". Typical usage to run a program under\ncontrol of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\nChanged in version 3.3: Tab-completion via the "readline" module is\navailable for commands and command arguments, e.g. the current global\nand local names are offered as arguments of the "print" command.\n\n"pdb.py" can also be invoked as a script to debug other scripts. For\nexample:\n\n python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: "pdb.py" now accepts a "-c" option that executes\ncommands as if given in a ".pdbrc" file, see *Debugger Commands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the "continue" command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print(spam)\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print(spam)\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n Execute the *statement* (given as a string or a code object) under\n debugger control. The debugger prompt appears before any code is\n executed; you can set breakpoints and type "continue", or you can\n step through the statement using "step" or "next" (all these\n commands are explained below). The optional *globals* and *locals*\n arguments specify the environment in which the code is executed; by\n default the dictionary of the module "__main__" is used. (See the\n explanation of the built-in "exec()" or "eval()" functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n Evaluate the *expression* (given as a string or a code object)\n under debugger control. When "runeval()" returns, it returns the\n value of the expression. Otherwise this function is similar to\n "run()".\n\npdb.runcall(function, *args, **kwds)\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When "runcall()" returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem(traceback=None)\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n "sys.last_traceback".\n\nThe "run*" functions and "set_trace()" are aliases for instantiating\nthe "Pdb" class and calling the method of the same name. If you want\nto access further features, you have to do this yourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n "Pdb" is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying "cmd.Cmd" class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n By default, Pdb sets a handler for the SIGINT signal (which is sent\n when the user presses Ctrl-C on the console) when you give a\n "continue" command. This allows you to break into the debugger\n again by pressing Ctrl-C. If you want Pdb not to touch the SIGINT\n handler, set *nosigint* tot true.\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 3.1: The *skip* argument.\n\n New in version 3.2: The *nosigint* argument. Previously, a SIGINT\n handler was never set by Pdb.\n\n run(statement, globals=None, locals=None)\n runeval(expression, globals=None, locals=None)\n runcall(function, *args, **kwds)\n set_trace()\n\n See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below. Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n"h(elp)" means that either "h" or "help" can be used to enter the help\ncommand (but not "he" or "hel", nor "H" or "Help" or "HELP").\nArguments to commands must be separated by whitespace (spaces or\ntabs). Optional arguments are enclosed in square brackets ("[]") in\nthe command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n("|").\n\nEntering a blank line repeats the last command entered. Exception: if\nthe last command was a "list" command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged. Python statements can also be prefixed with an exclamation\npoint ("!"). This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*. Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by ";;".\n(A single ";" is not used as it is the separator for multiple commands\nin a line that is passed to the Python parser.) No intelligence is\napplied to separating the commands; the input is split at the first\n";;" pair, even if it is in the middle of a quoted string.\n\nIf a file ".pdbrc" exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt. This is particularly useful for aliases. If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ".pdbrc" can now contain commands that\ncontinue debugging, such as "continue" or "next". Previously, these\ncommands had no effect.\n\nh(elp) [command]\n\n Without argument, print the list of available commands. With a\n *command* as argument, print help about that command. "help pdb"\n displays the full documentation (the docstring of the "pdb"\n module). Since the *command* argument must be an identifier, "help\n exec" must be entered to get help on the "!" command.\n\nw(here)\n\n Print a stack trace, with the most recent frame at the bottom. An\n arrow indicates the current frame, which determines the context of\n most commands.\n\nd(own) [count]\n\n Move the current frame *count* (default one) levels down in the\n stack trace (to a newer frame).\n\nu(p) [count]\n\n Move the current frame *count* (default one) levels up in the stack\n trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n With a *lineno* argument, set a break there in the current file.\n With a *function* argument, set a break at the first executable\n statement within that function. The line number may be prefixed\n with a filename and a colon, to specify a breakpoint in another\n file (probably one that hasn\'t been loaded yet). The file is\n searched on "sys.path". Note that each breakpoint is assigned a\n number to which all the other breakpoint commands refer.\n\n If a second argument is present, it is an expression which must\n evaluate to true before the breakpoint is honored.\n\n Without argument, list all breaks, including for each breakpoint,\n the number of times that breakpoint has been hit, the current\n ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n Temporary breakpoint, which is removed automatically when it is\n first hit. The arguments are the same as for "break".\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n With a *filename:lineno* argument, clear all the breakpoints at\n this line. With a space separated list of breakpoint numbers, clear\n those breakpoints. Without argument, clear all breaks (but first\n ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n Disable the breakpoints given as a space separated list of\n breakpoint numbers. Disabling a breakpoint means it cannot cause\n the program to stop execution, but unlike clearing a breakpoint, it\n remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n Set the ignore count for the given breakpoint number. If count is\n omitted, the ignore count is set to 0. A breakpoint becomes active\n when the ignore count is zero. When non-zero, the count is\n decremented each time the breakpoint is reached and the breakpoint\n is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n Set a new *condition* for the breakpoint, an expression which must\n evaluate to true before the breakpoint is honored. If *condition*\n is absent, any existing condition is removed; i.e., the breakpoint\n is made unconditional.\n\ncommands [bpnumber]\n\n Specify a list of commands for breakpoint number *bpnumber*. The\n commands themselves appear on the following lines. Type a line\n containing just "end" to terminate the commands. An example:\n\n (Pdb) commands 1\n (com) print some_variable\n (com) end\n (Pdb)\n\n To remove all commands from a breakpoint, type commands and follow\n it immediately with "end"; that is, give no commands.\n\n With no *bpnumber* argument, commands refers to the last breakpoint\n set.\n\n You can use breakpoint commands to start your program up again.\n Simply use the continue command, or step, or any other command that\n resumes execution.\n\n Specifying any command resuming execution (currently continue,\n step, next, return, jump, quit and their abbreviations) terminates\n the command list (as if that command was immediately followed by\n end). This is because any time you resume execution (even with a\n simple next or step), you may encounter another breakpoint--which\n could have its own command list, leading to ambiguities about which\n list to execute.\n\n If you use the \'silent\' command in the command list, the usual\n message about stopping at a breakpoint is not printed. This may be\n desirable for breakpoints that are to print a specific message and\n then continue. If none of the other commands print anything, you\n see no sign that the breakpoint was reached.\n\ns(tep)\n\n Execute the current line, stop at the first possible occasion\n (either in a function that is called or on the next line in the\n current function).\n\nn(ext)\n\n Continue execution until the next line in the current function is\n reached or it returns. (The difference between "next" and "step"\n is that "step" stops inside a called function, while "next"\n executes called functions at (nearly) full speed, only stopping at\n the next line in the current function.)\n\nunt(il) [lineno]\n\n Without argument, continue execution until the line with a number\n greater than the current one is reached.\n\n With a line number, continue execution until a line with a number\n greater or equal to that is reached. In both cases, also stop when\n the current frame returns.\n\n Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n Continue execution until the current function returns.\n\nc(ont(inue))\n\n Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n Set the next line that will be executed. Only available in the\n bottom-most frame. This lets you jump back and execute code again,\n or jump forward to skip code that you don\'t want to run.\n\n It should be noted that not all jumps are allowed -- for instance\n it is not possible to jump into the middle of a "for" loop or out\n of a "finally" clause.\n\nl(ist) [first[, last]]\n\n List source code for the current file. Without arguments, list 11\n lines around the current line or continue the previous listing.\n With "." as argument, list 11 lines around the current line. With\n one argument, list 11 lines around at that line. With two\n arguments, list the given range; if the second argument is less\n than the first, it is interpreted as a count.\n\n The current line in the current frame is indicated by "->". If an\n exception is being debugged, the line where the exception was\n originally raised or propagated is indicated by ">>", if it differs\n from the current line.\n\n New in version 3.2: The ">>" marker.\n\nll | longlist\n\n List all source code for the current function or frame.\n Interesting lines are marked as for "list".\n\n New in version 3.2.\n\na(rgs)\n\n Print the argument list of the current function.\n\np(rint) expression\n\n Evaluate the *expression* in the current context and print its\n value.\n\npp expression\n\n Like the "print" command, except the value of the expression is\n pretty-printed using the "pprint" module.\n\nwhatis expression\n\n Print the type of the *expression*.\n\nsource expression\n\n Try to get source code for the given object and display it.\n\n New in version 3.2.\n\ndisplay [expression]\n\n Display the value of the expression if it changed, each time\n execution stops in the current frame.\n\n Without expression, list all display expressions for the current\n frame.\n\n New in version 3.2.\n\nundisplay [expression]\n\n Do not display the expression any more in the current frame.\n Without expression, clear all display expressions for the current\n frame.\n\n New in version 3.2.\n\ninteract\n\n Start an interative interpreter (using the "code" module) whose\n global namespace contains all the (global and local) names found in\n the current scope.\n\n New in version 3.2.\n\nalias [name [command]]\n\n Create an alias called *name* that executes *command*. The command\n must *not* be enclosed in quotes. Replaceable parameters can be\n indicated by "%1", "%2", and so on, while "%*" is replaced by all\n the parameters. If no command is given, the current alias for\n *name* is shown. If no arguments are given, all aliases are listed.\n\n Aliases may be nested and can contain anything that can be legally\n typed at the pdb prompt. Note that internal pdb commands *can* be\n overridden by aliases. Such a command is then hidden until the\n alias is removed. Aliasing is recursively applied to the first\n word of the command line; all other words in the line are left\n alone.\n\n As an example, here are two useful aliases (especially when placed\n in the ".pdbrc" file):\n\n # Print instance variables (usage "pi classInst")\n alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n # Print instance variables in self\n alias ps pi self\n\nunalias name\n\n Delete the specified alias.\n\n! statement\n\n Execute the (one-line) *statement* in the context of the current\n stack frame. The exclamation point can be omitted unless the first\n word of the statement resembles a debugger command. To set a\n global variable, you can prefix the assignment command with a\n "global" statement on the same line, e.g.:\n\n (Pdb) global list_options; list_options = [\'-l\']\n (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n Restart the debugged Python program. If an argument is supplied,\n it is split with "shlex" and the result is used as the new\n "sys.argv". History, breakpoints, actions and debugger options are\n preserved. "restart" is an alias for "run".\n\nq(uit)\n\n Quit from the debugger. The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module\n is determined by the "__name__" in the frame globals.\n',
+ 'customization': '\nBasic customization\n*******************\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])". As a special constraint on\n constructors, no value may be returned; doing so will cause a\n "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing "None" in\n "sys.last_traceback". Circular references which are garbage are\n detected and cleaned up when the cyclic garbage collector is\n enabled (it\'s on by default). Refer to the documentation for the\n "gc" module for more information about this topic.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\n Changed in version 3.4: The __format__ method of "object" itself\n raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n <ParentClass>.__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n',
+ 'debugger': '\n"pdb" --- The Python Debugger\n*****************************\n\nThe module "pdb" defines an interactive source code debugger for\nPython programs. It supports setting (conditional) breakpoints and\nsingle stepping at the source line level, inspection of stack frames,\nsource code listing, and evaluation of arbitrary Python code in the\ncontext of any stack frame. It also supports post-mortem debugging\nand can be called under program control.\n\nThe debugger is extensible -- it is actually defined as the class\n"Pdb". This is currently undocumented but easily understood by reading\nthe source. The extension interface uses the modules "bdb" and "cmd".\n\nThe debugger\'s prompt is "(Pdb)". Typical usage to run a program under\ncontrol of the debugger is:\n\n >>> import pdb\n >>> import mymodule\n >>> pdb.run(\'mymodule.test()\')\n > <string>(0)?()\n (Pdb) continue\n > <string>(1)?()\n (Pdb) continue\n NameError: \'spam\'\n > <string>(1)?()\n (Pdb)\n\nChanged in version 3.3: Tab-completion via the "readline" module is\navailable for commands and command arguments, e.g. the current global\nand local names are offered as arguments of the "p" command.\n\n"pdb.py" can also be invoked as a script to debug other scripts. For\nexample:\n\n python3 -m pdb myscript.py\n\nWhen invoked as a script, pdb will automatically enter post-mortem\ndebugging if the program being debugged exits abnormally. After post-\nmortem debugging (or after normal exit of the program), pdb will\nrestart the program. Automatic restarting preserves pdb\'s state (such\nas breakpoints) and in most cases is more useful than quitting the\ndebugger upon program\'s exit.\n\nNew in version 3.2: "pdb.py" now accepts a "-c" option that executes\ncommands as if given in a ".pdbrc" file, see *Debugger Commands*.\n\nThe typical usage to break into the debugger from a running program is\nto insert\n\n import pdb; pdb.set_trace()\n\nat the location you want to break into the debugger. You can then\nstep through the code following this statement, and continue running\nwithout the debugger using the "continue" command.\n\nThe typical usage to inspect a crashed program is:\n\n >>> import pdb\n >>> import mymodule\n >>> mymodule.test()\n Traceback (most recent call last):\n File "<stdin>", line 1, in ?\n File "./mymodule.py", line 4, in test\n test2()\n File "./mymodule.py", line 3, in test2\n print(spam)\n NameError: spam\n >>> pdb.pm()\n > ./mymodule.py(3)test2()\n -> print(spam)\n (Pdb)\n\nThe module defines the following functions; each enters the debugger\nin a slightly different way:\n\npdb.run(statement, globals=None, locals=None)\n\n Execute the *statement* (given as a string or a code object) under\n debugger control. The debugger prompt appears before any code is\n executed; you can set breakpoints and type "continue", or you can\n step through the statement using "step" or "next" (all these\n commands are explained below). The optional *globals* and *locals*\n arguments specify the environment in which the code is executed; by\n default the dictionary of the module "__main__" is used. (See the\n explanation of the built-in "exec()" or "eval()" functions.)\n\npdb.runeval(expression, globals=None, locals=None)\n\n Evaluate the *expression* (given as a string or a code object)\n under debugger control. When "runeval()" returns, it returns the\n value of the expression. Otherwise this function is similar to\n "run()".\n\npdb.runcall(function, *args, **kwds)\n\n Call the *function* (a function or method object, not a string)\n with the given arguments. When "runcall()" returns, it returns\n whatever the function call returned. The debugger prompt appears\n as soon as the function is entered.\n\npdb.set_trace()\n\n Enter the debugger at the calling stack frame. This is useful to\n hard-code a breakpoint at a given point in a program, even if the\n code is not otherwise being debugged (e.g. when an assertion\n fails).\n\npdb.post_mortem(traceback=None)\n\n Enter post-mortem debugging of the given *traceback* object. If no\n *traceback* is given, it uses the one of the exception that is\n currently being handled (an exception must be being handled if the\n default is to be used).\n\npdb.pm()\n\n Enter post-mortem debugging of the traceback found in\n "sys.last_traceback".\n\nThe "run*" functions and "set_trace()" are aliases for instantiating\nthe "Pdb" class and calling the method of the same name. If you want\nto access further features, you have to do this yourself:\n\nclass class pdb.Pdb(completekey=\'tab\', stdin=None, stdout=None, skip=None, nosigint=False)\n\n "Pdb" is the debugger class.\n\n The *completekey*, *stdin* and *stdout* arguments are passed to the\n underlying "cmd.Cmd" class; see the description there.\n\n The *skip* argument, if given, must be an iterable of glob-style\n module name patterns. The debugger will not step into frames that\n originate in a module that matches one of these patterns. [1]\n\n By default, Pdb sets a handler for the SIGINT signal (which is sent\n when the user presses Ctrl-C on the console) when you give a\n "continue" command. This allows you to break into the debugger\n again by pressing Ctrl-C. If you want Pdb not to touch the SIGINT\n handler, set *nosigint* tot true.\n\n Example call to enable tracing with *skip*:\n\n import pdb; pdb.Pdb(skip=[\'django.*\']).set_trace()\n\n New in version 3.1: The *skip* argument.\n\n New in version 3.2: The *nosigint* argument. Previously, a SIGINT\n handler was never set by Pdb.\n\n run(statement, globals=None, locals=None)\n runeval(expression, globals=None, locals=None)\n runcall(function, *args, **kwds)\n set_trace()\n\n See the documentation for the functions explained above.\n\n\nDebugger Commands\n=================\n\nThe commands recognized by the debugger are listed below. Most\ncommands can be abbreviated to one or two letters as indicated; e.g.\n"h(elp)" means that either "h" or "help" can be used to enter the help\ncommand (but not "he" or "hel", nor "H" or "Help" or "HELP").\nArguments to commands must be separated by whitespace (spaces or\ntabs). Optional arguments are enclosed in square brackets ("[]") in\nthe command syntax; the square brackets must not be typed.\nAlternatives in the command syntax are separated by a vertical bar\n("|").\n\nEntering a blank line repeats the last command entered. Exception: if\nthe last command was a "list" command, the next 11 lines are listed.\n\nCommands that the debugger doesn\'t recognize are assumed to be Python\nstatements and are executed in the context of the program being\ndebugged. Python statements can also be prefixed with an exclamation\npoint ("!"). This is a powerful way to inspect the program being\ndebugged; it is even possible to change a variable or call a function.\nWhen an exception occurs in such a statement, the exception name is\nprinted but the debugger\'s state is not changed.\n\nThe debugger supports *aliases*. Aliases can have parameters which\nallows one a certain level of adaptability to the context under\nexamination.\n\nMultiple commands may be entered on a single line, separated by ";;".\n(A single ";" is not used as it is the separator for multiple commands\nin a line that is passed to the Python parser.) No intelligence is\napplied to separating the commands; the input is split at the first\n";;" pair, even if it is in the middle of a quoted string.\n\nIf a file ".pdbrc" exists in the user\'s home directory or in the\ncurrent directory, it is read in and executed as if it had been typed\nat the debugger prompt. This is particularly useful for aliases. If\nboth files exist, the one in the home directory is read first and\naliases defined there can be overridden by the local file.\n\nChanged in version 3.2: ".pdbrc" can now contain commands that\ncontinue debugging, such as "continue" or "next". Previously, these\ncommands had no effect.\n\nh(elp) [command]\n\n Without argument, print the list of available commands. With a\n *command* as argument, print help about that command. "help pdb"\n displays the full documentation (the docstring of the "pdb"\n module). Since the *command* argument must be an identifier, "help\n exec" must be entered to get help on the "!" command.\n\nw(here)\n\n Print a stack trace, with the most recent frame at the bottom. An\n arrow indicates the current frame, which determines the context of\n most commands.\n\nd(own) [count]\n\n Move the current frame *count* (default one) levels down in the\n stack trace (to a newer frame).\n\nu(p) [count]\n\n Move the current frame *count* (default one) levels up in the stack\n trace (to an older frame).\n\nb(reak) [([filename:]lineno | function) [, condition]]\n\n With a *lineno* argument, set a break there in the current file.\n With a *function* argument, set a break at the first executable\n statement within that function. The line number may be prefixed\n with a filename and a colon, to specify a breakpoint in another\n file (probably one that hasn\'t been loaded yet). The file is\n searched on "sys.path". Note that each breakpoint is assigned a\n number to which all the other breakpoint commands refer.\n\n If a second argument is present, it is an expression which must\n evaluate to true before the breakpoint is honored.\n\n Without argument, list all breaks, including for each breakpoint,\n the number of times that breakpoint has been hit, the current\n ignore count, and the associated condition if any.\n\ntbreak [([filename:]lineno | function) [, condition]]\n\n Temporary breakpoint, which is removed automatically when it is\n first hit. The arguments are the same as for "break".\n\ncl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n\n With a *filename:lineno* argument, clear all the breakpoints at\n this line. With a space separated list of breakpoint numbers, clear\n those breakpoints. Without argument, clear all breaks (but first\n ask confirmation).\n\ndisable [bpnumber [bpnumber ...]]\n\n Disable the breakpoints given as a space separated list of\n breakpoint numbers. Disabling a breakpoint means it cannot cause\n the program to stop execution, but unlike clearing a breakpoint, it\n remains in the list of breakpoints and can be (re-)enabled.\n\nenable [bpnumber [bpnumber ...]]\n\n Enable the breakpoints specified.\n\nignore bpnumber [count]\n\n Set the ignore count for the given breakpoint number. If count is\n omitted, the ignore count is set to 0. A breakpoint becomes active\n when the ignore count is zero. When non-zero, the count is\n decremented each time the breakpoint is reached and the breakpoint\n is not disabled and any associated condition evaluates to true.\n\ncondition bpnumber [condition]\n\n Set a new *condition* for the breakpoint, an expression which must\n evaluate to true before the breakpoint is honored. If *condition*\n is absent, any existing condition is removed; i.e., the breakpoint\n is made unconditional.\n\ncommands [bpnumber]\n\n Specify a list of commands for breakpoint number *bpnumber*. The\n commands themselves appear on the following lines. Type a line\n containing just "end" to terminate the commands. An example:\n\n (Pdb) commands 1\n (com) p some_variable\n (com) end\n (Pdb)\n\n To remove all commands from a breakpoint, type commands and follow\n it immediately with "end"; that is, give no commands.\n\n With no *bpnumber* argument, commands refers to the last breakpoint\n set.\n\n You can use breakpoint commands to start your program up again.\n Simply use the continue command, or step, or any other command that\n resumes execution.\n\n Specifying any command resuming execution (currently continue,\n step, next, return, jump, quit and their abbreviations) terminates\n the command list (as if that command was immediately followed by\n end). This is because any time you resume execution (even with a\n simple next or step), you may encounter another breakpoint--which\n could have its own command list, leading to ambiguities about which\n list to execute.\n\n If you use the \'silent\' command in the command list, the usual\n message about stopping at a breakpoint is not printed. This may be\n desirable for breakpoints that are to print a specific message and\n then continue. If none of the other commands print anything, you\n see no sign that the breakpoint was reached.\n\ns(tep)\n\n Execute the current line, stop at the first possible occasion\n (either in a function that is called or on the next line in the\n current function).\n\nn(ext)\n\n Continue execution until the next line in the current function is\n reached or it returns. (The difference between "next" and "step"\n is that "step" stops inside a called function, while "next"\n executes called functions at (nearly) full speed, only stopping at\n the next line in the current function.)\n\nunt(il) [lineno]\n\n Without argument, continue execution until the line with a number\n greater than the current one is reached.\n\n With a line number, continue execution until a line with a number\n greater or equal to that is reached. In both cases, also stop when\n the current frame returns.\n\n Changed in version 3.2: Allow giving an explicit line number.\n\nr(eturn)\n\n Continue execution until the current function returns.\n\nc(ont(inue))\n\n Continue execution, only stop when a breakpoint is encountered.\n\nj(ump) lineno\n\n Set the next line that will be executed. Only available in the\n bottom-most frame. This lets you jump back and execute code again,\n or jump forward to skip code that you don\'t want to run.\n\n It should be noted that not all jumps are allowed -- for instance\n it is not possible to jump into the middle of a "for" loop or out\n of a "finally" clause.\n\nl(ist) [first[, last]]\n\n List source code for the current file. Without arguments, list 11\n lines around the current line or continue the previous listing.\n With "." as argument, list 11 lines around the current line. With\n one argument, list 11 lines around at that line. With two\n arguments, list the given range; if the second argument is less\n than the first, it is interpreted as a count.\n\n The current line in the current frame is indicated by "->". If an\n exception is being debugged, the line where the exception was\n originally raised or propagated is indicated by ">>", if it differs\n from the current line.\n\n New in version 3.2: The ">>" marker.\n\nll | longlist\n\n List all source code for the current function or frame.\n Interesting lines are marked as for "list".\n\n New in version 3.2.\n\na(rgs)\n\n Print the argument list of the current function.\n\np expression\n\n Evaluate the *expression* in the current context and print its\n value.\n\n Note: "print()" can also be used, but is not a debugger command\n --- this executes the Python "print()" function.\n\npp expression\n\n Like the "p" command, except the value of the expression is pretty-\n printed using the "pprint" module.\n\nwhatis expression\n\n Print the type of the *expression*.\n\nsource expression\n\n Try to get source code for the given object and display it.\n\n New in version 3.2.\n\ndisplay [expression]\n\n Display the value of the expression if it changed, each time\n execution stops in the current frame.\n\n Without expression, list all display expressions for the current\n frame.\n\n New in version 3.2.\n\nundisplay [expression]\n\n Do not display the expression any more in the current frame.\n Without expression, clear all display expressions for the current\n frame.\n\n New in version 3.2.\n\ninteract\n\n Start an interative interpreter (using the "code" module) whose\n global namespace contains all the (global and local) names found in\n the current scope.\n\n New in version 3.2.\n\nalias [name [command]]\n\n Create an alias called *name* that executes *command*. The command\n must *not* be enclosed in quotes. Replaceable parameters can be\n indicated by "%1", "%2", and so on, while "%*" is replaced by all\n the parameters. If no command is given, the current alias for\n *name* is shown. If no arguments are given, all aliases are listed.\n\n Aliases may be nested and can contain anything that can be legally\n typed at the pdb prompt. Note that internal pdb commands *can* be\n overridden by aliases. Such a command is then hidden until the\n alias is removed. Aliasing is recursively applied to the first\n word of the command line; all other words in the line are left\n alone.\n\n As an example, here are two useful aliases (especially when placed\n in the ".pdbrc" file):\n\n # Print instance variables (usage "pi classInst")\n alias pi for k in %1.__dict__.keys(): print("%1.",k,"=",%1.__dict__[k])\n # Print instance variables in self\n alias ps pi self\n\nunalias name\n\n Delete the specified alias.\n\n! statement\n\n Execute the (one-line) *statement* in the context of the current\n stack frame. The exclamation point can be omitted unless the first\n word of the statement resembles a debugger command. To set a\n global variable, you can prefix the assignment command with a\n "global" statement on the same line, e.g.:\n\n (Pdb) global list_options; list_options = [\'-l\']\n (Pdb)\n\nrun [args ...]\nrestart [args ...]\n\n Restart the debugged Python program. If an argument is supplied,\n it is split with "shlex" and the result is used as the new\n "sys.argv". History, breakpoints, actions and debugger options are\n preserved. "restart" is an alias for "run".\n\nq(uit)\n\n Quit from the debugger. The program being executed is aborted.\n\n-[ Footnotes ]-\n\n[1] Whether a frame is considered to originate in a certain module\n is determined by the "__name__" in the frame globals.\n',
'del': '\nThe "del" statement\n*******************\n\n del_stmt ::= "del" target_list\n\nDeletion is recursively defined very similar to the way assignment is\ndefined. Rather than spelling it out in full details, here are some\nhints.\n\nDeletion of a target list recursively deletes each target, from left\nto right.\n\nDeletion of a name removes the binding of that name from the local or\nglobal namespace, depending on whether the name occurs in a "global"\nstatement in the same code block. If the name is unbound, a\n"NameError" exception will be raised.\n\nDeletion of attribute references, subscriptions and slicings is passed\nto the primary object involved; deletion of a slicing is in general\nequivalent to assignment of an empty slice of the right type (but even\nthis is determined by the sliced object).\n\nChanged in version 3.2: Previously it was illegal to delete a name\nfrom the local namespace if it occurs as a free variable in a nested\nblock.\n',
'dict': '\nDictionary displays\n*******************\n\nA dictionary display is a possibly empty series of key/datum pairs\nenclosed in curly braces:\n\n dict_display ::= "{" [key_datum_list | dict_comprehension] "}"\n key_datum_list ::= key_datum ("," key_datum)* [","]\n key_datum ::= expression ":" expression\n dict_comprehension ::= expression ":" expression comp_for\n\nA dictionary display yields a new dictionary object.\n\nIf a comma-separated sequence of key/datum pairs is given, they are\nevaluated from left to right to define the entries of the dictionary:\neach key object is used as a key into the dictionary to store the\ncorresponding datum. This means that you can specify the same key\nmultiple times in the key/datum list, and the final dictionary\'s value\nfor that key will be the last one given.\n\nA dict comprehension, in contrast to list and set comprehensions,\nneeds two expressions separated with a colon followed by the usual\n"for" and "if" clauses. When the comprehension is run, the resulting\nkey and value elements are inserted in the new dictionary in the order\nthey are produced.\n\nRestrictions on the types of the key values are listed earlier in\nsection *The standard type hierarchy*. (To summarize, the key type\nshould be *hashable*, which excludes all mutable objects.) Clashes\nbetween duplicate keys are not detected; the last datum (textually\nrightmost in the display) stored for a given key value prevails.\n',
'dynamic-features': '\nInteraction with dynamic features\n*********************************\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- "import *" --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a "SyntaxError".\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names. Names may be resolved in the local\nand global namespaces of the caller. Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace. [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace. If only one namespace is\nspecified, it is used for both.\n',
@@ -35,7 +35,7 @@ topics = {'assert': '\nThe "assert" statement\n**********************\n\nAssert
'floating': '\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts are always interpreted using\nradix 10. For example, "077e010" is legal, and denotes the same number\nas "77e10". The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator "-" and the\nliteral "1".\n',
'for': '\nThe "for" statement\n*******************\n\nThe "for" statement is used to iterate over the elements of a sequence\n(such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n"expression_list". The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments (see *Assignment statements*), and then the suite is\nexecuted. When the items are exhausted (which is immediately when the\nsequence is empty or an iterator raises a "StopIteration" exception),\nthe suite in the "else" clause, if present, is executed, and the loop\nterminates.\n\nA "break" statement executed in the first suite terminates the loop\nwithout executing the "else" clause\'s suite. A "continue" statement\nexecuted in the first suite skips the rest of the suite and continues\nwith the next item, or with the "else" clause if there was no next\nitem.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nNames in the target list are not deleted when the loop is finished,\nbut if the sequence is empty, it will not have been assigned to at all\nby the loop. Hint: the built-in function "range()" returns an\niterator of integers suitable to emulate the effect of Pascal\'s "for i\n:= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n\nNote: There is a subtlety when the sequence is being modified by the\n loop (this can only occur for mutable sequences, i.e. lists). An\n internal counter is used to keep track of which item is used next,\n and this is incremented on each iteration. When this counter has\n reached the length of the sequence the loop terminates. This means\n that if the suite deletes the current (or a previous) item from the\n sequence, the next item will be skipped (since it gets the index of\n the current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n',
'formatstrings': '\nFormat String Syntax\n********************\n\nThe "str.format()" method and the "Formatter" class share the same\nsyntax for format strings (although in the case of "Formatter",\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n"{}". Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n"{{" and "}}".\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= <any source character except "]"> +\n conversion ::= "r" | "s" | "a"\n format_spec ::= <described in the next section>\n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point "\'!\'", and a *format_spec*, which is\npreceded by a colon "\':\'". These specify a non-default format for the\nreplacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings "\'10\'" or\n"\':-]\'") within a format string. The *arg_name* can be followed by any\nnumber of index or attribute expressions. An expression of the form\n"\'.name\'" selects the named attribute using "getattr()", while an\nexpression of the form "\'[index]\'" does an index lookup using\n"__getitem__()".\n\nChanged in version 3.1: The positional argument specifiers can be\nomitted, so "\'{} {}\'" is equivalent to "\'{0} {1}\'".\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the "__format__()"\nmethod of the value itself. However, in some cases it is desirable to\nforce a type to be formatted as a string, overriding its own\ndefinition of formatting. By converting the value to a string before\ncalling "__format__()", the normal formatting logic is bypassed.\n\nThree conversion flags are currently supported: "\'!s\'" which calls\n"str()" on the value, "\'!r\'" which calls "repr()" and "\'!a\'" which\ncalls "ascii()".\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n "More {!a}" # Calls ascii() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in "format()" function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string ("""") produces\nthe same result as if you had called "str()" on the value. A non-empty\nformat string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= <any character>\n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nIf a valid *align* value is specified, it can be preceded by a *fill*\ncharacter that can be any character and defaults to a space if\nomitted. Note that it is not possible to use "{" and "}" as *fill*\nchar while using the "str.format()" method; this limitation however\ndoesn\'t affect the "format()" function.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | "\'<\'" | Forces the field to be left-aligned within the available |\n +-----------+------------------------------------------------------------+\n | "\'>\'" | Forces the field to be right-aligned within the available |\n +-----------+------------------------------------------------------------+\n | "\'=\'" | Forces the padding to be placed after the sign (if any) |\n +-----------+------------------------------------------------------------+\n | "\'^\'" | Forces the field to be centered within the available |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | "\'+\'" | indicates that a sign should be used for both positive as |\n +-----------+------------------------------------------------------------+\n | "\'-\'" | indicates that a sign should be used only for negative |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n +-----------+------------------------------------------------------------+\n\nThe "\'#\'" option causes the "alternate form" to be used for the\nconversion. The alternate form is defined differently for different\ntypes. This option is only valid for integer, float, complex and\nDecimal types. For integers, when binary, octal, or hexadecimal output\nis used, this option adds the prefix respective "\'0b\'", "\'0o\'", or\n"\'0x\'" to the output value. For floats, complex and Decimal the\nalternate form causes the result of the conversion to always contain a\ndecimal-point character, even if no digits follow it. Normally, a\ndecimal-point character appears in the result of these conversions\nonly if a digit follows it. In addition, for "\'g\'" and "\'G\'"\nconversions, trailing zeros are not removed from the result.\n\nThe "\',\'" option signals the use of a comma for a thousands separator.\nFor a locale aware separator, use the "\'n\'" integer presentation type\ninstead.\n\nChanged in version 3.1: Added the "\',\'" option (see also **PEP 378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero ("\'0\'") character enables sign-\naware zero-padding for numeric types. This is equivalent to a *fill*\ncharacter of "\'0\'" with an *alignment* type of "\'=\'".\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with "\'f\'" and "\'F\'", or before and after the decimal point\nfor a floating point value formatted with "\'g\'" or "\'G\'". For non-\nnumber types the field indicates the maximum field size - in other\nwords, how many characters will be used from the field content. The\n*precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'s\'" | String format. This is the default type for strings and |\n +-----------+------------------------------------------------------------+\n | None | The same as "\'s\'". |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'b\'" | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | "\'c\'" | Character. Converts the integer to the corresponding |\n +-----------+------------------------------------------------------------+\n | "\'d\'" | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | "\'o\'" | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | "\'x\'" | Hex format. Outputs the number in base 16, using lower- |\n +-----------+------------------------------------------------------------+\n | "\'X\'" | Hex format. Outputs the number in base 16, using upper- |\n +-----------+------------------------------------------------------------+\n | "\'n\'" | Number. This is the same as "\'d\'", except that it uses the |\n +-----------+------------------------------------------------------------+\n | None | The same as "\'d\'". |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except "\'n\'"\nand None). When doing so, "float()" is used to convert the integer to\na floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | "\'e\'" | Exponent notation. Prints the number in scientific |\n +-----------+------------------------------------------------------------+\n | "\'E\'" | Exponent notation. Same as "\'e\'" except it uses an upper |\n +-----------+------------------------------------------------------------+\n | "\'f\'" | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | "\'F\'" | Fixed point. Same as "\'f\'", but converts "nan" to "NAN" |\n +-----------+------------------------------------------------------------+\n | "\'g\'" | General format. For a given precision "p >= 1", this |\n +-----------+------------------------------------------------------------+\n | "\'G\'" | General format. Same as "\'g\'" except switches to "\'E\'" if |\n +-----------+------------------------------------------------------------+\n | "\'n\'" | Number. This is the same as "\'g\'", except that it uses the |\n +-----------+------------------------------------------------------------+\n | "\'%\'" | Percentage. Multiplies the number by 100 and displays in |\n +-----------+------------------------------------------------------------+\n | None | Similar to "\'g\'", except with at least one digit past the |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old "%"-formatting.\n\nIn most of the cases the syntax is similar to the old "%"-formatting,\nwith the addition of the "{}" and with ":" used instead of "%". For\nexample, "\'%03.2f\'" can be translated to "\'{:03.2f}\'".\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 3.1+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point:\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing "%s" and "%r":\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing "%+f", "%-f", and "% f" and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing "%x" and "%o" and converting the value to different bases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 86.36%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE\n ... for base in \'dXob\':\n ... print(\'{0:{width}{base}}\'.format(num, base=base, width=width), end=\' \')\n ... print()\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n',
- 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use "None" as the default,\nand explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n',
+ 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n funcdef ::= [decorators] "def" funcname "(" [parameter_list] ")" ["->" expression] ":" suite\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [parameter_list [","]] ")"] NEWLINE\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" [parameter] ("," defparameter)* ["," "**" parameter]\n | "**" parameter\n | defparameter [","] )\n parameter ::= identifier [":" expression]\n defparameter ::= parameter ["=" expression]\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more *parameters* have the form *parameter* "="\n*expression*, the function is said to have "default parameter values."\nFor a parameter with a default value, the corresponding *argument* may\nbe omitted from a call, in which case the parameter\'s default value is\nsubstituted. If a parameter has a default value, all following\nparameters up until the ""*"" must also have a default value --- this\nis a syntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated from left to right when the\nfunction definition is executed.** This means that the expression is\nevaluated once, when the function is defined, and that the same "pre-\ncomputed" value is used for each call. This is especially important\nto understand when a default parameter is a mutable object, such as a\nlist or a dictionary: if the function modifies the object (e.g. by\nappending an item to a list), the default value is in effect modified.\nThis is generally not what was intended. A way around this is to use\n"None" as the default, and explicitly test for it in the body of the\nfunction, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n""*identifier"" is present, it is initialized to a tuple receiving any\nexcess positional parameters, defaulting to the empty tuple. If the\nform ""**identifier"" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary. Parameters after ""*"" or ""*identifier"" are\nkeyword-only parameters and may only be passed used keyword arguments.\n\nParameters may have annotations of the form "": expression"" following\nthe parameter name. Any parameter may have an annotation even those\nof the form "*identifier" or "**identifier". Functions may have\n"return" annotation of the form ""-> expression"" after the parameter\nlist. These annotations can be any valid Python expression and are\nevaluated when the function definition is executed. Annotations may\nbe evaluated in a different order than they appear in the source code.\nThe presence of annotations does not change the semantics of a\nfunction. The annotation values are available as values of a\ndictionary keyed by the parameters\' names in the "__annotations__"\nattribute of the function object.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda\nexpressions, described in section *Lambdas*. Note that the lambda\nexpression is merely a shorthand for a simplified function definition;\na function defined in a ""def"" statement can be passed around or\nassigned to another name just like a function defined by a lambda\nexpression. The ""def"" form is actually more powerful since it\nallows the execution of multiple statements and annotations.\n\n**Programmer\'s note:** Functions are first-class objects. A ""def""\nstatement executed inside a function definition defines a local\nfunction that can be returned or passed around. Free variables used\nin the nested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n\nSee also: **PEP 3107** - Function Annotations\n\n The original specification for function annotations.\n',
'global': '\nThe "global" statement\n**********************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe "global" statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without "global", although free variables may refer to\nglobals without being declared global.\n\nNames listed in a "global" statement must not be used in the same code\nblock textually preceding that "global" statement.\n\nNames listed in a "global" statement must not be defined as formal\nparameters or in a "for" loop control target, "class" definition,\nfunction definition, or "import" statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the "global" is a directive to the parser. It\napplies only to code parsed at the same time as the "global"\nstatement. In particular, a "global" statement contained in a string\nor code object supplied to the built-in "exec()" function does not\naffect the code block *containing* the function call, and code\ncontained in such a string is unaffected by "global" statements in the\ncode containing the function call. The same applies to the "eval()"\nand "compile()" functions.\n',
'id-classes': '\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n Not imported by "from module import *". The special identifier "_"\n is used in the interactive interpreter to store the result of the\n last evaluation; it is stored in the "builtins" module. When not\n in interactive mode, "_" has no special meaning and is not defined.\n See section *The import statement*.\n\n Note: The name "_" is often used in conjunction with\n internationalization; refer to the documentation for the\n "gettext" module for more information on this convention.\n\n"__*__"\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of "__*__" names, in any context, that does not\n follow explicitly documented use, is subject to breakage without\n warning.\n\n"__*"\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n',
'identifiers': '\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions.\n\nThe syntax of identifiers in Python is based on the Unicode standard\nannex UAX-31, with elaboration and changes as defined below; see also\n**PEP 3131** for further details.\n\nWithin the ASCII range (U+0001..U+007F), the valid characters for\nidentifiers are the same as in Python 2.x: the uppercase and lowercase\nletters "A" through "Z", the underscore "_" and, except for the first\ncharacter, the digits "0" through "9".\n\nPython 3.0 introduces additional characters from outside the ASCII\nrange (see **PEP 3131**). For these characters, the classification\nuses the version of the Unicode Character Database as included in the\n"unicodedata" module.\n\nIdentifiers are unlimited in length. Case is significant.\n\n identifier ::= xid_start xid_continue*\n id_start ::= <all characters in general categories Lu, Ll, Lt, Lm, Lo, Nl, the underscore, and characters with the Other_ID_Start property>\n id_continue ::= <all characters in id_start, plus characters in the categories Mn, Mc, Nd, Pc and others with the Other_ID_Continue property>\n xid_start ::= <all characters in id_start whose NFKC normalization is in "id_start xid_continue*">\n xid_continue ::= <all characters in id_continue whose NFKC normalization is in "id_continue*">\n\nThe Unicode category codes mentioned above stand for:\n\n* *Lu* - uppercase letters\n\n* *Ll* - lowercase letters\n\n* *Lt* - titlecase letters\n\n* *Lm* - modifier letters\n\n* *Lo* - other letters\n\n* *Nl* - letter numbers\n\n* *Mn* - nonspacing marks\n\n* *Mc* - spacing combining marks\n\n* *Nd* - decimal numbers\n\n* *Pc* - connector punctuations\n\n* *Other_ID_Start* - explicit list of characters in PropList.txt to\n support backwards compatibility\n\n* *Other_ID_Continue* - likewise\n\nAll identifiers are converted into the normal form NFKC while parsing;\ncomparison of identifiers is based on NFKC.\n\nA non-normative HTML file listing all valid identifier characters for\nUnicode 4.1 can be found at http://www.dcl.hpi.uni-\npotsdam.de/home/loewis/table-3131.html.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n False class finally is return\n None continue for lambda try\n True def from nonlocal while\n and del global not with\n as elif if or yield\n assert else import pass\n break except in raise\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n"_*"\n Not imported by "from module import *". The special identifier "_"\n is used in the interactive interpreter to store the result of the\n last evaluation; it is stored in the "builtins" module. When not\n in interactive mode, "_" has no special meaning and is not defined.\n See section *The import statement*.\n\n Note: The name "_" is often used in conjunction with\n internationalization; refer to the documentation for the\n "gettext" module for more information on this convention.\n\n"__*__"\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of "__*__" names, in any context, that does not\n follow explicitly documented use, is subject to breakage without\n warning.\n\n"__*"\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n',
@@ -49,24 +49,24 @@ topics = {'assert': '\nThe "assert" statement\n**********************\n\nAssert
'naming': '\nNaming and binding\n******************\n\n*Names* refer to objects. Names are introduced by name binding\noperations. Each occurrence of a name in the program text refers to\nthe *binding* of that name established in the innermost function block\ncontaining the use.\n\nA *block* is a piece of Python program text that is executed as a\nunit. The following are blocks: a module, a function body, and a class\ndefinition. Each command typed interactively is a block. A script\nfile (a file given as standard input to the interpreter or specified\non the interpreter command line the first argument) is a code block.\nA script command (a command specified on the interpreter command line\nwith the \'**-c**\' option) is a code block. The string argument passed\nto the built-in functions "eval()" and "exec()" is a code block.\n\nA code block is executed in an *execution frame*. A frame contains\nsome administrative information (used for debugging) and determines\nwhere and how execution continues after the code block\'s execution has\ncompleted.\n\nA *scope* defines the visibility of a name within a block. If a local\nvariable is defined in a block, its scope includes that block. If the\ndefinition occurs in a function block, the scope extends to any blocks\ncontained within the defining one, unless a contained block introduces\na different binding for the name. The scope of names defined in a\nclass block is limited to the class block; it does not extend to the\ncode blocks of methods -- this includes comprehensions and generator\nexpressions since they are implemented using a function scope. This\nmeans that the following will fail:\n\n class A:\n a = 42\n b = list(a + i for i in range(10))\n\nWhen a name is used in a code block, it is resolved using the nearest\nenclosing scope. The set of all such scopes visible to a code block\nis called the block\'s *environment*.\n\nIf a name is bound in a block, it is a local variable of that block,\nunless declared as "nonlocal". If a name is bound at the module\nlevel, it is a global variable. (The variables of the module code\nblock are local and global.) If a variable is used in a code block\nbut not defined there, it is a *free variable*.\n\nWhen a name is not found at all, a "NameError" exception is raised.\nIf the name refers to a local variable that has not been bound, a\n"UnboundLocalError" exception is raised. "UnboundLocalError" is a\nsubclass of "NameError".\n\nThe following constructs bind names: formal parameters to functions,\n"import" statements, class and function definitions (these bind the\nclass or function name in the defining block), and targets that are\nidentifiers if occurring in an assignment, "for" loop header, or after\n"as" in a "with" statement or "except" clause. The "import" statement\nof the form "from ... import *" binds all names defined in the\nimported module, except those beginning with an underscore. This form\nmay only be used at the module level.\n\nA target occurring in a "del" statement is also considered bound for\nthis purpose (though the actual semantics are to unbind the name).\n\nEach assignment or import statement occurs within a block defined by a\nclass or function definition or at the module level (the top-level\ncode block).\n\nIf a name binding operation occurs anywhere within a code block, all\nuses of the name within the block are treated as references to the\ncurrent block. This can lead to errors when a name is used within a\nblock before it is bound. This rule is subtle. Python lacks\ndeclarations and allows name binding operations to occur anywhere\nwithin a code block. The local variables of a code block can be\ndetermined by scanning the entire text of the block for name binding\noperations.\n\nIf the "global" statement occurs within a block, all uses of the name\nspecified in the statement refer to the binding of that name in the\ntop-level namespace. Names are resolved in the top-level namespace by\nsearching the global namespace, i.e. the namespace of the module\ncontaining the code block, and the builtins namespace, the namespace\nof the module "builtins". The global namespace is searched first. If\nthe name is not found there, the builtins namespace is searched. The\nglobal statement must precede all uses of the name.\n\nThe builtins namespace associated with the execution of a code block\nis actually found by looking up the name "__builtins__" in its global\nnamespace; this should be a dictionary or a module (in the latter case\nthe module\'s dictionary is used). By default, when in the "__main__"\nmodule, "__builtins__" is the built-in module "builtins"; when in any\nother module, "__builtins__" is an alias for the dictionary of the\n"builtins" module itself. "__builtins__" can be set to a user-created\ndictionary to create a weak form of restricted execution.\n\n**CPython implementation detail:** Users should not touch\n"__builtins__"; it is strictly an implementation detail. Users\nwanting to override values in the builtins namespace should "import"\nthe "builtins" module and modify its attributes appropriately.\n\nThe namespace for a module is automatically created the first time a\nmodule is imported. The main module for a script is always called\n"__main__".\n\nThe "global" statement has the same scope as a name binding operation\nin the same block. If the nearest enclosing scope for a free variable\ncontains a global statement, the free variable is treated as a global.\n\nA class definition is an executable statement that may use and define\nnames. These references follow the normal rules for name resolution.\nThe namespace of the class definition becomes the attribute dictionary\nof the class. Names defined at the class scope are not visible in\nmethods.\n\n\nInteraction with dynamic features\n=================================\n\nThere are several cases where Python statements are illegal when used\nin conjunction with nested scopes that contain free variables.\n\nIf a variable is referenced in an enclosing scope, it is illegal to\ndelete the name. An error will be reported at compile time.\n\nIf the wild card form of import --- "import *" --- is used in a\nfunction and the function contains or is a nested block with free\nvariables, the compiler will raise a "SyntaxError".\n\nThe "eval()" and "exec()" functions do not have access to the full\nenvironment for resolving names. Names may be resolved in the local\nand global namespaces of the caller. Free variables are not resolved\nin the nearest enclosing namespace, but in the global namespace. [1]\nThe "exec()" and "eval()" functions have optional arguments to\noverride the global and local namespace. If only one namespace is\nspecified, it is used for both.\n',
'nonlocal': '\nThe "nonlocal" statement\n************************\n\n nonlocal_stmt ::= "nonlocal" identifier ("," identifier)*\n\nThe "nonlocal" statement causes the listed identifiers to refer to\npreviously bound variables in the nearest enclosing scope. This is\nimportant because the default behavior for binding is to search the\nlocal namespace first. The statement allows encapsulated code to\nrebind variables outside of the local scope besides the global\n(module) scope.\n\nNames listed in a "nonlocal" statement, unlike to those listed in a\n"global" statement, must refer to pre-existing bindings in an\nenclosing scope (the scope in which a new binding should be created\ncannot be determined unambiguously).\n\nNames listed in a "nonlocal" statement must not collide with pre-\nexisting bindings in the local scope.\n\nSee also: **PEP 3104** - Access to Names in Outer Scopes\n\n The specification for the "nonlocal" statement.\n',
'numbers': '\nNumeric literals\n****************\n\nThere are three types of numeric literals: integers, floating point\nnumbers, and imaginary numbers. There are no complex literals\n(complex numbers can be formed by adding a real number and an\nimaginary number).\n\nNote that numeric literals do not include a sign; a phrase like "-1"\nis actually an expression composed of the unary operator \'"-"\' and the\nliteral "1".\n',
- 'numeric-types': '\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|"). For instance, to evaluate the\n expression "x + y", where *x* is an instance of a class that has an\n "__add__()" method, "x.__add__(y)" is called. The "__divmod__()"\n method should be the equivalent to using "__floordiv__()" and\n "__mod__()"; it should not be related to "__truediv__()". Note\n that "__pow__()" should be defined to accept an optional third\n argument if the ternary version of the built-in "pow()" function is\n to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|") with reflected (swapped) operands.\n These functions are only called if the left operand does not\n support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "/=", "//=", "%=", "**=", "<<=",\n ">>=", "&=", "^=", "|="). These methods should attempt to do the\n operation in-place (modifying *self*) and return the result (which\n could be, but does not have to be, *self*). If a specific method\n is not defined, the augmented assignment falls back to the normal\n methods. For instance, to execute the statement "x += y", where\n *x* is an instance of a class that has an "__iadd__()" method,\n "x.__iadd__(y)" is called. If *x* is an instance of a class that\n does not define a "__iadd__()" method, "x.__add__(y)" and\n "y.__radd__(x)" are considered, as with the evaluation of "x + y".\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()". Also called whenever\n Python needs an integer object (such as in slicing, or in the\n built-in "bin()", "hex()" and "oct()" functions). Must return an\n integer.\n',
+ 'numeric-types': '\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|"). For instance, to evaluate the\n expression "x + y", where *x* is an instance of a class that has an\n "__add__()" method, "x.__add__(y)" is called. The "__divmod__()"\n method should be the equivalent to using "__floordiv__()" and\n "__mod__()"; it should not be related to "__truediv__()". Note\n that "__pow__()" should be defined to accept an optional third\n argument if the ternary version of the built-in "pow()" function is\n to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|") with reflected (swapped) operands.\n These functions are only called if the left operand does not\n support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "/=", "//=", "%=", "**=", "<<=",\n ">>=", "&=", "^=", "|="). These methods should attempt to do the\n operation in-place (modifying *self*) and return the result (which\n could be, but does not have to be, *self*). If a specific method\n is not defined, the augmented assignment falls back to the normal\n methods. For instance, if *x* is an instance of a class with an\n "__iadd__()" method, "x += y" is equivalent to "x = x.__iadd__(y)"\n . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are considered, as\n with the evaluation of "x + y". In certain situations, augmented\n assignment can result in unexpected errors (see *Why does\n a_tuple[i] += [\'item\'] raise an exception when the addition\n works?*), but this behavior is in fact part of the data model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()", and whenever Python needs\n to losslessly convert the numeric object to an integer object (such\n as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n functions). Presence of this method indicates that the numeric\n object is an integer type. Must return an integer.\n\n Note: When "__index__()" is defined, "__int__()" should also be\n defined, and both shuld return the same value, in order to have a\n coherent integer type class.\n',
'objects': '\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'"is"\' operator compares the\nidentity of two objects; the "id()" function returns an integer\nrepresenting its identity.\n\n**CPython implementation detail:** For CPython, "id(x)" is the memory\naddress where "x" is stored.\n\nAn object\'s type determines the operations that the object supports\n(e.g., "does it have a length?") and also defines the possible values\nfor objects of that type. The "type()" function returns an object\'s\ntype (which is an object itself). Like its identity, an object\'s\n*type* is also unchangeable. [1]\n\nThe *value* of some objects can change. Objects whose value can\nchange are said to be *mutable*; objects whose value is unchangeable\nonce they are created are called *immutable*. (The value of an\nimmutable container object that contains a reference to a mutable\nobject can change when the latter\'s value is changed; however the\ncontainer is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the "gc" module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'"try"..."except"\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a "close()" method. Programs\nare strongly recommended to explicitly close such objects. The\n\'"try"..."finally"\' statement and the \'"with"\' statement provide\nconvenient ways to do this.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after "a = 1; b = 1",\n"a" and "b" may or may not refer to the same object with the value\none, depending on the implementation, but after "c = []; d = []", "c"\nand "d" are guaranteed to refer to two different, unique, newly\ncreated empty lists. (Note that "c = d = []" assigns the same object\nto both "c" and "d".)\n',
'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| "lambda" | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| "if" -- "else" | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| "or" | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| "and" | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| "not" "x" | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| "in", "not in", "is", "is not", "<", "<=", ">", | Comparisons, including membership |\n| ">=", "!=", "==" | tests and identity tests |\n+-------------------------------------------------+---------------------------------------+\n| "|" | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| "^" | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| "&" | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| "<<", ">>" | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| "+", "-" | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| "*", "/", "//", "%" | Multiplication, division, remainder |\n+-------------------------------------------------+---------------------------------------+\n| "+x", "-x", "~x" | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| "**" | Exponentiation [6] |\n+-------------------------------------------------+---------------------------------------+\n| "x[index]", "x[index:index]", | Subscription, slicing, call, |\n| "x(arguments...)", "x.attribute" | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| "(expressions...)", "[expressions...]", "{key: | Binding or tuple display, list |\n| value...}", "{expressions...}" | display, dictionary display, set |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] While "abs(x%y) < abs(y)" is true mathematically, for floats\n it may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that "-1e-100 % 1e100" have the same\n sign as "1e100", the computed result is "-1e-100 + 1e100", which\n is numerically exactly equal to "1e100". The function\n "math.fmod()" returns a result whose sign matches the sign of the\n first argument instead, and so returns "-1e-100" in this case.\n Which approach is more appropriate depends on the application.\n\n[2] If x is very close to an exact integer multiple of y, it\'s\n possible for "x//y" to be one larger than "(x-x%y)//y" due to\n rounding. In such cases, Python returns the latter result, in\n order to preserve that "divmod(x,y)[0] * y + x % y" be very close\n to "x".\n\n[3] While comparisons between strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ""\\u00C7"" and ""\\u0327\\u0043"" compare differently, even\n though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using "unicodedata.normalize()".\n\n[4] Due to automatic garbage-collection, free lists, and the\n dynamic nature of descriptors, you may notice seemingly unusual\n behaviour in certain uses of the "is" operator, like those\n involving comparisons between instance methods, or constants.\n Check their documentation for more info.\n\n[5] The "%" operator is also used for string formatting; the same\n precedence applies.\n\n[6] The power operator "**" binds less tightly than an arithmetic\n or bitwise unary operator on its right, that is, "2**-1" is "0.5".\n',
'pass': '\nThe "pass" statement\n********************\n\n pass_stmt ::= "pass"\n\n"pass" is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n',
'power': '\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): "-1**2" results in "-1".\n\nThe power operator has the same semantics as the built-in "pow()"\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type, and the result is of that type.\n\nFor int operands, the result has the same type as the operands unless\nthe second argument is negative; in that case, all arguments are\nconverted to float and a float result is delivered. For example,\n"10**2" returns "100", but "10**-2" returns "0.01".\n\nRaising "0.0" to a negative power results in a "ZeroDivisionError".\nRaising a negative number to a fractional power results in a "complex"\nnumber. (In earlier versions it raised a "ValueError".)\n',
'raise': '\nThe "raise" statement\n*********************\n\n raise_stmt ::= "raise" [expression ["from" expression]]\n\nIf no expressions are present, "raise" re-raises the last exception\nthat was active in the current scope. If no exception is active in\nthe current scope, a "RuntimeError" exception is raised indicating\nthat this is an error.\n\nOtherwise, "raise" evaluates the first expression as the exception\nobject. It must be either a subclass or an instance of\n"BaseException". If it is a class, the exception instance will be\nobtained when needed by instantiating the class with no arguments.\n\nThe *type* of the exception is the exception instance\'s class, the\n*value* is the instance itself.\n\nA traceback object is normally created automatically when an exception\nis raised and attached to it as the "__traceback__" attribute, which\nis writable. You can create an exception and set your own traceback in\none step using the "with_traceback()" exception method (which returns\nthe same exception instance, with its traceback set to its argument),\nlike so:\n\n raise Exception("foo occurred").with_traceback(tracebackobj)\n\nThe "from" clause is used for exception chaining: if given, the second\n*expression* must be another exception class or instance, which will\nthen be attached to the raised exception as the "__cause__" attribute\n(which is writable). If the raised exception is not handled, both\nexceptions will be printed:\n\n >>> try:\n ... print(1 / 0)\n ... except Exception as exc:\n ... raise RuntimeError("Something bad happened") from exc\n ...\n Traceback (most recent call last):\n File "<stdin>", line 2, in <module>\n ZeroDivisionError: int division or modulo by zero\n\n The above exception was the direct cause of the following exception:\n\n Traceback (most recent call last):\n File "<stdin>", line 4, in <module>\n RuntimeError: Something bad happened\n\nA similar mechanism works implicitly if an exception is raised inside\nan exception handler: the previous exception is then attached as the\nnew exception\'s "__context__" attribute:\n\n >>> try:\n ... print(1 / 0)\n ... except:\n ... raise RuntimeError("Something bad happened")\n ...\n Traceback (most recent call last):\n File "<stdin>", line 2, in <module>\n ZeroDivisionError: int division or modulo by zero\n\n During handling of the above exception, another exception occurred:\n\n Traceback (most recent call last):\n File "<stdin>", line 4, in <module>\n RuntimeError: Something bad happened\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n',
'return': '\nThe "return" statement\n**********************\n\n return_stmt ::= "return" [expression_list]\n\n"return" may only occur syntactically nested in a function definition,\nnot within a nested class definition.\n\nIf an expression list is present, it is evaluated, else "None" is\nsubstituted.\n\n"return" leaves the current function call with the expression list (or\n"None") as return value.\n\nWhen "return" passes control out of a "try" statement with a "finally"\nclause, that "finally" clause is executed before really leaving the\nfunction.\n\nIn a generator function, the "return" statement indicates that the\ngenerator is done and will cause "StopIteration" to be raised. The\nreturned value (if any) is used as an argument to construct\n"StopIteration" and becomes the "StopIteration.value" attribute.\n',
- 'sequence-types': '\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method "keys()".\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n',
+ 'sequence-types': '\nEmulating container types\n*************************\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nobject.__length_hint__(self)\n\n Called to implement "operator.length_hint()". Should return an\n estimated length for the object (which may be greater or less than\n the actual length). The length must be an integer ">=" 0. This\n method is purely an optimization and is never required for\n correctness.\n\n New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method "keys()".\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n',
'shifting': '\nShifting operations\n*******************\n\nThe shifting operations have lower priority than the arithmetic\noperations:\n\n shift_expr ::= a_expr | shift_expr ( "<<" | ">>" ) a_expr\n\nThese operators accept integers as arguments. They shift the first\nargument to the left or right by the number of bits given by the\nsecond argument.\n\nA right shift by *n* bits is defined as floor division by "pow(2,n)".\nA left shift by *n* bits is defined as multiplication with "pow(2,n)".\n\nNote: In the current implementation, the right-hand operand is\n required to be at most "sys.maxsize". If the right-hand operand is\n larger than "sys.maxsize" an "OverflowError" exception is raised.\n',
'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or "del" statements. The syntax for a slicing:\n\n slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice\n proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows. The primary must evaluate\nto a mapping object, and it is indexed (using the same "__getitem__()"\nmethod as normal subscription) with a key that is constructed from the\nslice list, as follows. If the slice list contains at least one\ncomma, the key is a tuple containing the conversion of the slice\nitems; otherwise, the conversion of the lone slice item is the key.\nThe conversion of a slice item that is an expression is that\nexpression. The conversion of a proper slice is a slice object (see\nsection *The standard type hierarchy*) whose "start", "stop" and\n"step" attributes are the values of the expressions given as lower\nbound, upper bound and stride, respectively, substituting "None" for\nmissing expressions.\n',
'specialattrs': '\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the "dir()" built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object\'s\n (writable) attributes.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nclass.__qualname__\n\n The *qualified name* of the class or type.\n\n New in version 3.3.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in "__mro__".\n\nclass.__subclasses__()\n\n Each class keeps a list of weak references to its immediate\n subclasses. This method returns a list of all those references\n still alive. Example:\n\n >>> int.__subclasses__()\n [<class \'bool\'>]\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found\n in the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list "[1, 2]" is considered equal to\n "[1.0, 2.0]", and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n operands.\n\n[4] Cased characters are those with general category property\n being one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase),\n or "Lt" (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a\n singleton tuple whose only element is the tuple to be formatted.\n',
- 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named "__getitem__()", and "x" is an instance of this class,\nthen "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)".\nExcept where mentioned, attempts to execute an operation raise an\nexception when no appropriate method is defined (typically\n"AttributeError" or "TypeError").\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n"NodeList" interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])". As a special constraint on\n constructors, no value may be returned; doing so will cause a\n "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing "None" in\n "sys.last_traceback". Circular references which are garbage are\n detected when the option cycle detector is enabled (it\'s on by\n default), but can only be cleaned up if there are no Python-\n level "__del__()" methods involved. Refer to the documentation\n for the "gc" module for more information about how "__del__()"\n methods are handled by the cycle detector, particularly the\n description of the "garbage" value.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n <ParentClass>.__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for "self"). "name" is the attribute name. This\n method should return the (computed) attribute value or raise an\n "AttributeError" exception.\n\n Note that if the attribute is found through the normal mechanism,\n "__getattr__()" is not called. (This is an intentional asymmetry\n between "__getattr__()" and "__setattr__()".) This is done both for\n efficiency reasons and because otherwise "__getattr__()" would have\n no way to access other attributes of the instance. Note that at\n least for instance variables, you can fake total control by not\n inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n "__getattribute__()" method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines "__getattr__()",\n the latter will not be called unless "__getattribute__()" either\n calls it explicitly or raises an "AttributeError". This method\n should return the (computed) attribute value or raise an\n "AttributeError" exception. In order to avoid infinite recursion in\n this method, its implementation should always call the base class\n method with the same name to access any attributes it needs, for\n example, "object.__getattribute__(self, name)".\n\n Note: This method may still be bypassed when looking up special\n methods as the result of implicit invocation via language syntax\n or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If "__setattr__()" wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n Like "__setattr__()" but for attribute deletion instead of\n assignment. This should only be implemented if "del obj.name" is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when "dir()" is called on the object. A sequence must be\n returned. "dir()" converts the returned sequence to a list and\n sorts it.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or "None" when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an "AttributeError"\n exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: "x.__get__(a)".\n\nInstance Binding\n If binding to an object instance, "a.x" is transformed into the\n call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n If binding to a class, "A.x" is transformed into the call:\n "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n If "a" is an instance of "super", then the binding "super(B,\n obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n immediately preceding "B" and then invokes the descriptor with the\n call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()". If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary. If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor. Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method. Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary. In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n class, *__slots__* reserves space for the declared variables and\n prevents the automatic creation of *__dict__* and *__weakref__* for\n each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises "AttributeError". If\n dynamic assignment of new variables is desired, then add\n "\'__dict__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n defining *__slots__* do not support weak references to its\n instances. If weak reference support is needed, then add\n "\'__weakref__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n instance variable defined by the base class slot is inaccessible\n (except by retrieving its descriptor directly from the base class).\n This renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n may also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using "type()". The class body is\nexecuted in a new namespace and the class name is bound locally to the\nresult of "type(name, bases, namespace)".\n\nThe class creation process can be customised by passing the\n"metaclass" keyword argument in the class definition line, or by\ninheriting from an existing class that included such an argument. In\nthe following example, both "MyClass" and "MySubclass" are instances\nof "Meta":\n\n class Meta(type):\n pass\n\n class MyClass(metaclass=Meta):\n pass\n\n class MySubclass(MyClass):\n pass\n\nAny other keyword arguments that are specified in the class definition\nare passed through to all metaclass operations described below.\n\nWhen a class definition is executed, the following steps occur:\n\n* the appropriate metaclass is determined\n\n* the class namespace is prepared\n\n* the class body is executed\n\n* the class object is created\n\n\nDetermining the appropriate metaclass\n-------------------------------------\n\nThe appropriate metaclass for a class definition is determined as\nfollows:\n\n* if no bases and no explicit metaclass are given, then "type()" is\n used\n\n* if an explicit metaclass is given and it is *not* an instance of\n "type()", then it is used directly as the metaclass\n\n* if an instance of "type()" is given as the explicit metaclass, or\n bases are defined, then the most derived metaclass is used\n\nThe most derived metaclass is selected from the explicitly specified\nmetaclass (if any) and the metaclasses (i.e. "type(cls)") of all\nspecified base classes. The most derived metaclass is one which is a\nsubtype of *all* of these candidate metaclasses. If none of the\ncandidate metaclasses meets that criterion, then the class definition\nwill fail with "TypeError".\n\n\nPreparing the class namespace\n-----------------------------\n\nOnce the appropriate metaclass has been identified, then the class\nnamespace is prepared. If the metaclass has a "__prepare__" attribute,\nit is called as "namespace = metaclass.__prepare__(name, bases,\n**kwds)" (where the additional keyword arguments, if any, come from\nthe class definition).\n\nIf the metaclass has no "__prepare__" attribute, then the class\nnamespace is initialised as an empty "dict()" instance.\n\nSee also: **PEP 3115** - Metaclasses in Python 3000\n\n Introduced the "__prepare__" namespace hook\n\n\nExecuting the class body\n------------------------\n\nThe class body is executed (approximately) as "exec(body, globals(),\nnamespace)". The key difference from a normal call to "exec()" is that\nlexical scoping allows the class body (including any methods) to\nreference names from the current and outer scopes when the class\ndefinition occurs inside a function.\n\nHowever, even when the class definition occurs inside the function,\nmethods defined inside the class still cannot see names defined at the\nclass scope. Class variables must be accessed through the first\nparameter of instance or class methods, and cannot be accessed at all\nfrom static methods.\n\n\nCreating the class object\n-------------------------\n\nOnce the class namespace has been populated by executing the class\nbody, the class object is created by calling "metaclass(name, bases,\nnamespace, **kwds)" (the additional keywords passed here are the same\nas those passed to "__prepare__").\n\nThis class object is the one that will be referenced by the zero-\nargument form of "super()". "__class__" is an implicit closure\nreference created by the compiler if any methods in a class body refer\nto either "__class__" or "super". This allows the zero argument form\nof "super()" to correctly identify the class being defined based on\nlexical scoping, while the class or instance that was used to make the\ncurrent call is identified based on the first argument passed to the\nmethod.\n\nAfter the class object is created, it is passed to the class\ndecorators included in the class definition (if any) and the resulting\nobject is bound in the local namespace as the defined class.\n\nSee also: **PEP 3135** - New super\n\n Describes the implicit "__class__" closure reference\n\n\nMetaclass example\n-----------------\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored include logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n"collections.OrderedDict" to remember the order that class members\nwere defined:\n\n class OrderedClass(type):\n\n @classmethod\n def __prepare__(metacls, name, bases, **kwds):\n return collections.OrderedDict()\n\n def __new__(cls, name, bases, namespace, **kwds):\n result = type.__new__(cls, name, bases, dict(namespace))\n result.members = tuple(namespace)\n return result\n\n class A(metaclass=OrderedClass):\n def one(self): pass\n def two(self): pass\n def three(self): pass\n def four(self): pass\n\n >>> A.members\n (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s "__prepare__()" method which returns an\nempty "collections.OrderedDict". That mapping records the methods and\nattributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s "__new__()" method gets\ninvoked. That method builds the new type and it saves the ordered\ndictionary keys in an attribute called "members".\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n"isinstance()" and "issubclass()" built-in functions.\n\nIn particular, the metaclass "abc.ABCMeta" implements these methods in\norder to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n "isinstance(instance, class)".\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n "issubclass(subclass, class)".\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also: **PEP 3119** - Introducing Abstract Base Classes\n\n Includes the specification for customizing "isinstance()" and\n "issubclass()" behavior through "__instancecheck__()" and\n "__subclasscheck__()", with motivation for this functionality in\n the context of adding Abstract Base Classes (see the "abc"\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, "x(arg1, arg2, ...)" is a shorthand for\n "x.__call__(arg1, arg2, ...)".\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method "keys()".\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|"). For instance, to evaluate the\n expression "x + y", where *x* is an instance of a class that has an\n "__add__()" method, "x.__add__(y)" is called. The "__divmod__()"\n method should be the equivalent to using "__floordiv__()" and\n "__mod__()"; it should not be related to "__truediv__()". Note\n that "__pow__()" should be defined to accept an optional third\n argument if the ternary version of the built-in "pow()" function is\n to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|") with reflected (swapped) operands.\n These functions are only called if the left operand does not\n support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "/=", "//=", "%=", "**=", "<<=",\n ">>=", "&=", "^=", "|="). These methods should attempt to do the\n operation in-place (modifying *self*) and return the result (which\n could be, but does not have to be, *self*). If a specific method\n is not defined, the augmented assignment falls back to the normal\n methods. For instance, to execute the statement "x += y", where\n *x* is an instance of a class that has an "__iadd__()" method,\n "x.__iadd__(y)" is called. If *x* is an instance of a class that\n does not define a "__iadd__()" method, "x.__add__(y)" and\n "y.__radd__(x)" are considered, as with the evaluation of "x + y".\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()". Also called whenever\n Python needs an integer object (such as in slicing, or in the\n built-in "bin()", "hex()" and "oct()" functions). Must return an\n integer.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code. Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The "with"\n statement will bind this method\'s return value to the target(s)\n specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be "None".\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that "__exit__()" methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception:\n\n >>> class C:\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as "__hash__()" and "__repr__()" that are implemented by\nall objects, including type objects. If the implicit lookup of these\nmethods used the conventional lookup process, they would fail when\ninvoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe "__getattribute__()" method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print("Metaclass getattribute invoked")\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object, metaclass=Meta):\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print("Class getattribute invoked")\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the "__getattribute__()" machinery in this fashion provides\nsignificant scope for speed optimisations within the interpreter, at\nthe cost of some flexibility in the handling of special methods (the\nspecial method *must* be set on the class object itself in order to be\nconsistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type,\n under certain controlled conditions. It generally isn\'t a good\n idea though, since it can lead to some very strange behaviour if\n it is handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as "__add__()") fails the operation is not\n supported, which is why the reflected method is not called.\n',
- 'string-methods': '\nString Methods\n**************\n\nStrings implement all of the *common* sequence operations, along with\nthe additional methods described below.\n\nStrings also support two styles of string formatting, one providing a\nlarge degree of flexibility and customization (see "str.format()",\n*Format String Syntax* and *String Formatting*) and the other based on\nC "printf" style formatting that handles a narrower range of types and\nis slightly harder to use correctly, but is often faster for the cases\nit can handle (*printf-style String Formatting*).\n\nThe *Text Processing Services* section of the standard library covers\na number of other modules that provide various text related utilities\n(including regular expression support in the "re" module).\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.casefold()\n\n Return a casefolded copy of the string. Casefolded strings may be\n used for caseless matching.\n\n Casefolding is similar to lowercasing but more aggressive because\n it is intended to remove all case distinctions in a string. For\n example, the German lowercase letter "\'\xc3\x9f\'" is equivalent to ""ss"".\n Since it is already lowercase, "lower()" would do nothing to "\'\xc3\x9f\'";\n "casefold()" converts it to ""ss"".\n\n The casefolding algorithm is described in section 3.13 of the\n Unicode Standard.\n\n New in version 3.3.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is "\'utf-8\'". *errors* may be given to set a different\n error handling scheme. The default for *errors* is "\'strict\'",\n meaning that encoding errors raise a "UnicodeError". Other possible\n values are "\'ignore\'", "\'replace\'", "\'xmlcharrefreplace\'",\n "\'backslashreplace\'" and any other name registered via\n "codecs.register_error()", see section *Codec Base Classes*. For a\n list of possible encodings, see section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return "True" if the string ends with the specified *suffix*,\n otherwise return "False". *suffix* can also be a tuple of suffixes\n to look for. With optional *start*, test beginning at that\n position. With optional *end*, stop comparing at that position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab ("\\t"), one or more space characters are inserted in the result\n until the current column is equal to the next tab position. (The\n tab character itself is not copied.) If the character is a newline\n ("\\n") or return ("\\r"), it is copied and the current column is\n reset to zero. Any other character is copied unchanged and the\n current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" if *sub* is not found.\n\n Note: The "find()" method should be used only if you need to know\n the position of *sub*. To check if *sub* is a substring or not,\n use the "in" operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces "{}". Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to "str.format(**mapping)", except that "mapping" is used\n directly and not copied to a "dict". This is useful if for example\n "mapping" is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like "find()", but raise "ValueError" when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character "c"\n is alphanumeric if one of the following returns "True":\n "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()".\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that can be used to\n form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\n Use "keyword.iskeyword()" to test for reserved identifiers such as\n "def" and "class".\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when "repr()" is\n invoked on a string. It has no bearing on the handling of strings\n written to "sys.stdout" or "sys.stderr".)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A "TypeError" will be raised if there are\n any non-string values in *iterable*, including "bytes" objects.\n The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n The lowercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n "str.translate()".\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like "rfind()" but raises "ValueError" when the substring *sub* is\n not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n "None", any whitespace string is a separator. Except for splitting\n from the right, "rsplit()" behaves like "split()" which is\n described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most "maxsplit+1"\n elements). If *maxsplit* is not specified or "-1", then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', \'2\']"). The *sep* argument\n may consist of multiple characters (for example,\n "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', \'3\']"). Splitting an\n empty string with a specified separator returns "[\'\']".\n\n If *sep* is not specified or is "None", a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a "None" separator returns "[]".\n\n For example, "\' 1 2 3 \'.split()" returns "[\'1\', \'2\', \'3\']", and\n "\' 1 2 3 \'.split(None, 1)" returns "[\'1\', \'2 3 \']".\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, "\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()" returns "[\'ab\n c\', \'\', \'de fg\', \'kl\']", while the same call with\n "splitlines(True)" returns "[\'ab c\\n\', \'\\n\', \'de fg\\r\', \'kl\\r\\n\']".\n\n Unlike "split()" when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return "True" if string starts with the *prefix*, otherwise return\n "False". *prefix* can also be a tuple of prefixes to look for.\n With optional *start*, test string beginning at that position.\n With optional *end*, stop comparing string at that position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or "None", the *chars*\n argument defaults to removing whitespace. The *chars* argument is\n not a prefix or suffix; rather, all combinations of its values are\n stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa. Note that it is not necessarily true that\n "s.swapcase().swapcase() == s".\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or "None". Unmapped\n characters are left untouched. Characters mapped to "None" are\n deleted.\n\n You can use "str.maketrans()" to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom\n character mapping codec using the "codecs" module (see\n "encodings.cp1251" for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that "str.upper().isupper()" might be\n "False" if "s" contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n The uppercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to "len(s)".\n',
+ 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named "__getitem__()", and "x" is an instance of this class,\nthen "x[i]" is roughly equivalent to "type(x).__getitem__(x, i)".\nExcept where mentioned, attempts to execute an operation raise an\nexception when no appropriate method is defined (typically\n"AttributeError" or "TypeError").\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n"NodeList" interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. "__new__()" is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of "__new__()" should be the new object instance (usually an\n instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s "__new__()" method using\n "super(currentclass, cls).__new__(cls[, ...])" with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If "__new__()" returns an instance of *cls*, then the new\n instance\'s "__init__()" method will be invoked like\n "__init__(self[, ...])", where *self* is the new instance and the\n remaining arguments are the same as were passed to "__new__()".\n\n If "__new__()" does not return an instance of *cls*, then the new\n instance\'s "__init__()" method will not be invoked.\n\n "__new__()" is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n "__init__()" method, the derived class\'s "__init__()" method, if\n any, must explicitly call it to ensure proper initialization of the\n base class part of the instance; for example:\n "BaseClass.__init__(self, [args...])". As a special constraint on\n constructors, no value may be returned; doing so will cause a\n "TypeError" to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a "__del__()" method, the\n derived class\'s "__del__()" method, if any, must explicitly call it\n to ensure proper deletion of the base class part of the instance.\n Note that it is possible (though not recommended!) for the\n "__del__()" method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n "__del__()" methods are called for objects that still exist when\n the interpreter exits.\n\n Note: "del x" doesn\'t directly call "x.__del__()" --- the former\n decrements the reference count for "x" by one, and the latter is\n only called when "x"\'s reference count reaches zero. Some common\n situations that may prevent the reference count of an object from\n going to zero include: circular references between objects (e.g.,\n a doubly-linked list or a tree data structure with parent and\n child pointers); a reference to the object on the stack frame of\n a function that caught an exception (the traceback stored in\n "sys.exc_info()[2]" keeps the stack frame alive); or a reference\n to the object on the stack frame that raised an unhandled\n exception in interactive mode (the traceback stored in\n "sys.last_traceback" keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing "None" in\n "sys.last_traceback". Circular references which are garbage are\n detected and cleaned up when the cyclic garbage collector is\n enabled (it\'s on by default). Refer to the documentation for the\n "gc" module for more information about this topic.\n\n Warning: Due to the precarious circumstances under which\n "__del__()" methods are invoked, exceptions that occur during\n their execution are ignored, and a warning is printed to\n "sys.stderr" instead. Also, when "__del__()" is invoked in\n response to a module being deleted (e.g., when execution of the\n program is done), other globals referenced by the "__del__()"\n method may already have been deleted or in the process of being\n torn down (e.g. the import machinery shutting down). For this\n reason, "__del__()" methods should do the absolute minimum needed\n to maintain external invariants. Starting with version 1.5,\n Python guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the "__del__()" method is called.\n\nobject.__repr__(self)\n\n Called by the "repr()" built-in function to compute the "official"\n string representation of an object. If at all possible, this\n should look like a valid Python expression that could be used to\n recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n "<...some useful description...>" should be returned. The return\n value must be a string object. If a class defines "__repr__()" but\n not "__str__()", then "__repr__()" is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by "str(object)" and the built-in functions "format()" and\n "print()" to compute the "informal" or nicely printable string\n representation of an object. The return value must be a *string*\n object.\n\n This method differs from "object.__repr__()" in that there is no\n expectation that "__str__()" return a valid Python expression: a\n more convenient or concise representation can be used.\n\n The default implementation defined by the built-in type "object"\n calls "object.__repr__()".\n\nobject.__bytes__(self)\n\n Called by "bytes()" to compute a byte-string representation of an\n object. This should return a "bytes" object.\n\nobject.__format__(self, format_spec)\n\n Called by the "format()" built-in function (and by extension, the\n "str.format()" method of class "str") to produce a "formatted"\n string representation of an object. The "format_spec" argument is a\n string that contains a description of the formatting options\n desired. The interpretation of the "format_spec" argument is up to\n the type implementing "__format__()", however most classes will\n either delegate formatting to one of the built-in types, or use a\n similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\n Changed in version 3.4: The __format__ method of "object" itself\n raises a "TypeError" if passed any non-empty string.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: "x<y" calls "x.__lt__(y)", "x<=y" calls "x.__le__(y)",\n "x==y" calls "x.__eq__(y)", "x!=y" calls "x.__ne__(y)", "x>y" calls\n "x.__gt__(y)", and "x>=y" calls "x.__ge__(y)".\n\n A rich comparison method may return the singleton "NotImplemented"\n if it does not implement the operation for a given pair of\n arguments. By convention, "False" and "True" are returned for a\n successful comparison. However, these methods can return any value,\n so if the comparison operator is used in a Boolean context (e.g.,\n in the condition of an "if" statement), Python will call "bool()"\n on the value to determine if the result is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of "x==y" does not imply that "x!=y" is false.\n Accordingly, when defining "__eq__()", one should also define\n "__ne__()" so that the operators will behave as expected. See the\n paragraph on "__hash__()" for some important notes on creating\n *hashable* objects which support custom comparison operations and\n are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, "__lt__()" and "__gt__()" are each other\'s\n reflection, "__le__()" and "__ge__()" are each other\'s reflection,\n and "__eq__()" and "__ne__()" are their own reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see "functools.total_ordering()".\n\nobject.__hash__(self)\n\n Called by built-in function "hash()" and for operations on members\n of hashed collections including "set", "frozenset", and "dict".\n "__hash__()" should return an integer. The only required property\n is that objects which compare equal have the same hash value; it is\n advised to somehow mix together (e.g. using exclusive or) the hash\n values for the components of the object that also play a part in\n comparison of objects.\n\n Note: "hash()" truncates the value returned from an object\'s\n custom "__hash__()" method to the size of a "Py_ssize_t". This\n is typically 8 bytes on 64-bit builds and 4 bytes on 32-bit\n builds. If an object\'s "__hash__()" must interoperate on builds\n of different bit sizes, be sure to check the width on all\n supported builds. An easy way to do this is with "python -c\n "import sys; print(sys.hash_info.width)""\n\n If a class does not define an "__eq__()" method it should not\n define a "__hash__()" operation either; if it defines "__eq__()"\n but not "__hash__()", its instances will not be usable as items in\n hashable collections. If a class defines mutable objects and\n implements an "__eq__()" method, it should not implement\n "__hash__()", since the implementation of hashable collections\n requires that a key\'s hash value is immutable (if the object\'s hash\n value changes, it will be in the wrong hash bucket).\n\n User-defined classes have "__eq__()" and "__hash__()" methods by\n default; with them, all objects compare unequal (except with\n themselves) and "x.__hash__()" returns an appropriate value such\n that "x == y" implies both that "x is y" and "hash(x) == hash(y)".\n\n A class that overrides "__eq__()" and does not define "__hash__()"\n will have its "__hash__()" implicitly set to "None". When the\n "__hash__()" method of a class is "None", instances of the class\n will raise an appropriate "TypeError" when a program attempts to\n retrieve their hash value, and will also be correctly identified as\n unhashable when checking "isinstance(obj, collections.Hashable").\n\n If a class that overrides "__eq__()" needs to retain the\n implementation of "__hash__()" from a parent class, the interpreter\n must be told this explicitly by setting "__hash__ =\n <ParentClass>.__hash__".\n\n If a class that does not override "__eq__()" wishes to suppress\n hash support, it should include "__hash__ = None" in the class\n definition. A class which defines its own "__hash__()" that\n explicitly raises a "TypeError" would be incorrectly identified as\n hashable by an "isinstance(obj, collections.Hashable)" call.\n\n Note: By default, the "__hash__()" values of str, bytes and\n datetime objects are "salted" with an unpredictable random value.\n Although they remain constant within an individual Python\n process, they are not predictable between repeated invocations of\n Python.This is intended to provide protection against a denial-\n of-service caused by carefully-chosen inputs that exploit the\n worst case performance of a dict insertion, O(n^2) complexity.\n See http://www.ocert.org/advisories/ocert-2011-003.html for\n details.Changing hash values affects the iteration order of\n dicts, sets and other mappings. Python has never made guarantees\n about this ordering (and it typically varies between 32-bit and\n 64-bit builds).See also "PYTHONHASHSEED".\n\n Changed in version 3.3: Hash randomization is enabled by default.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n "bool()"; should return "False" or "True". When this method is not\n defined, "__len__()" is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither "__len__()" nor "__bool__()", all its instances are\n considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of "x.name") for\nclass instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for "self"). "name" is the attribute name. This\n method should return the (computed) attribute value or raise an\n "AttributeError" exception.\n\n Note that if the attribute is found through the normal mechanism,\n "__getattr__()" is not called. (This is an intentional asymmetry\n between "__getattr__()" and "__setattr__()".) This is done both for\n efficiency reasons and because otherwise "__getattr__()" would have\n no way to access other attributes of the instance. Note that at\n least for instance variables, you can fake total control by not\n inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n "__getattribute__()" method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines "__getattr__()",\n the latter will not be called unless "__getattribute__()" either\n calls it explicitly or raises an "AttributeError". This method\n should return the (computed) attribute value or raise an\n "AttributeError" exception. In order to avoid infinite recursion in\n this method, its implementation should always call the base class\n method with the same name to access any attributes it needs, for\n example, "object.__getattribute__(self, name)".\n\n Note: This method may still be bypassed when looking up special\n methods as the result of implicit invocation via language syntax\n or built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If "__setattr__()" wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n "object.__setattr__(self, name, value)".\n\nobject.__delattr__(self, name)\n\n Like "__setattr__()" but for attribute deletion instead of\n assignment. This should only be implemented if "del obj.name" is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when "dir()" is called on the object. A sequence must be\n returned. "dir()" converts the returned sequence to a list and\n sorts it.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' "__dict__".\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or "None" when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an "AttributeError"\n exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: "__get__()", "__set__()", and\n"__delete__()". If any of those methods are defined for an object, it\nis said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, "a.x" has a\nlookup chain starting with "a.__dict__[\'x\']", then\n"type(a).__dict__[\'x\']", and continuing through the base classes of\n"type(a)" excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, "a.x". How\nthe arguments are assembled depends on "a":\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: "x.__get__(a)".\n\nInstance Binding\n If binding to an object instance, "a.x" is transformed into the\n call: "type(a).__dict__[\'x\'].__get__(a, type(a))".\n\nClass Binding\n If binding to a class, "A.x" is transformed into the call:\n "A.__dict__[\'x\'].__get__(None, A)".\n\nSuper Binding\n If "a" is an instance of "super", then the binding "super(B,\n obj).m()" searches "obj.__class__.__mro__" for the base class "A"\n immediately preceding "B" and then invokes the descriptor with the\n call: "A.__dict__[\'m\'].__get__(obj, obj.__class__)".\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of "__get__()", "__set__()" and "__delete__()". If it\ndoes not define "__get__()", then accessing the attribute will return\nthe descriptor object itself unless there is a value in the object\'s\ninstance dictionary. If the descriptor defines "__set__()" and/or\n"__delete__()", it is a data descriptor; if it defines neither, it is\na non-data descriptor. Normally, data descriptors define both\n"__get__()" and "__set__()", while non-data descriptors have just the\n"__get__()" method. Data descriptors with "__set__()" and "__get__()"\ndefined always override a redefinition in an instance dictionary. In\ncontrast, non-data descriptors can be overridden by instances.\n\nPython methods (including "staticmethod()" and "classmethod()") are\nimplemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe "property()" function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n class, *__slots__* reserves space for the declared variables and\n prevents the automatic creation of *__dict__* and *__weakref__* for\n each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises "AttributeError". If\n dynamic assignment of new variables is desired, then add\n "\'__dict__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes\n defining *__slots__* do not support weak references to its\n instances. If weak reference support is needed, then add\n "\'__weakref__\'" to the sequence of strings in the *__slots__*\n declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the\n instance variable defined by the base class slot is inaccessible\n (except by retrieving its descriptor directly from the base class).\n This renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as "int", "bytes" and "tuple".\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings\n may also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using "type()". The class body is\nexecuted in a new namespace and the class name is bound locally to the\nresult of "type(name, bases, namespace)".\n\nThe class creation process can be customised by passing the\n"metaclass" keyword argument in the class definition line, or by\ninheriting from an existing class that included such an argument. In\nthe following example, both "MyClass" and "MySubclass" are instances\nof "Meta":\n\n class Meta(type):\n pass\n\n class MyClass(metaclass=Meta):\n pass\n\n class MySubclass(MyClass):\n pass\n\nAny other keyword arguments that are specified in the class definition\nare passed through to all metaclass operations described below.\n\nWhen a class definition is executed, the following steps occur:\n\n* the appropriate metaclass is determined\n\n* the class namespace is prepared\n\n* the class body is executed\n\n* the class object is created\n\n\nDetermining the appropriate metaclass\n-------------------------------------\n\nThe appropriate metaclass for a class definition is determined as\nfollows:\n\n* if no bases and no explicit metaclass are given, then "type()" is\n used\n\n* if an explicit metaclass is given and it is *not* an instance of\n "type()", then it is used directly as the metaclass\n\n* if an instance of "type()" is given as the explicit metaclass, or\n bases are defined, then the most derived metaclass is used\n\nThe most derived metaclass is selected from the explicitly specified\nmetaclass (if any) and the metaclasses (i.e. "type(cls)") of all\nspecified base classes. The most derived metaclass is one which is a\nsubtype of *all* of these candidate metaclasses. If none of the\ncandidate metaclasses meets that criterion, then the class definition\nwill fail with "TypeError".\n\n\nPreparing the class namespace\n-----------------------------\n\nOnce the appropriate metaclass has been identified, then the class\nnamespace is prepared. If the metaclass has a "__prepare__" attribute,\nit is called as "namespace = metaclass.__prepare__(name, bases,\n**kwds)" (where the additional keyword arguments, if any, come from\nthe class definition).\n\nIf the metaclass has no "__prepare__" attribute, then the class\nnamespace is initialised as an empty "dict()" instance.\n\nSee also: **PEP 3115** - Metaclasses in Python 3000\n\n Introduced the "__prepare__" namespace hook\n\n\nExecuting the class body\n------------------------\n\nThe class body is executed (approximately) as "exec(body, globals(),\nnamespace)". The key difference from a normal call to "exec()" is that\nlexical scoping allows the class body (including any methods) to\nreference names from the current and outer scopes when the class\ndefinition occurs inside a function.\n\nHowever, even when the class definition occurs inside the function,\nmethods defined inside the class still cannot see names defined at the\nclass scope. Class variables must be accessed through the first\nparameter of instance or class methods, and cannot be accessed at all\nfrom static methods.\n\n\nCreating the class object\n-------------------------\n\nOnce the class namespace has been populated by executing the class\nbody, the class object is created by calling "metaclass(name, bases,\nnamespace, **kwds)" (the additional keywords passed here are the same\nas those passed to "__prepare__").\n\nThis class object is the one that will be referenced by the zero-\nargument form of "super()". "__class__" is an implicit closure\nreference created by the compiler if any methods in a class body refer\nto either "__class__" or "super". This allows the zero argument form\nof "super()" to correctly identify the class being defined based on\nlexical scoping, while the class or instance that was used to make the\ncurrent call is identified based on the first argument passed to the\nmethod.\n\nAfter the class object is created, it is passed to the class\ndecorators included in the class definition (if any) and the resulting\nobject is bound in the local namespace as the defined class.\n\nSee also: **PEP 3135** - New super\n\n Describes the implicit "__class__" closure reference\n\n\nMetaclass example\n-----------------\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored include logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n"collections.OrderedDict" to remember the order that class members\nwere defined:\n\n class OrderedClass(type):\n\n @classmethod\n def __prepare__(metacls, name, bases, **kwds):\n return collections.OrderedDict()\n\n def __new__(cls, name, bases, namespace, **kwds):\n result = type.__new__(cls, name, bases, dict(namespace))\n result.members = tuple(namespace)\n return result\n\n class A(metaclass=OrderedClass):\n def one(self): pass\n def two(self): pass\n def three(self): pass\n def four(self): pass\n\n >>> A.members\n (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s "__prepare__()" method which returns an\nempty "collections.OrderedDict". That mapping records the methods and\nattributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s "__new__()" method gets\ninvoked. That method builds the new type and it saves the ordered\ndictionary keys in an attribute called "members".\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n"isinstance()" and "issubclass()" built-in functions.\n\nIn particular, the metaclass "abc.ABCMeta" implements these methods in\norder to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n "isinstance(instance, class)".\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n "issubclass(subclass, class)".\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also: **PEP 3119** - Introducing Abstract Base Classes\n\n Includes the specification for customizing "isinstance()" and\n "issubclass()" behavior through "__instancecheck__()" and\n "__subclasscheck__()", with motivation for this functionality in\n the context of adding Abstract Base Classes (see the "abc"\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, "x(arg1, arg2, ...)" is a shorthand for\n "x.__call__(arg1, arg2, ...)".\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which "0 <= k < N" where\n*N* is the length of the sequence, or slice objects, which define a\nrange of items. It is also recommended that mappings provide the\nmethods "keys()", "values()", "items()", "get()", "clear()",\n"setdefault()", "pop()", "popitem()", "copy()", and "update()"\nbehaving similar to those for Python\'s standard dictionary objects.\nThe "collections" module provides a "MutableMapping" abstract base\nclass to help create those methods from a base set of "__getitem__()",\n"__setitem__()", "__delitem__()", and "keys()". Mutable sequences\nshould provide methods "append()", "count()", "index()", "extend()",\n"insert()", "pop()", "remove()", "reverse()" and "sort()", like Python\nstandard list objects. Finally, sequence types should implement\naddition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods "__add__()", "__radd__()",\n"__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described\nbelow; they should not define other numerical operators. It is\nrecommended that both mappings and sequences implement the\n"__contains__()" method to allow efficient use of the "in" operator;\nfor mappings, "in" should search the mapping\'s keys; for sequences, it\nshould search through the values. It is further recommended that both\nmappings and sequences implement the "__iter__()" method to allow\nefficient iteration through the container; for mappings, "__iter__()"\nshould be the same as "keys()"; for sequences, it should iterate\nthrough the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function "len()". Should return\n the length of the object, an integer ">=" 0. Also, an object that\n doesn\'t define a "__bool__()" method and whose "__len__()" method\n returns zero is considered to be false in a Boolean context.\n\nobject.__length_hint__(self)\n\n Called to implement "operator.length_hint()". Should return an\n estimated length for the object (which may be greater or less than\n the actual length). The length must be an integer ">=" 0. This\n method is purely an optimization and is never required for\n correctness.\n\n New in version 3.4.\n\nNote: Slicing is done exclusively with the following three methods.\n A call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with "None".\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of "self[key]". For sequence types,\n the accepted keys should be integers and slice objects. Note that\n the special interpretation of negative indexes (if the class wishes\n to emulate a sequence type) is up to the "__getitem__()" method. If\n *key* is of an inappropriate type, "TypeError" may be raised; if of\n a value outside the set of indexes for the sequence (after any\n special interpretation of negative values), "IndexError" should be\n raised. For mapping types, if *key* is missing (not in the\n container), "KeyError" should be raised.\n\n Note: "for" loops expect that an "IndexError" will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the "__getitem__()" method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of "self[key]". Same note as for\n "__getitem__()". This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the "__getitem__()" method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method "keys()".\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the "reversed()" built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the "__reversed__()" method is not provided, the "reversed()"\n built-in will fall back to using the sequence protocol ("__len__()"\n and "__getitem__()"). Objects that support the sequence protocol\n should only provide "__reversed__()" if they can provide an\n implementation that is more efficient than the one provided by\n "reversed()".\n\nThe membership test operators ("in" and "not in") are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define "__contains__()", the membership test\n first tries iteration via "__iter__()", then the old sequence\n iteration protocol via "__getitem__()", see *this section in the\n language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|"). For instance, to evaluate the\n expression "x + y", where *x* is an instance of a class that has an\n "__add__()" method, "x.__add__(y)" is called. The "__divmod__()"\n method should be the equivalent to using "__floordiv__()" and\n "__mod__()"; it should not be related to "__truediv__()". Note\n that "__pow__()" should be defined to accept an optional third\n argument if the ternary version of the built-in "pow()" function is\n to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return "NotImplemented".\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations ("+", "-", "*", "/", "//", "%", "divmod()", "pow()",\n "**", "<<", ">>", "&", "^", "|") with reflected (swapped) operands.\n These functions are only called if the left operand does not\n support the corresponding operation and the operands are of\n different types. [2] For instance, to evaluate the expression "x -\n y", where *y* is an instance of a class that has an "__rsub__()"\n method, "y.__rsub__(x)" is called if "x.__sub__(y)" returns\n *NotImplemented*.\n\n Note that ternary "pow()" will not try calling "__rpow__()" (the\n coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left\n operand\'s type and that subclass provides the reflected method\n for the operation, this method will be called before the left\n operand\'s non-reflected method. This behavior allows subclasses\n to override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments ("+=", "-=", "*=", "/=", "//=", "%=", "**=", "<<=",\n ">>=", "&=", "^=", "|="). These methods should attempt to do the\n operation in-place (modifying *self*) and return the result (which\n could be, but does not have to be, *self*). If a specific method\n is not defined, the augmented assignment falls back to the normal\n methods. For instance, if *x* is an instance of a class with an\n "__iadd__()" method, "x += y" is equivalent to "x = x.__iadd__(y)"\n . Otherwise, "x.__add__(y)" and "y.__radd__(x)" are considered, as\n with the evaluation of "x + y". In certain situations, augmented\n assignment can result in unexpected errors (see *Why does\n a_tuple[i] += [\'item\'] raise an exception when the addition\n works?*), but this behavior is in fact part of the data model.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations ("-", "+",\n "abs()" and "~").\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions "complex()", "int()",\n "float()" and "round()". Should return a value of the appropriate\n type.\n\nobject.__index__(self)\n\n Called to implement "operator.index()", and whenever Python needs\n to losslessly convert the numeric object to an integer object (such\n as in slicing, or in the built-in "bin()", "hex()" and "oct()"\n functions). Presence of this method indicates that the numeric\n object is an integer type. Must return an integer.\n\n Note: When "__index__()" is defined, "__int__()" should also be\n defined, and both shuld return the same value, in order to have a\n coherent integer type class.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a "with" statement. The context manager\nhandles the entry into, and the exit from, the desired runtime context\nfor the execution of the block of code. Context managers are normally\ninvoked using the "with" statement (described in section *The with\nstatement*), but can also be used by directly invoking their methods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The "with"\n statement will bind this method\'s return value to the target(s)\n specified in the "as" clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be "None".\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that "__exit__()" methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also: **PEP 0343** - The "with" statement\n\n The specification, background, and examples for the Python "with"\n statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception:\n\n >>> class C:\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as "__hash__()" and "__repr__()" that are implemented by\nall objects, including type objects. If the implicit lookup of these\nmethods used the conventional lookup process, they would fail when\ninvoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe "__getattribute__()" method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print("Metaclass getattribute invoked")\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object, metaclass=Meta):\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print("Class getattribute invoked")\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the "__getattribute__()" machinery in this fashion provides\nsignificant scope for speed optimisations within the interpreter, at\nthe cost of some flexibility in the handling of special methods (the\nspecial method *must* be set on the class object itself in order to be\nconsistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type,\n under certain controlled conditions. It generally isn\'t a good\n idea though, since it can lead to some very strange behaviour if\n it is handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as "__add__()") fails the operation is not\n supported, which is why the reflected method is not called.\n',
+ 'string-methods': '\nString Methods\n**************\n\nStrings implement all of the *common* sequence operations, along with\nthe additional methods described below.\n\nStrings also support two styles of string formatting, one providing a\nlarge degree of flexibility and customization (see "str.format()",\n*Format String Syntax* and *String Formatting*) and the other based on\nC "printf" style formatting that handles a narrower range of types and\nis slightly harder to use correctly, but is often faster for the cases\nit can handle (*printf-style String Formatting*).\n\nThe *Text Processing Services* section of the standard library covers\na number of other modules that provide various text related utilities\n(including regular expression support in the "re" module).\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.casefold()\n\n Return a casefolded copy of the string. Casefolded strings may be\n used for caseless matching.\n\n Casefolding is similar to lowercasing but more aggressive because\n it is intended to remove all case distinctions in a string. For\n example, the German lowercase letter "\'\xc3\x9f\'" is equivalent to ""ss"".\n Since it is already lowercase, "lower()" would do nothing to "\'\xc3\x9f\'";\n "casefold()" converts it to ""ss"".\n\n The casefolding algorithm is described in section 3.13 of the\n Unicode Standard.\n\n New in version 3.3.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode(encoding="utf-8", errors="strict")\n\n Return an encoded version of the string as a bytes object. Default\n encoding is "\'utf-8\'". *errors* may be given to set a different\n error handling scheme. The default for *errors* is "\'strict\'",\n meaning that encoding errors raise a "UnicodeError". Other possible\n values are "\'ignore\'", "\'replace\'", "\'xmlcharrefreplace\'",\n "\'backslashreplace\'" and any other name registered via\n "codecs.register_error()", see section *Codec Base Classes*. For a\n list of possible encodings, see section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return "True" if the string ends with the specified *suffix*,\n otherwise return "False". *suffix* can also be a tuple of suffixes\n to look for. With optional *start*, test beginning at that\n position. With optional *end*, stop comparing at that position.\n\nstr.expandtabs(tabsize=8)\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab ("\\t"), one or more space characters are inserted in the result\n until the current column is equal to the next tab position. (The\n tab character itself is not copied.) If the character is a newline\n ("\\n") or return ("\\r"), it is copied and the current column is\n reset to zero. Any other character is copied unchanged and the\n current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" if *sub* is not found.\n\n Note: The "find()" method should be used only if you need to know\n the position of *sub*. To check if *sub* is a substring or not,\n use the "in" operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces "{}". Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.format_map(mapping)\n\n Similar to "str.format(**mapping)", except that "mapping" is used\n directly and not copied to a "dict". This is useful if for example\n "mapping" is a dict subclass:\n\n >>> class Default(dict):\n ... def __missing__(self, key):\n ... return key\n ...\n >>> \'{name} was born in {country}\'.format_map(Default(name=\'Guido\'))\n \'Guido was born in country\'\n\n New in version 3.2.\n\nstr.index(sub[, start[, end]])\n\n Like "find()", but raise "ValueError" when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character "c"\n is alphanumeric if one of the following returns "True":\n "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()".\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that can be used to\n form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\n Use "keyword.iskeyword()" to test for reserved identifiers such as\n "def" and "class".\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when "repr()" is\n invoked on a string. It has no bearing on the handling of strings\n written to "sys.stdout" or "sys.stderr".)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A "TypeError" will be raised if there are\n any non-string values in *iterable*, including "bytes" objects.\n The separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n The lowercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n "str.translate()".\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within "s[start:end]".\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return "-1" on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like "rfind()" but raises "ValueError" when the substring *sub* is\n not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to "len(s)".\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n "None", any whitespace string is a separator. Except for splitting\n from the right, "rsplit()" behaves like "split()" which is\n described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or "None", the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split(sep=None, maxsplit=-1)\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most "maxsplit+1"\n elements). If *maxsplit* is not specified or "-1", then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n "\'1,,2\'.split(\',\')" returns "[\'1\', \'\', \'2\']"). The *sep* argument\n may consist of multiple characters (for example,\n "\'1<>2<>3\'.split(\'<>\')" returns "[\'1\', \'2\', \'3\']"). Splitting an\n empty string with a specified separator returns "[\'\']".\n\n If *sep* is not specified or is "None", a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a "None" separator returns "[]".\n\n For example, "\' 1 2 3 \'.split()" returns "[\'1\', \'2\', \'3\']", and\n "\' 1 2 3 \'.split(None, 1)" returns "[\'1\', \'2 3 \']".\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, "\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()" returns "[\'ab\n c\', \'\', \'de fg\', \'kl\']", while the same call with\n "splitlines(True)" returns "[\'ab c\\n\', \'\\n\', \'de fg\\r\', \'kl\\r\\n\']".\n\n Unlike "split()" when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return "True" if string starts with the *prefix*, otherwise return\n "False". *prefix* can also be a tuple of prefixes to look for.\n With optional *start*, test string beginning at that position.\n With optional *end*, stop comparing string at that position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or "None", the *chars*\n argument defaults to removing whitespace. The *chars* argument is\n not a prefix or suffix; rather, all combinations of its values are\n stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa. Note that it is not necessarily true that\n "s.swapcase().swapcase() == s".\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or "None". Unmapped\n characters are left untouched. Characters mapped to "None" are\n deleted.\n\n You can use "str.maketrans()" to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom\n character mapping codec using the "codecs" module (see\n "encodings.cp1251" for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that "str.upper().isupper()" might be\n "False" if "s" contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n The uppercasing algorithm used is described in section 3.13 of the\n Unicode Standard.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to "len(s)".\n',
'strings': '\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "R" | "U"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | stringescapeseq\n longstringitem ::= longstringchar | stringescapeseq\n shortstringchar ::= <any source character except "\\" or newline or the quote>\n longstringchar ::= <any source character except "\\">\n stringescapeseq ::= "\\" <any source character>\n\n bytesliteral ::= bytesprefix(shortbytes | longbytes)\n bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB"\n shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n shortbytesitem ::= shortbyteschar | bytesescapeseq\n longbytesitem ::= longbyteschar | bytesescapeseq\n shortbyteschar ::= <any ASCII character except "\\" or newline or the quote>\n longbyteschar ::= <any ASCII character except "\\">\n bytesescapeseq ::= "\\" <any ASCII character>\n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the "stringprefix" or "bytesprefix"\nand the rest of the literal. The source character set is defined by\nthe encoding declaration; it is UTF-8 if no encoding declaration is\ngiven in the source file; see section *Encoding declarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes ("\'") or double quotes ("""). They can also be enclosed\nin matching groups of three single or double quotes (these are\ngenerally referred to as *triple-quoted strings*). The backslash\n("\\") character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with "\'b\'" or "\'B\'"; they produce\nan instance of the "bytes" type instead of the "str" type. They may\nonly contain ASCII characters; bytes with a numeric value of 128 or\ngreater must be expressed with escapes.\n\nAs of Python 3.3 it is possible again to prefix unicode strings with a\n"u" prefix to simplify maintenance of dual 2.x and 3.x codebases.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter "\'r\'" or "\'R\'"; such strings are called *raw strings* and treat\nbackslashes as literal characters. As a result, in string literals,\n"\'\\U\'" and "\'\\u\'" escapes in raw strings are not treated specially.\nGiven that Python 2.x\'s raw unicode literals behave differently than\nPython 3.x\'s the "\'ur\'" syntax is not supported.\n\n New in version 3.3: The "\'rb\'" prefix of raw bytes literals has\n been added as a synonym of "\'br\'".\n\n New in version 3.3: Support for the unicode legacy literal\n ("u\'value\'") was reintroduced to simplify the maintenance of dual\n Python 2.x and 3.x codebases. See **PEP 414** for more information.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either "\'" or """.)\n\nUnless an "\'r\'" or "\'R\'" prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n+-------------------+-----------------------------------+---------+\n| "\\ooo" | Character with octal value *ooo* | (1,3) |\n+-------------------+-----------------------------------+---------+\n| "\\xhh" | Character with hex value *hh* | (2,3) |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| "\\N{name}" | Character named *name* in the | (4) |\n+-------------------+-----------------------------------+---------+\n| "\\uxxxx" | Character with 16-bit hex value | (5) |\n+-------------------+-----------------------------------+---------+\n| "\\Uxxxxxxxx" | Character with 32-bit hex value | (6) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the\n byte with the given value. In a string literal, these escapes\n denote a Unicode character with the given value.\n\n4. Changed in version 3.3: Support for name aliases [1] has been\n added.\n\n5. Individual code units which form parts of a surrogate pair can\n be encoded using this escape sequence. Exactly four hex digits are\n required.\n\n6. Any Unicode character can be encoded this way. Exactly eight\n hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw string, string quotes can be escaped with a backslash,\nbut the backslash remains in the string; for example, "r"\\""" is a\nvalid string literal consisting of two characters: a backslash and a\ndouble quote; "r"\\"" is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n',
'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription,\ne.g. a list or dictionary. User-defined objects can support\nsubscription by defining a "__getitem__()" method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a "__getitem__()"\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that "x[-1]" selects the last item of "x").\nThe resulting value must be a nonnegative integer less than the number\nof items in the sequence, and the subscription selects the item whose\nindex is that value (counting from zero). Since the support for\nnegative indices and slicing occurs in the object\'s "__getitem__()"\nmethod, subclasses overriding this method will need to explicitly add\nthat support.\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n',
'truth': '\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an "if" or\n"while" condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* "None"\n\n* "False"\n\n* zero of any numeric type, for example, "0", "0.0", "0j".\n\n* any empty sequence, for example, "\'\'", "()", "[]".\n\n* any empty mapping, for example, "{}".\n\n* instances of user-defined classes, if the class defines a\n "__bool__()" or "__len__()" method, when that method returns the\n integer zero or "bool" value "False". [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn "0" or "False" for false and "1" or "True" for true, unless\notherwise stated. (Important exception: the Boolean operations "or"\nand "and" always return one of their operands.)\n',
'try': '\nThe "try" statement\n*******************\n\nThe "try" statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression ["as" target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nThe "except" clause(s) specify one or more exception handlers. When no\nexception occurs in the "try" clause, no exception handler is\nexecuted. When an exception occurs in the "try" suite, a search for an\nexception handler is started. This search inspects the except clauses\nin turn until one is found that matches the exception. An expression-\nless except clause, if present, must be last; it matches any\nexception. For an except clause with an expression, that expression\nis evaluated, and the clause matches the exception if the resulting\nobject is "compatible" with the exception. An object is compatible\nwith an exception if it is the class or a base class of the exception\nobject or a tuple containing an item compatible with the exception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire "try" statement raised\nthe exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified after the "as" keyword in that except clause, if\npresent, and the except clause\'s suite is executed. All except\nclauses must have an executable block. When the end of this block is\nreached, execution continues normally after the entire try statement.\n(This means that if two nested handlers exist for the same exception,\nand the exception occurs in the try clause of the inner handler, the\nouter handler will not handle the exception.)\n\nWhen an exception has been assigned using "as target", it is cleared\nat the end of the except clause. This is as if\n\n except E as N:\n foo\n\nwas translated to\n\n except E as N:\n try:\n foo\n finally:\n del N\n\nThis means the exception must be assigned to a different name to be\nable to refer to it after the except clause. Exceptions are cleared\nbecause with the traceback attached to them, they form a reference\ncycle with the stack frame, keeping all locals in that frame alive\nuntil the next garbage collection occurs.\n\nBefore an except clause\'s suite is executed, details about the\nexception are stored in the "sys" module and can be access via\n"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of the\nexception class, the exception instance and a traceback object (see\nsection *The standard type hierarchy*) identifying the point in the\nprogram where the exception occurred. "sys.exc_info()" values are\nrestored to their previous values (before the call) when returning\nfrom a function that handled an exception.\n\nThe optional "else" clause is executed if and when control flows off\nthe end of the "try" clause. [2] Exceptions in the "else" clause are\nnot handled by the preceding "except" clauses.\n\nIf "finally" is present, it specifies a \'cleanup\' handler. The "try"\nclause is executed, including any "except" and "else" clauses. If an\nexception occurs in any of the clauses and is not handled, the\nexception is temporarily saved. The "finally" clause is executed. If\nthere is a saved exception it is re-raised at the end of the "finally"\nclause. If the "finally" clause raises another exception, the saved\nexception is set as the context of the new exception. If the "finally"\nclause executes a "return" or "break" statement, the saved exception\nis discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the "finally" clause.\n\nWhen a "return", "break" or "continue" statement is executed in the\n"try" suite of a "try"..."finally" statement, the "finally" clause is\nalso executed \'on the way out.\' A "continue" statement is illegal in\nthe "finally" clause. (The reason is a problem with the current\nimplementation --- this restriction may be lifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the "raise" statement to\ngenerate exceptions may be found in section *The raise statement*.\n',
- 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name "None". It\n is used to signify the absence of a value in many situations, e.g.,\n it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n "NotImplemented". Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the literal "..." or the\n built-in name "Ellipsis". Its truth value is true.\n\n"numbers.Number"\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n "numbers.Integral"\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are two types of integers:\n\n Integers ("int")\n\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans ("bool")\n These represent the truth values False and True. The two\n objects representing the values "False" and "True" are the\n only Boolean objects. The Boolean type is a subtype of the\n integer type, and Boolean values behave like the values 0 and\n 1, respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ""False"" or\n ""True"" are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers.\n\n "numbers.Real" ("float")\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n "numbers.Complex" ("complex")\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number "z" can be retrieved through the read-only\n attributes "z.real" and "z.imag".\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function "len()" returns the number of items\n of a sequence. When the length of a sequence is *n*, the index set\n contains the numbers 0, 1, ..., *n*-1. Item *i* of sequence *a* is\n selected by "a[i]".\n\n Sequences also support slicing: "a[i:j]" selects all items with\n index *k* such that *i* "<=" *k* "<" *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: "a[i:j:k]" selects all items of *a* with index *x* where\n "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n A string is a sequence of values that represent Unicode\n codepoints. All the codepoints in range "U+0000 - U+10FFFF"\n can be represented in a string. Python doesn\'t have a "chr"\n type, and every character in the string is represented as a\n string object with length "1". The built-in function "ord()"\n converts a character to its codepoint (as an integer);\n "chr()" converts an integer in range "0 - 10FFFF" to the\n corresponding character. "str.encode()" can be used to\n convert a "str" to "bytes" using the given encoding, and\n "bytes.decode()" can be used to achieve the opposite.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Bytes\n A bytes object is an immutable array. The items are 8-bit\n bytes, represented by integers in the range 0 <= x < 256.\n Bytes literals (like "b\'abc\'") and the built-in function\n "bytes()" can be used to construct bytes objects. Also,\n bytes objects can be decoded to strings via the "decode()"\n method.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and "del" (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in "bytearray()" constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module "array" provides an additional example of a\n mutable sequence type, as does the "collections" module.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function "len()"\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., "1" and\n "1.0"), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n "set()" constructor and can be modified afterwards by several\n methods, such as "add()".\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in "frozenset()" constructor. As a frozenset is immutable\n and *hashable*, it can be used again as an element of another\n set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation "a[k]" selects the item indexed by "k"\n from the mapping "a"; this can be used in expressions and as the\n target of assignments or "del" statements. The built-in function\n "len()" returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., "1" and "1.0")\n then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the "{...}"\n notation (see section *Dictionary displays*).\n\n The extension modules "dbm.ndbm" and "dbm.gnu" provide\n additional examples of mapping types, as does the "collections"\n module.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +---------------------------+---------------------------------+-------------+\n +===========================+=================================+=============+\n | "__doc__" | The function\'s documentation | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__name__" | The function\'s name | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__qualname__" | The function\'s *qualified name* | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__module__" | The name of the module the | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__defaults__" | A tuple containing default | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__code__" | The code object representing | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__globals__" | A reference to the dictionary | Read-only |\n +---------------------------+---------------------------------+-------------+\n | "__dict__" | The namespace supporting | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__closure__" | "None" or a tuple of cells that | Read-only |\n +---------------------------+---------------------------------+-------------+\n | "__annotations__" | A dict containing annotations | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__kwdefaults__" | A dict containing defaults for | Writable |\n +---------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n Instance methods\n An instance method object combines a class, a class instance and\n any callable object (normally a user-defined function).\n\n Special read-only attributes: "__self__" is the class instance\n object, "__func__" is the function object; "__doc__" is the\n method\'s documentation (same as "__func__.__doc__"); "__name__"\n is the method name (same as "__func__.__name__"); "__module__"\n is the name of the module the method was defined in, or "None"\n if unavailable.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object or a class\n method object.\n\n When an instance method object is created by retrieving a user-\n defined function object from a class via one of its instances,\n its "__self__" attribute is the instance, and the method object\n is said to be bound. The new method\'s "__func__" attribute is\n the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the "__func__"\n attribute of the new instance is not the original method object\n but its "__func__" attribute.\n\n When an instance method object is created by retrieving a class\n method object from a class or instance, its "__self__" attribute\n is the class itself, and its "__func__" attribute is the\n function object underlying the class method.\n\n When an instance method object is called, the underlying\n function ("__func__") is called, inserting the class instance\n ("__self__") in front of the argument list. For instance, when\n "C" is a class which contains a definition for a function "f()",\n and "x" is an instance of "C", calling "x.f(1)" is equivalent to\n calling "C.f(x, 1)".\n\n When an instance method object is derived from a class method\n object, the "class instance" stored in "__self__" will actually\n be the class itself, so that calling either "x.f(1)" or "C.f(1)"\n is equivalent to calling "f(C,1)" where "f" is the underlying\n function.\n\n Note that the transformation from function object to instance\n method object happens each time the attribute is retrieved from\n the instance. In some cases, a fruitful optimization is to\n assign the attribute to a local variable and call that local\n variable. Also notice that this transformation only happens for\n user-defined functions; other callable objects (and all non-\n callable objects) are retrieved without transformation. It is\n also important to note that user-defined functions which are\n attributes of a class instance are not converted to bound\n methods; this *only* happens when the function is an attribute\n of the class.\n\n Generator functions\n A function or method which uses the "yield" statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s "iterator.__next__()" method will cause the\n function to execute until it provides a value using the "yield"\n statement. When the function executes a "return" statement or\n falls off the end, a "StopIteration" exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are "len()" and "math.sin()"\n ("math" is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: "__doc__" is the function\'s documentation\n string, or "None" if unavailable; "__name__" is the function\'s\n name; "__self__" is set to "None" (but see the next item);\n "__module__" is the name of the module the function was defined\n in or "None" if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n "alist.append()", assuming *alist* is a list object. In this\n case, the special read-only attribute "__self__" is set to the\n object denoted by *alist*.\n\n Classes\n Classes are callable. These objects normally act as factories\n for new instances of themselves, but variations are possible for\n class types that override "__new__()". The arguments of the\n call are passed to "__new__()" and, in the typical case, to\n "__init__()" to initialize the new instance.\n\n Class Instances\n Instances of arbitrary classes can be made callable by defining\n a "__call__()" method in their class.\n\nModules\n Modules are a basic organizational unit of Python code, and are\n created by the *import system* as invoked either by the "import"\n statement (see "import"), or by calling functions such as\n "importlib.import_module()" and built-in "__import__()". A module\n object has a namespace implemented by a dictionary object (this is\n the dictionary referenced by the "__globals__" attribute of\n functions defined in the module). Attribute references are\n translated to lookups in this dictionary, e.g., "m.x" is equivalent\n to "m.__dict__["x"]". A module object does not contain the code\n object used to initialize the module (since it isn\'t needed once\n the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n\n Special read-only attribute: "__dict__" is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: "__name__" is the module\'s name;\n "__doc__" is the module\'s documentation string, or "None" if\n unavailable; "__file__" is the pathname of the file from which the\n module was loaded, if it was loaded from a file. The "__file__"\n attribute may be missing for certain types of modules, such as C\n modules that are statically linked into the interpreter; for\n extension modules loaded dynamically from a shared library, it is\n the pathname of the shared library file.\n\nCustom classes\n Custom class types are typically created by class definitions (see\n section *Class definitions*). A class has a namespace implemented\n by a dictionary object. Class attribute references are translated\n to lookups in this dictionary, e.g., "C.x" is translated to\n "C.__dict__["x"]" (although there are a number of hooks which allow\n for other means of locating attributes). When the attribute name is\n not found there, the attribute search continues in the base\n classes. This search of the base classes uses the C3 method\n resolution order which behaves correctly even in the presence of\n \'diamond\' inheritance structures where there are multiple\n inheritance paths leading back to a common ancestor. Additional\n details on the C3 MRO used by Python can be found in the\n documentation accompanying the 2.3 release at\n http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class "C", say) would yield a\n class method object, it is transformed into an instance method\n object whose "__self__" attributes is "C". When it would yield a\n static method object, it is transformed into the object wrapped by\n the static method object. See section *Implementing Descriptors*\n for another way in which attributes retrieved from a class may\n differ from those actually contained in its "__dict__".\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: "__name__" is the class name; "__module__" is\n the module name in which the class was defined; "__dict__" is the\n dictionary containing the class\'s namespace; "__bases__" is a tuple\n (possibly empty or a singleton) containing the base classes, in the\n order of their occurrence in the base class list; "__doc__" is the\n class\'s documentation string, or None if undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object, it is transformed into an instance method object\n whose "__self__" attribute is the instance. Static method and\n class method objects are also transformed; see above under\n "Classes". See section *Implementing Descriptors* for another way\n in which attributes of a class retrieved via its instances may\n differ from the objects actually stored in the class\'s "__dict__".\n If no class attribute is found, and the object\'s class has a\n "__getattr__()" method, that is called to satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n "__setattr__()" or "__delattr__()" method, this is called instead\n of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: "__dict__" is the attribute dictionary;\n "__class__" is the instance\'s class.\n\nI/O objects (also known as file objects)\n A *file object* represents an open file. Various shortcuts are\n available to create file objects: the "open()" built-in function,\n and also "os.popen()", "os.fdopen()", and the "makefile()" method\n of socket objects (and perhaps by other functions or methods\n provided by extension modules).\n\n The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n initialized to file objects corresponding to the interpreter\'s\n standard input, output and error streams; they are all open in text\n mode and therefore follow the interface defined by the\n "io.TextIOBase" abstract class.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: "co_name" gives the function name;\n "co_argcount" is the number of positional arguments (including\n arguments with default values); "co_nlocals" is the number of\n local variables used by the function (including arguments);\n "co_varnames" is a tuple containing the names of the local\n variables (starting with the argument names); "co_cellvars" is a\n tuple containing the names of local variables that are\n referenced by nested functions; "co_freevars" is a tuple\n containing the names of free variables; "co_code" is a string\n representing the sequence of bytecode instructions; "co_consts"\n is a tuple containing the literals used by the bytecode;\n "co_names" is a tuple containing the names used by the bytecode;\n "co_filename" is the filename from which the code was compiled;\n "co_firstlineno" is the first line number of the function;\n "co_lnotab" is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); "co_stacksize" is the required stack size\n (including local variables); "co_flags" is an integer encoding a\n number of flags for the interpreter.\n\n The following flag bits are defined for "co_flags": bit "0x04"\n is set if the function uses the "*arguments" syntax to accept an\n arbitrary number of positional arguments; bit "0x08" is set if\n the function uses the "**keywords" syntax to accept arbitrary\n keyword arguments; bit "0x20" is set if the function is a\n generator.\n\n Future feature declarations ("from __future__ import division")\n also use bits in "co_flags" to indicate whether a code object\n was compiled with a particular feature enabled: bit "0x2000" is\n set if the function was compiled with future division enabled;\n bits "0x10" and "0x1000" were used in earlier versions of\n Python.\n\n Other bits in "co_flags" are reserved for internal use.\n\n If a code object represents a function, the first item in\n "co_consts" is the documentation string of the function, or\n "None" if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: "f_back" is to the previous stack\n frame (towards the caller), or "None" if this is the bottom\n stack frame; "f_code" is the code object being executed in this\n frame; "f_locals" is the dictionary used to look up local\n variables; "f_globals" is used for global variables;\n "f_builtins" is used for built-in (intrinsic) names; "f_lasti"\n gives the precise instruction (this is an index into the\n bytecode string of the code object).\n\n Special writable attributes: "f_trace", if not "None", is a\n function called at the start of each source code line (this is\n used by the debugger); "f_lineno" is the current line number of\n the frame --- writing to this from within a trace function jumps\n to the given line (only for the bottom-most frame). A debugger\n can implement a Jump command (aka Set Next Statement) by writing\n to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as the third item of the\n tuple returned by "sys.exc_info()". When the program contains no\n suitable handler, the stack trace is written (nicely formatted)\n to the standard error stream; if the interpreter is interactive,\n it is also made available to the user as "sys.last_traceback".\n\n Special read-only attributes: "tb_next" is the next level in the\n stack trace (towards the frame where the exception occurred), or\n "None" if there is no next level; "tb_frame" points to the\n execution frame of the current level; "tb_lineno" gives the line\n number where the exception occurred; "tb_lasti" indicates the\n precise instruction. The line number and last instruction in\n the traceback may differ from the line number of its frame\n object if the exception occurred in a "try" statement with no\n matching except clause or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices for "__getitem__()"\n methods. They are also created by the built-in "slice()"\n function.\n\n Special read-only attributes: "start" is the lower bound; "stop"\n is the upper bound; "step" is the step value; each is "None" if\n omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the slice that the slice object\n would describe if applied to a sequence of *length* items.\n It returns a tuple of three integers; respectively these are\n the *start* and *stop* indices and the *step* or stride\n length of the slice. Missing or out-of-bounds indices are\n handled in a manner consistent with regular slices.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n "staticmethod()" constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in "classmethod()" constructor.\n',
+ 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.), although such additions\nwill often be provided via the standard library instead.\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name "None". It\n is used to signify the absence of a value in many situations, e.g.,\n it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n "NotImplemented". Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the literal "..." or the\n built-in name "Ellipsis". Its truth value is true.\n\n"numbers.Number"\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n "numbers.Integral"\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are two types of integers:\n\n Integers ("int")\n\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans ("bool")\n These represent the truth values False and True. The two\n objects representing the values "False" and "True" are the\n only Boolean objects. The Boolean type is a subtype of the\n integer type, and Boolean values behave like the values 0 and\n 1, respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ""False"" or\n ""True"" are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers.\n\n "numbers.Real" ("float")\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n "numbers.Complex" ("complex")\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number "z" can be retrieved through the read-only\n attributes "z.real" and "z.imag".\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function "len()" returns the number of items\n of a sequence. When the length of a sequence is *n*, the index set\n contains the numbers 0, 1, ..., *n*-1. Item *i* of sequence *a* is\n selected by "a[i]".\n\n Sequences also support slicing: "a[i:j]" selects all items with\n index *k* such that *i* "<=" *k* "<" *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: "a[i:j:k]" selects all items of *a* with index *x* where\n "x = i + n*k", *n* ">=" "0" and *i* "<=" *x* "<" *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n A string is a sequence of values that represent Unicode\n codepoints. All the codepoints in range "U+0000 - U+10FFFF"\n can be represented in a string. Python doesn\'t have a "chr"\n type, and every character in the string is represented as a\n string object with length "1". The built-in function "ord()"\n converts a character to its codepoint (as an integer);\n "chr()" converts an integer in range "0 - 10FFFF" to the\n corresponding character. "str.encode()" can be used to\n convert a "str" to "bytes" using the given encoding, and\n "bytes.decode()" can be used to achieve the opposite.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Bytes\n A bytes object is an immutable array. The items are 8-bit\n bytes, represented by integers in the range 0 <= x < 256.\n Bytes literals (like "b\'abc\'") and the built-in function\n "bytes()" can be used to construct bytes objects. Also,\n bytes objects can be decoded to strings via the "decode()"\n method.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and "del" (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in "bytearray()" constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module "array" provides an additional example of a\n mutable sequence type, as does the "collections" module.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function "len()"\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., "1" and\n "1.0"), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n "set()" constructor and can be modified afterwards by several\n methods, such as "add()".\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in "frozenset()" constructor. As a frozenset is immutable\n and *hashable*, it can be used again as an element of another\n set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation "a[k]" selects the item indexed by "k"\n from the mapping "a"; this can be used in expressions and as the\n target of assignments or "del" statements. The built-in function\n "len()" returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., "1" and "1.0")\n then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the "{...}"\n notation (see section *Dictionary displays*).\n\n The extension modules "dbm.ndbm" and "dbm.gnu" provide\n additional examples of mapping types, as does the "collections"\n module.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +---------------------------+---------------------------------+-------------+\n +===========================+=================================+=============+\n | "__doc__" | The function\'s documentation | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__name__" | The function\'s name | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__qualname__" | The function\'s *qualified name* | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__module__" | The name of the module the | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__defaults__" | A tuple containing default | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__code__" | The code object representing | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__globals__" | A reference to the dictionary | Read-only |\n +---------------------------+---------------------------------+-------------+\n | "__dict__" | The namespace supporting | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__closure__" | "None" or a tuple of cells that | Read-only |\n +---------------------------+---------------------------------+-------------+\n | "__annotations__" | A dict containing annotations | Writable |\n +---------------------------+---------------------------------+-------------+\n | "__kwdefaults__" | A dict containing defaults for | Writable |\n +---------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n Instance methods\n An instance method object combines a class, a class instance and\n any callable object (normally a user-defined function).\n\n Special read-only attributes: "__self__" is the class instance\n object, "__func__" is the function object; "__doc__" is the\n method\'s documentation (same as "__func__.__doc__"); "__name__"\n is the method name (same as "__func__.__name__"); "__module__"\n is the name of the module the method was defined in, or "None"\n if unavailable.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object or a class\n method object.\n\n When an instance method object is created by retrieving a user-\n defined function object from a class via one of its instances,\n its "__self__" attribute is the instance, and the method object\n is said to be bound. The new method\'s "__func__" attribute is\n the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the "__func__"\n attribute of the new instance is not the original method object\n but its "__func__" attribute.\n\n When an instance method object is created by retrieving a class\n method object from a class or instance, its "__self__" attribute\n is the class itself, and its "__func__" attribute is the\n function object underlying the class method.\n\n When an instance method object is called, the underlying\n function ("__func__") is called, inserting the class instance\n ("__self__") in front of the argument list. For instance, when\n "C" is a class which contains a definition for a function "f()",\n and "x" is an instance of "C", calling "x.f(1)" is equivalent to\n calling "C.f(x, 1)".\n\n When an instance method object is derived from a class method\n object, the "class instance" stored in "__self__" will actually\n be the class itself, so that calling either "x.f(1)" or "C.f(1)"\n is equivalent to calling "f(C,1)" where "f" is the underlying\n function.\n\n Note that the transformation from function object to instance\n method object happens each time the attribute is retrieved from\n the instance. In some cases, a fruitful optimization is to\n assign the attribute to a local variable and call that local\n variable. Also notice that this transformation only happens for\n user-defined functions; other callable objects (and all non-\n callable objects) are retrieved without transformation. It is\n also important to note that user-defined functions which are\n attributes of a class instance are not converted to bound\n methods; this *only* happens when the function is an attribute\n of the class.\n\n Generator functions\n A function or method which uses the "yield" statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s "iterator.__next__()" method will cause the\n function to execute until it provides a value using the "yield"\n statement. When the function executes a "return" statement or\n falls off the end, a "StopIteration" exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are "len()" and "math.sin()"\n ("math" is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: "__doc__" is the function\'s documentation\n string, or "None" if unavailable; "__name__" is the function\'s\n name; "__self__" is set to "None" (but see the next item);\n "__module__" is the name of the module the function was defined\n in or "None" if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n "alist.append()", assuming *alist* is a list object. In this\n case, the special read-only attribute "__self__" is set to the\n object denoted by *alist*.\n\n Classes\n Classes are callable. These objects normally act as factories\n for new instances of themselves, but variations are possible for\n class types that override "__new__()". The arguments of the\n call are passed to "__new__()" and, in the typical case, to\n "__init__()" to initialize the new instance.\n\n Class Instances\n Instances of arbitrary classes can be made callable by defining\n a "__call__()" method in their class.\n\nModules\n Modules are a basic organizational unit of Python code, and are\n created by the *import system* as invoked either by the "import"\n statement (see "import"), or by calling functions such as\n "importlib.import_module()" and built-in "__import__()". A module\n object has a namespace implemented by a dictionary object (this is\n the dictionary referenced by the "__globals__" attribute of\n functions defined in the module). Attribute references are\n translated to lookups in this dictionary, e.g., "m.x" is equivalent\n to "m.__dict__["x"]". A module object does not contain the code\n object used to initialize the module (since it isn\'t needed once\n the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., "m.x = 1" is equivalent to "m.__dict__["x"] = 1".\n\n Special read-only attribute: "__dict__" is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: "__name__" is the module\'s name;\n "__doc__" is the module\'s documentation string, or "None" if\n unavailable; "__file__" is the pathname of the file from which the\n module was loaded, if it was loaded from a file. The "__file__"\n attribute may be missing for certain types of modules, such as C\n modules that are statically linked into the interpreter; for\n extension modules loaded dynamically from a shared library, it is\n the pathname of the shared library file.\n\nCustom classes\n Custom class types are typically created by class definitions (see\n section *Class definitions*). A class has a namespace implemented\n by a dictionary object. Class attribute references are translated\n to lookups in this dictionary, e.g., "C.x" is translated to\n "C.__dict__["x"]" (although there are a number of hooks which allow\n for other means of locating attributes). When the attribute name is\n not found there, the attribute search continues in the base\n classes. This search of the base classes uses the C3 method\n resolution order which behaves correctly even in the presence of\n \'diamond\' inheritance structures where there are multiple\n inheritance paths leading back to a common ancestor. Additional\n details on the C3 MRO used by Python can be found in the\n documentation accompanying the 2.3 release at\n http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class "C", say) would yield a\n class method object, it is transformed into an instance method\n object whose "__self__" attributes is "C". When it would yield a\n static method object, it is transformed into the object wrapped by\n the static method object. See section *Implementing Descriptors*\n for another way in which attributes retrieved from a class may\n differ from those actually contained in its "__dict__".\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: "__name__" is the class name; "__module__" is\n the module name in which the class was defined; "__dict__" is the\n dictionary containing the class\'s namespace; "__bases__" is a tuple\n (possibly empty or a singleton) containing the base classes, in the\n order of their occurrence in the base class list; "__doc__" is the\n class\'s documentation string, or None if undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object, it is transformed into an instance method object\n whose "__self__" attribute is the instance. Static method and\n class method objects are also transformed; see above under\n "Classes". See section *Implementing Descriptors* for another way\n in which attributes of a class retrieved via its instances may\n differ from the objects actually stored in the class\'s "__dict__".\n If no class attribute is found, and the object\'s class has a\n "__getattr__()" method, that is called to satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n "__setattr__()" or "__delattr__()" method, this is called instead\n of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: "__dict__" is the attribute dictionary;\n "__class__" is the instance\'s class.\n\nI/O objects (also known as file objects)\n A *file object* represents an open file. Various shortcuts are\n available to create file objects: the "open()" built-in function,\n and also "os.popen()", "os.fdopen()", and the "makefile()" method\n of socket objects (and perhaps by other functions or methods\n provided by extension modules).\n\n The objects "sys.stdin", "sys.stdout" and "sys.stderr" are\n initialized to file objects corresponding to the interpreter\'s\n standard input, output and error streams; they are all open in text\n mode and therefore follow the interface defined by the\n "io.TextIOBase" abstract class.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: "co_name" gives the function name;\n "co_argcount" is the number of positional arguments (including\n arguments with default values); "co_nlocals" is the number of\n local variables used by the function (including arguments);\n "co_varnames" is a tuple containing the names of the local\n variables (starting with the argument names); "co_cellvars" is a\n tuple containing the names of local variables that are\n referenced by nested functions; "co_freevars" is a tuple\n containing the names of free variables; "co_code" is a string\n representing the sequence of bytecode instructions; "co_consts"\n is a tuple containing the literals used by the bytecode;\n "co_names" is a tuple containing the names used by the bytecode;\n "co_filename" is the filename from which the code was compiled;\n "co_firstlineno" is the first line number of the function;\n "co_lnotab" is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); "co_stacksize" is the required stack size\n (including local variables); "co_flags" is an integer encoding a\n number of flags for the interpreter.\n\n The following flag bits are defined for "co_flags": bit "0x04"\n is set if the function uses the "*arguments" syntax to accept an\n arbitrary number of positional arguments; bit "0x08" is set if\n the function uses the "**keywords" syntax to accept arbitrary\n keyword arguments; bit "0x20" is set if the function is a\n generator.\n\n Future feature declarations ("from __future__ import division")\n also use bits in "co_flags" to indicate whether a code object\n was compiled with a particular feature enabled: bit "0x2000" is\n set if the function was compiled with future division enabled;\n bits "0x10" and "0x1000" were used in earlier versions of\n Python.\n\n Other bits in "co_flags" are reserved for internal use.\n\n If a code object represents a function, the first item in\n "co_consts" is the documentation string of the function, or\n "None" if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: "f_back" is to the previous stack\n frame (towards the caller), or "None" if this is the bottom\n stack frame; "f_code" is the code object being executed in this\n frame; "f_locals" is the dictionary used to look up local\n variables; "f_globals" is used for global variables;\n "f_builtins" is used for built-in (intrinsic) names; "f_lasti"\n gives the precise instruction (this is an index into the\n bytecode string of the code object).\n\n Special writable attributes: "f_trace", if not "None", is a\n function called at the start of each source code line (this is\n used by the debugger); "f_lineno" is the current line number of\n the frame --- writing to this from within a trace function jumps\n to the given line (only for the bottom-most frame). A debugger\n can implement a Jump command (aka Set Next Statement) by writing\n to f_lineno.\n\n Frame objects support one method:\n\n frame.clear()\n\n This method clears all references to local variables held by\n the frame. Also, if the frame belonged to a generator, the\n generator is finalized. This helps break reference cycles\n involving frame objects (for example when catching an\n exception and storing its traceback for later use).\n\n "RuntimeError" is raised if the frame is currently executing.\n\n New in version 3.4.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as the third item of the\n tuple returned by "sys.exc_info()". When the program contains no\n suitable handler, the stack trace is written (nicely formatted)\n to the standard error stream; if the interpreter is interactive,\n it is also made available to the user as "sys.last_traceback".\n\n Special read-only attributes: "tb_next" is the next level in the\n stack trace (towards the frame where the exception occurred), or\n "None" if there is no next level; "tb_frame" points to the\n execution frame of the current level; "tb_lineno" gives the line\n number where the exception occurred; "tb_lasti" indicates the\n precise instruction. The line number and last instruction in\n the traceback may differ from the line number of its frame\n object if the exception occurred in a "try" statement with no\n matching except clause or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices for "__getitem__()"\n methods. They are also created by the built-in "slice()"\n function.\n\n Special read-only attributes: "start" is the lower bound; "stop"\n is the upper bound; "step" is the step value; each is "None" if\n omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the slice that the slice object\n would describe if applied to a sequence of *length* items.\n It returns a tuple of three integers; respectively these are\n the *start* and *stop* indices and the *step* or stride\n length of the slice. Missing or out-of-bounds indices are\n handled in a manner consistent with regular slices.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n "staticmethod()" constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in "classmethod()" constructor.\n',
'typesfunctions': '\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: "func(argument-list)".\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n',
'typesmapping': '\nMapping Types --- "dict"\n************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built-\nin "list", "set", and "tuple" classes, and the "collections" module.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as "1" and "1.0") then they can be used interchangeably to index\nthe same dictionary entry. (Note however, that since computers store\nfloating-point numbers as approximations it is usually unwise to use\nthem as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of "key:\nvalue" pairs within braces, for example: "{\'jack\': 4098, \'sjoerd\':\n4127}" or "{4098: \'jack\', 4127: \'sjoerd\'}", or by the "dict"\nconstructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n Return a new dictionary initialized from an optional positional\n argument and a possibly empty set of keyword arguments.\n\n If no positional argument is given, an empty dictionary is created.\n If a positional argument is given and it is a mapping object, a\n dictionary is created with the same key-value pairs as the mapping\n object. Otherwise, the positional argument must be an *iterator*\n object. Each item in the iterable must itself be an iterator with\n exactly two objects. The first object of each item becomes a key\n in the new dictionary, and the second object the corresponding\n value. If a key occurs more than once, the last value for that key\n becomes the corresponding value in the new dictionary.\n\n If keyword arguments are given, the keyword arguments and their\n values are added to the dictionary created from the positional\n argument. If a key being added is already present, the value from\n the keyword argument replaces the value from the positional\n argument.\n\n To illustrate, the following examples all return a dictionary equal\n to "{"one": 1, "two": 2, "three": 3}":\n\n >>> a = dict(one=1, two=2, three=3)\n >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n >>> a == b == c == d == e\n True\n\n Providing keyword arguments as in the first example only works for\n keys that are valid Python identifiers. Otherwise, any valid keys\n can be used.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a "KeyError" if\n *key* is not in the map.\n\n If a subclass of dict defines a method "__missing__()", if the\n key *key* is not present, the "d[key]" operation calls that\n method with the key *key* as argument. The "d[key]" operation\n then returns or raises whatever is returned or raised by the\n "__missing__(key)" call if the key is not present. No other\n operations or methods invoke "__missing__()". If "__missing__()"\n is not defined, "KeyError" is raised. "__missing__()" must be a\n method; it cannot be an instance variable:\n\n >>> class Counter(dict):\n ... def __missing__(self, key):\n ... return 0\n >>> c = Counter()\n >>> c[\'red\']\n 0\n >>> c[\'red\'] += 1\n >>> c[\'red\']\n 1\n\n See "collections.Counter" for a complete implementation\n including other methods helpful for accumulating and managing\n tallies.\n\n d[key] = value\n\n Set "d[key]" to *value*.\n\n del d[key]\n\n Remove "d[key]" from *d*. Raises a "KeyError" if *key* is not\n in the map.\n\n key in d\n\n Return "True" if *d* has a key *key*, else "False".\n\n key not in d\n\n Equivalent to "not key in d".\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for "iter(d.keys())".\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n classmethod fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n "fromkeys()" is a class method that returns a new dictionary.\n *value* defaults to "None".\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to "None", so\n that this method never raises a "KeyError".\n\n items()\n\n Return a new view of the dictionary\'s items ("(key, value)"\n pairs). See the *documentation of view objects*.\n\n keys()\n\n Return a new view of the dictionary\'s keys. See the\n *documentation of view objects*.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a "KeyError" is raised.\n\n popitem()\n\n Remove and return an arbitrary "(key, value)" pair from the\n dictionary.\n\n "popitem()" is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling "popitem()" raises a "KeyError".\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to "None".\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return "None".\n\n "update()" accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: "d.update(red=1,\n blue=2)".\n\n values()\n\n Return a new view of the dictionary\'s values. See the\n *documentation of view objects*.\n\nSee also: "types.MappingProxyType" can be used to create a read-only\n view of a "dict".\n\n\nDictionary view objects\n=======================\n\nThe objects returned by "dict.keys()", "dict.values()" and\n"dict.items()" are *view objects*. They provide a dynamic view on the\ndictionary\'s entries, which means that when the dictionary changes,\nthe view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of "(key, value)") in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of "(value, key)" pairs using\n "zip()": "pairs = zip(d.values(), d.keys())". Another way to\n create the same list is "pairs = [(v, k) for (k, v) in d.items()]".\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a "RuntimeError" or fail to iterate over all entries.\n\nx in dictview\n\n Return "True" if *x* is in the underlying dictionary\'s keys, values\n or items (in the latter case, *x* should be a "(key, value)"\n tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that "(key, value)" pairs are unique\nand hashable, then the items view is also set-like. (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class "collections.abc.Set" are available (for example, "==",\n"<", or "^").\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.keys()\n >>> values = dishes.values()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n >>> keys ^ {\'sausage\', \'juice\'}\n {\'juice\', \'sausage\', \'bacon\', \'spam\'}\n',
'typesmethods': '\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as "append()" on lists)\nand class instance methods. Built-in methods are described with the\ntypes that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the "self"\nargument to the argument list. Bound methods have two special read-\nonly attributes: "m.__self__" is the object on which the method\noperates, and "m.__func__" is the function implementing the method.\nCalling "m(arg-1, arg-2, ..., arg-n)" is completely equivalent to\ncalling "m.__func__(m.__self__, arg-1, arg-2, ..., arg-n)".\n\nLike function objects, bound method objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object ("meth.__func__"), setting method\nattributes on bound methods is disallowed. Attempting to set an\nattribute on a method results in an "AttributeError" being raised. In\norder to set a method attribute, you need to explicitly set it on the\nunderlying function object:\n\n >>> class C:\n ... def method(self):\n ... pass\n ...\n >>> c = C()\n >>> c.method.whoami = \'my name is method\' # can\'t set on the method\n Traceback (most recent call last):\n File "<stdin>", line 1, in <module>\n AttributeError: \'method\' object has no attribute \'whoami\'\n >>> c.method.__func__.whoami = \'my name is method\'\n >>> c.method.whoami\n \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n',
diff --git a/Lib/quopri.py b/Lib/quopri.py
index 3d0f0ac..e5bd010 100755
--- a/Lib/quopri.py
+++ b/Lib/quopri.py
@@ -223,7 +223,7 @@ def main():
else:
try:
fp = open(file, "rb")
- except IOError as msg:
+ except OSError as msg:
sys.stderr.write("%s: can't open (%s)\n" % (file, msg))
sts = 1
continue
diff --git a/Lib/random.py b/Lib/random.py
index 14ac5e5..808175a 100644
--- a/Lib/random.py
+++ b/Lib/random.py
@@ -41,7 +41,7 @@ from types import MethodType as _MethodType, BuiltinMethodType as _BuiltinMethod
from math import log as _log, exp as _exp, pi as _pi, e as _e, ceil as _ceil
from math import sqrt as _sqrt, acos as _acos, cos as _cos, sin as _sin
from os import urandom as _urandom
-from collections.abc import Set as _Set, Sequence as _Sequence
+from _collections_abc import Set as _Set, Sequence as _Sequence
from hashlib import sha512 as _sha512
__all__ = ["Random","seed","random","uniform","randint","choice","sample",
@@ -151,6 +151,9 @@ class Random(_random.Random):
## -------------------- pickle support -------------------
+ # Issue 17489: Since __reduce__ was defined to fix #759889 this is no
+ # longer called; we leave it here because it has been here since random was
+ # rewritten back in 2001 and why risk breaking something.
def __getstate__(self): # for pickle
return self.getstate()
@@ -216,10 +219,11 @@ class Random(_random.Random):
Method=_MethodType, BuiltinMethod=_BuiltinMethodType):
"Return a random int in the range [0,n). Raises ValueError if n==0."
+ random = self.random
getrandbits = self.getrandbits
# Only call self.getrandbits if the original random() builtin method
# has not been overridden or if a new getrandbits() was supplied.
- if type(self.random) is BuiltinMethod or type(getrandbits) is Method:
+ if type(random) is BuiltinMethod or type(getrandbits) is Method:
k = n.bit_length() # don't use (n-1) here because n can be 1
r = getrandbits(k) # 0 <= r < 2**k
while r >= n:
@@ -227,7 +231,6 @@ class Random(_random.Random):
return r
# There's an overriden random() method but no new getrandbits() method,
# so we can only use random() from here.
- random = self.random
if n >= maxsize:
_warn("Underlying random() generator does not supply \n"
"enough bits to choose from a population range this large.\n"
@@ -251,10 +254,11 @@ class Random(_random.Random):
return seq[i]
def shuffle(self, x, random=None):
- """x, random=random.random -> shuffle list x in place; return None.
+ """Shuffle list x in place, and return None.
- Optional arg random is a 0-argument function returning a random
- float in [0.0, 1.0); by default, the standard random.random.
+ Optional argument random is a 0-argument function returning a
+ random float in [0.0, 1.0); if it is the default None, the
+ standard random.random will be used.
"""
diff --git a/Lib/re.py b/Lib/re.py
index a46ecc8..2e4d87c 100644
--- a/Lib/re.py
+++ b/Lib/re.py
@@ -85,16 +85,17 @@ resulting RE will match the second character.
\\ Matches a literal backslash.
This module exports the following functions:
- match Match a regular expression pattern to the beginning of a string.
- search Search a string for the presence of a pattern.
- sub Substitute occurrences of a pattern found in a string.
- subn Same as sub, but also return the number of substitutions made.
- split Split a string by the occurrences of a pattern.
- findall Find all occurrences of a pattern in a string.
- finditer Return an iterator yielding a match object for each match.
- compile Compile a pattern into a RegexObject.
- purge Clear the regular expression cache.
- escape Backslash all non-alphanumerics in a string.
+ match Match a regular expression pattern to the beginning of a string.
+ fullmatch Match a regular expression pattern to all of a string.
+ search Search a string for the presence of a pattern.
+ sub Substitute occurrences of a pattern found in a string.
+ subn Same as sub, but also return the number of substitutions made.
+ split Split a string by the occurrences of a pattern.
+ findall Find all occurrences of a pattern in a string.
+ finditer Return an iterator yielding a match object for each match.
+ compile Compile a pattern into a RegexObject.
+ purge Clear the regular expression cache.
+ escape Backslash all non-alphanumerics in a string.
Some of the functions in this module takes flags as optional parameters:
A ASCII For string patterns, make \w, \W, \b, \B, \d, \D
@@ -121,10 +122,9 @@ This module also defines an exception 'error'.
import sys
import sre_compile
import sre_parse
-import functools
# public symbols
-__all__ = [ "match", "search", "sub", "subn", "split", "findall",
+__all__ = [ "match", "fullmatch", "search", "sub", "subn", "split", "findall",
"compile", "purge", "template", "escape", "A", "I", "L", "M", "S", "X",
"U", "ASCII", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
"UNICODE", "error" ]
@@ -155,6 +155,11 @@ def match(pattern, string, flags=0):
a match object, or None if no match was found."""
return _compile(pattern, flags).match(string)
+def fullmatch(pattern, string, flags=0):
+ """Try to apply the pattern to all of the string, returning
+ a match object, or None if no match was found."""
+ return _compile(pattern, flags).fullmatch(string)
+
def search(pattern, string, flags=0):
"""Scan through string looking for a match to the pattern, returning
a match object, or None if no match was found."""
diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py
index 3f97053..94f9341 100644
--- a/Lib/rlcompleter.py
+++ b/Lib/rlcompleter.py
@@ -29,6 +29,7 @@ Notes:
"""
+import atexit
import builtins
import __main__
@@ -158,3 +159,7 @@ except ImportError:
pass
else:
readline.set_completer(Completer().complete)
+ # Release references early at shutdown (the readline module's
+ # contents are quasi-immortal, and the completer function holds a
+ # reference to globals).
+ atexit.register(lambda: readline.set_completer(None))
diff --git a/Lib/runpy.py b/Lib/runpy.py
index 39c0e9f..0bb57d7 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -10,11 +10,11 @@ importers when locating support scripts as well as when importing modules.
# to implement PEP 338 (Executing Modules as Scripts)
-import os
import sys
import importlib.machinery # importlib first so we can test #15386 via -m
-import imp
-from pkgutil import read_code, get_loader, get_importer
+import importlib.util
+import types
+from pkgutil import read_code, get_importer
__all__ = [
"run_module", "run_path",
@@ -24,7 +24,7 @@ class _TempModule(object):
"""Temporarily replace a module in sys.modules with an empty namespace"""
def __init__(self, mod_name):
self.mod_name = mod_name
- self.module = imp.new_module(mod_name)
+ self.module = types.ModuleType(mod_name)
self._saved_module = []
def __enter__(self):
@@ -58,51 +58,59 @@ class _ModifiedArgv0(object):
self.value = self._sentinel
sys.argv[0] = self._saved_value
+# TODO: Replace these helpers with importlib._bootstrap._SpecMethods
def _run_code(code, run_globals, init_globals=None,
- mod_name=None, mod_fname=None,
- mod_loader=None, pkg_name=None):
+ mod_name=None, mod_spec=None,
+ pkg_name=None, script_name=None):
"""Helper to run code in nominated namespace"""
if init_globals is not None:
run_globals.update(init_globals)
+ if mod_spec is None:
+ loader = None
+ fname = script_name
+ cached = None
+ else:
+ loader = mod_spec.loader
+ fname = mod_spec.origin
+ cached = mod_spec.cached
+ if pkg_name is None:
+ pkg_name = mod_spec.parent
run_globals.update(__name__ = mod_name,
- __file__ = mod_fname,
- __cached__ = None,
+ __file__ = fname,
+ __cached__ = cached,
__doc__ = None,
- __loader__ = mod_loader,
- __package__ = pkg_name)
+ __loader__ = loader,
+ __package__ = pkg_name,
+ __spec__ = mod_spec)
exec(code, run_globals)
return run_globals
def _run_module_code(code, init_globals=None,
- mod_name=None, mod_fname=None,
- mod_loader=None, pkg_name=None):
+ mod_name=None, mod_spec=None,
+ pkg_name=None, script_name=None):
"""Helper to run code in new namespace with sys modified"""
- with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname):
+ fname = script_name if mod_spec is None else mod_spec.origin
+ with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname):
mod_globals = temp_module.module.__dict__
_run_code(code, mod_globals, init_globals,
- mod_name, mod_fname, mod_loader, pkg_name)
+ mod_name, mod_spec, pkg_name, script_name)
# Copy the globals of the temporary module, as they
# may be cleared when the temporary module goes away
return mod_globals.copy()
-
-# This helper is needed due to a missing component in the PEP 302
-# loader protocol (specifically, "get_filename" is non-standard)
-# Since we can't introduce new features in maintenance releases,
-# support was added to zipimporter under the name '_get_filename'
-def _get_filename(loader, mod_name):
- for attr in ("get_filename", "_get_filename"):
- meth = getattr(loader, attr, None)
- if meth is not None:
- return os.path.abspath(meth(mod_name))
- return None
-
# Helper to get the loader, code and filename for a module
def _get_module_details(mod_name):
- loader = get_loader(mod_name)
- if loader is None:
+ try:
+ spec = importlib.util.find_spec(mod_name)
+ except (ImportError, AttributeError, TypeError, ValueError) as ex:
+ # This hack fixes an impedance mismatch between pkgutil and
+ # importlib, where the latter raises other errors for cases where
+ # pkgutil previously raised ImportError
+ msg = "Error while finding spec for {!r} ({}: {})"
+ raise ImportError(msg.format(mod_name, type(ex), ex)) from ex
+ if spec is None:
raise ImportError("No module named %s" % mod_name)
- if loader.is_package(mod_name):
+ if spec.submodule_search_locations is not None:
if mod_name == "__main__" or mod_name.endswith(".__main__"):
raise ImportError("Cannot use package as __main__ module")
try:
@@ -111,11 +119,14 @@ def _get_module_details(mod_name):
except ImportError as e:
raise ImportError(("%s; %r is a package and cannot " +
"be directly executed") %(e, mod_name))
+ loader = spec.loader
+ if loader is None:
+ raise ImportError("%r is a namespace package and cannot be executed"
+ % mod_name)
code = loader.get_code(mod_name)
if code is None:
raise ImportError("No code object available for %s" % mod_name)
- filename = _get_filename(loader, mod_name)
- return mod_name, loader, code, filename
+ return mod_name, spec, code
# XXX ncoghlan: Should this be documented and made public?
# (Current thoughts: don't repeat the mistake that lead to its
@@ -137,9 +148,9 @@ def _run_module_as_main(mod_name, alter_argv=True):
"""
try:
if alter_argv or mod_name != "__main__": # i.e. -m switch
- mod_name, loader, code, fname = _get_module_details(mod_name)
+ mod_name, mod_spec, code = _get_module_details(mod_name)
else: # i.e. directory or zipfile execution
- mod_name, loader, code, fname = _get_main_module_details()
+ mod_name, mod_spec, code = _get_main_module_details()
except ImportError as exc:
# Try to provide a good error message
# for directories, zip files and the -m switch
@@ -152,12 +163,11 @@ def _run_module_as_main(mod_name, alter_argv=True):
info = "can't find '__main__' module in %r" % sys.argv[0]
msg = "%s: %s" % (sys.executable, info)
sys.exit(msg)
- pkg_name = mod_name.rpartition('.')[0]
main_globals = sys.modules["__main__"].__dict__
if alter_argv:
- sys.argv[0] = fname
+ sys.argv[0] = mod_spec.origin
return _run_code(code, main_globals, None,
- "__main__", fname, loader, pkg_name)
+ "__main__", mod_spec)
def run_module(mod_name, init_globals=None,
run_name=None, alter_sys=False):
@@ -165,17 +175,14 @@ def run_module(mod_name, init_globals=None,
Returns the resulting top level namespace dictionary
"""
- mod_name, loader, code, fname = _get_module_details(mod_name)
+ mod_name, mod_spec, code = _get_module_details(mod_name)
if run_name is None:
run_name = mod_name
- pkg_name = mod_name.rpartition('.')[0]
if alter_sys:
- return _run_module_code(code, init_globals, run_name,
- fname, loader, pkg_name)
+ return _run_module_code(code, init_globals, run_name, mod_spec)
else:
# Leave the sys module alone
- return _run_code(code, {}, init_globals, run_name,
- fname, loader, pkg_name)
+ return _run_code(code, {}, init_globals, run_name, mod_spec)
def _get_main_module_details():
# Helper that gives a nicer error message when attempting to
@@ -204,10 +211,7 @@ def _get_code_from_file(run_name, fname):
# That didn't work, so try it as normal source code
with open(fname, "rb") as f:
code = compile(f.read(), fname, 'exec')
- loader = importlib.machinery.SourceFileLoader(run_name, fname)
- else:
- loader = importlib.machinery.SourcelessFileLoader(run_name, fname)
- return code, loader
+ return code, fname
def run_path(path_name, init_globals=None, run_name=None):
"""Execute code located at the specified filesystem location
@@ -223,12 +227,17 @@ def run_path(path_name, init_globals=None, run_name=None):
run_name = "<run_path>"
pkg_name = run_name.rpartition(".")[0]
importer = get_importer(path_name)
- if isinstance(importer, (type(None), imp.NullImporter)):
+ # Trying to avoid importing imp so as to not consume the deprecation warning.
+ is_NullImporter = False
+ if type(importer).__module__ == 'imp':
+ if type(importer).__name__ == 'NullImporter':
+ is_NullImporter = True
+ if isinstance(importer, type(None)) or is_NullImporter:
# Not a valid sys.path entry, so run the code directly
# execfile() doesn't help as we want to allow compiled files
- code, mod_loader = _get_code_from_file(run_name, path_name)
- return _run_module_code(code, init_globals, run_name, path_name,
- mod_loader, pkg_name)
+ code, fname = _get_code_from_file(run_name, path_name)
+ return _run_module_code(code, init_globals, run_name,
+ pkg_name=pkg_name, script_name=fname)
else:
# Importer is defined for path, so add it to
# the start of sys.path
@@ -240,12 +249,12 @@ def run_path(path_name, init_globals=None, run_name=None):
# have no choice and we have to remove it even while we read the
# code. If we don't do this, a __loader__ attribute in the
# existing __main__ module may prevent location of the new module.
- mod_name, loader, code, fname = _get_main_module_details()
+ mod_name, mod_spec, code = _get_main_module_details()
with _TempModule(run_name) as temp_module, \
_ModifiedArgv0(path_name):
mod_globals = temp_module.module.__dict__
return _run_code(code, mod_globals, init_globals,
- run_name, fname, loader, pkg_name).copy()
+ run_name, mod_spec, pkg_name).copy()
finally:
try:
sys.path.remove(path_name)
diff --git a/Lib/sched.py b/Lib/sched.py
index b9a7ad1..2e6b00a 100644
--- a/Lib/sched.py
+++ b/Lib/sched.py
@@ -71,10 +71,10 @@ class scheduler:
"""
if kwargs is _sentinel:
kwargs = {}
+ event = Event(time, priority, action, argument, kwargs)
with self._lock:
- event = Event(time, priority, action, argument, kwargs)
heapq.heappush(self._queue, event)
- return event # The ID
+ return event # The ID
def enter(self, delay, priority, action, argument=(), kwargs=_sentinel):
"""A variant that specifies the time as a relative time.
@@ -82,9 +82,8 @@ class scheduler:
This is actually the more commonly used interface.
"""
- with self._lock:
- time = self.timefunc() + delay
- return self.enterabs(time, priority, action, argument, kwargs)
+ time = self.timefunc() + delay
+ return self.enterabs(time, priority, action, argument, kwargs)
def cancel(self, event):
"""Remove an event from the queue.
@@ -165,4 +164,4 @@ class scheduler:
# the actual order they would be retrieved.
with self._lock:
events = self._queue[:]
- return list(map(heapq.heappop, [events]*len(events)))
+ return list(map(heapq.heappop, [events]*len(events)))
diff --git a/Lib/selectors.py b/Lib/selectors.py
new file mode 100644
index 0000000..9be9225
--- /dev/null
+++ b/Lib/selectors.py
@@ -0,0 +1,525 @@
+"""Selectors module.
+
+This module allows high-level and efficient I/O multiplexing, built upon the
+`select` module primitives.
+"""
+
+
+from abc import ABCMeta, abstractmethod
+from collections import namedtuple, Mapping
+import math
+import select
+import sys
+
+
+# generic events, that must be mapped to implementation-specific ones
+EVENT_READ = (1 << 0)
+EVENT_WRITE = (1 << 1)
+
+
+def _fileobj_to_fd(fileobj):
+ """Return a file descriptor from a file object.
+
+ Parameters:
+ fileobj -- file object or file descriptor
+
+ Returns:
+ corresponding file descriptor
+
+ Raises:
+ ValueError if the object is invalid
+ """
+ if isinstance(fileobj, int):
+ fd = fileobj
+ else:
+ try:
+ fd = int(fileobj.fileno())
+ except (AttributeError, TypeError, ValueError):
+ raise ValueError("Invalid file object: "
+ "{!r}".format(fileobj)) from None
+ if fd < 0:
+ raise ValueError("Invalid file descriptor: {}".format(fd))
+ return fd
+
+
+SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
+"""Object used to associate a file object to its backing file descriptor,
+selected event mask and attached data."""
+
+
+class _SelectorMapping(Mapping):
+ """Mapping of file objects to selector keys."""
+
+ def __init__(self, selector):
+ self._selector = selector
+
+ def __len__(self):
+ return len(self._selector._fd_to_key)
+
+ def __getitem__(self, fileobj):
+ try:
+ fd = self._selector._fileobj_lookup(fileobj)
+ return self._selector._fd_to_key[fd]
+ except KeyError:
+ raise KeyError("{!r} is not registered".format(fileobj)) from None
+
+ def __iter__(self):
+ return iter(self._selector._fd_to_key)
+
+
+class BaseSelector(metaclass=ABCMeta):
+ """Selector abstract base class.
+
+ A selector supports registering file objects to be monitored for specific
+ I/O events.
+
+ A file object is a file descriptor or any object with a `fileno()` method.
+ An arbitrary object can be attached to the file object, which can be used
+ for example to store context information, a callback, etc.
+
+ A selector can use various implementations (select(), poll(), epoll()...)
+ depending on the platform. The default `Selector` class uses the most
+ efficient implementation on the current platform.
+ """
+
+ @abstractmethod
+ def register(self, fileobj, events, data=None):
+ """Register a file object.
+
+ Parameters:
+ fileobj -- file object or file descriptor
+ events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE)
+ data -- attached data
+
+ Returns:
+ SelectorKey instance
+
+ Raises:
+ ValueError if events is invalid
+ KeyError if fileobj is already registered
+ OSError if fileobj is closed or otherwise is unacceptable to
+ the underlying system call (if a system call is made)
+
+ Note:
+ OSError may or may not be raised
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def unregister(self, fileobj):
+ """Unregister a file object.
+
+ Parameters:
+ fileobj -- file object or file descriptor
+
+ Returns:
+ SelectorKey instance
+
+ Raises:
+ KeyError if fileobj is not registered
+
+ Note:
+ If fileobj is registered but has since been closed this does
+ *not* raise OSError (even if the wrapped syscall does)
+ """
+ raise NotImplementedError
+
+ def modify(self, fileobj, events, data=None):
+ """Change a registered file object monitored events or attached data.
+
+ Parameters:
+ fileobj -- file object or file descriptor
+ events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE)
+ data -- attached data
+
+ Returns:
+ SelectorKey instance
+
+ Raises:
+ Anything that unregister() or register() raises
+ """
+ self.unregister(fileobj)
+ return self.register(fileobj, events, data)
+
+ @abstractmethod
+ def select(self, timeout=None):
+ """Perform the actual selection, until some monitored file objects are
+ ready or a timeout expires.
+
+ Parameters:
+ timeout -- if timeout > 0, this specifies the maximum wait time, in
+ seconds
+ if timeout <= 0, the select() call won't block, and will
+ report the currently ready file objects
+ if timeout is None, select() will block until a monitored
+ file object becomes ready
+
+ Returns:
+ list of (key, events) for ready file objects
+ `events` is a bitwise mask of EVENT_READ|EVENT_WRITE
+ """
+ raise NotImplementedError
+
+ def close(self):
+ """Close the selector.
+
+ This must be called to make sure that any underlying resource is freed.
+ """
+ pass
+
+ def get_key(self, fileobj):
+ """Return the key associated to a registered file object.
+
+ Returns:
+ SelectorKey for this file object
+ """
+ mapping = self.get_map()
+ try:
+ return mapping[fileobj]
+ except KeyError:
+ raise KeyError("{!r} is not registered".format(fileobj)) from None
+
+ @abstractmethod
+ def get_map(self):
+ """Return a mapping of file objects to selector keys."""
+ raise NotImplementedError
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
+
+class _BaseSelectorImpl(BaseSelector):
+ """Base selector implementation."""
+
+ def __init__(self):
+ # this maps file descriptors to keys
+ self._fd_to_key = {}
+ # read-only mapping returned by get_map()
+ self._map = _SelectorMapping(self)
+
+ def _fileobj_lookup(self, fileobj):
+ """Return a file descriptor from a file object.
+
+ This wraps _fileobj_to_fd() to do an exhaustive search in case
+ the object is invalid but we still have it in our map. This
+ is used by unregister() so we can unregister an object that
+ was previously registered even if it is closed. It is also
+ used by _SelectorMapping.
+ """
+ try:
+ return _fileobj_to_fd(fileobj)
+ except ValueError:
+ # Do an exhaustive search.
+ for key in self._fd_to_key.values():
+ if key.fileobj is fileobj:
+ return key.fd
+ # Raise ValueError after all.
+ raise
+
+ def register(self, fileobj, events, data=None):
+ if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
+ raise ValueError("Invalid events: {!r}".format(events))
+
+ key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
+
+ if key.fd in self._fd_to_key:
+ raise KeyError("{!r} (FD {}) is already registered"
+ .format(fileobj, key.fd))
+
+ self._fd_to_key[key.fd] = key
+ return key
+
+ def unregister(self, fileobj):
+ try:
+ key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
+ except KeyError:
+ raise KeyError("{!r} is not registered".format(fileobj)) from None
+ return key
+
+ def modify(self, fileobj, events, data=None):
+ # TODO: Subclasses can probably optimize this even further.
+ try:
+ key = self._fd_to_key[self._fileobj_lookup(fileobj)]
+ except KeyError:
+ raise KeyError("{!r} is not registered".format(fileobj)) from None
+ if events != key.events:
+ self.unregister(fileobj)
+ key = self.register(fileobj, events, data)
+ elif data != key.data:
+ # Use a shortcut to update the data.
+ key = key._replace(data=data)
+ self._fd_to_key[key.fd] = key
+ return key
+
+ def close(self):
+ self._fd_to_key.clear()
+
+ def get_map(self):
+ return self._map
+
+ def _key_from_fd(self, fd):
+ """Return the key associated to a given file descriptor.
+
+ Parameters:
+ fd -- file descriptor
+
+ Returns:
+ corresponding key, or None if not found
+ """
+ try:
+ return self._fd_to_key[fd]
+ except KeyError:
+ return None
+
+
+class SelectSelector(_BaseSelectorImpl):
+ """Select-based selector."""
+
+ def __init__(self):
+ super().__init__()
+ self._readers = set()
+ self._writers = set()
+
+ def register(self, fileobj, events, data=None):
+ key = super().register(fileobj, events, data)
+ if events & EVENT_READ:
+ self._readers.add(key.fd)
+ if events & EVENT_WRITE:
+ self._writers.add(key.fd)
+ return key
+
+ def unregister(self, fileobj):
+ key = super().unregister(fileobj)
+ self._readers.discard(key.fd)
+ self._writers.discard(key.fd)
+ return key
+
+ if sys.platform == 'win32':
+ def _select(self, r, w, _, timeout=None):
+ r, w, x = select.select(r, w, w, timeout)
+ return r, w + x, []
+ else:
+ _select = select.select
+
+ def select(self, timeout=None):
+ timeout = None if timeout is None else max(timeout, 0)
+ ready = []
+ try:
+ r, w, _ = self._select(self._readers, self._writers, [], timeout)
+ except InterruptedError:
+ return ready
+ r = set(r)
+ w = set(w)
+ for fd in r | w:
+ events = 0
+ if fd in r:
+ events |= EVENT_READ
+ if fd in w:
+ events |= EVENT_WRITE
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+
+if hasattr(select, 'poll'):
+
+ class PollSelector(_BaseSelectorImpl):
+ """Poll-based selector."""
+
+ def __init__(self):
+ super().__init__()
+ self._poll = select.poll()
+
+ def register(self, fileobj, events, data=None):
+ key = super().register(fileobj, events, data)
+ poll_events = 0
+ if events & EVENT_READ:
+ poll_events |= select.POLLIN
+ if events & EVENT_WRITE:
+ poll_events |= select.POLLOUT
+ self._poll.register(key.fd, poll_events)
+ return key
+
+ def unregister(self, fileobj):
+ key = super().unregister(fileobj)
+ self._poll.unregister(key.fd)
+ return key
+
+ def select(self, timeout=None):
+ if timeout is None:
+ timeout = None
+ elif timeout <= 0:
+ timeout = 0
+ else:
+ # poll() has a resolution of 1 millisecond, round away from
+ # zero to wait *at least* timeout seconds.
+ timeout = math.ceil(timeout * 1e3)
+ ready = []
+ try:
+ fd_event_list = self._poll.poll(timeout)
+ except InterruptedError:
+ return ready
+ for fd, event in fd_event_list:
+ events = 0
+ if event & ~select.POLLIN:
+ events |= EVENT_WRITE
+ if event & ~select.POLLOUT:
+ events |= EVENT_READ
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+
+if hasattr(select, 'epoll'):
+
+ class EpollSelector(_BaseSelectorImpl):
+ """Epoll-based selector."""
+
+ def __init__(self):
+ super().__init__()
+ self._epoll = select.epoll()
+
+ def fileno(self):
+ return self._epoll.fileno()
+
+ def register(self, fileobj, events, data=None):
+ key = super().register(fileobj, events, data)
+ epoll_events = 0
+ if events & EVENT_READ:
+ epoll_events |= select.EPOLLIN
+ if events & EVENT_WRITE:
+ epoll_events |= select.EPOLLOUT
+ self._epoll.register(key.fd, epoll_events)
+ return key
+
+ def unregister(self, fileobj):
+ key = super().unregister(fileobj)
+ try:
+ self._epoll.unregister(key.fd)
+ except OSError:
+ # This can happen if the FD was closed since it
+ # was registered.
+ pass
+ return key
+
+ def select(self, timeout=None):
+ if timeout is None:
+ timeout = -1
+ elif timeout <= 0:
+ timeout = 0
+ else:
+ # epoll_wait() has a resolution of 1 millisecond, round away
+ # from zero to wait *at least* timeout seconds.
+ timeout = math.ceil(timeout * 1e3) * 1e-3
+ max_ev = len(self._fd_to_key)
+ ready = []
+ try:
+ fd_event_list = self._epoll.poll(timeout, max_ev)
+ except InterruptedError:
+ return ready
+ for fd, event in fd_event_list:
+ events = 0
+ if event & ~select.EPOLLIN:
+ events |= EVENT_WRITE
+ if event & ~select.EPOLLOUT:
+ events |= EVENT_READ
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+ def close(self):
+ self._epoll.close()
+ super().close()
+
+
+if hasattr(select, 'kqueue'):
+
+ class KqueueSelector(_BaseSelectorImpl):
+ """Kqueue-based selector."""
+
+ def __init__(self):
+ super().__init__()
+ self._kqueue = select.kqueue()
+
+ def fileno(self):
+ return self._kqueue.fileno()
+
+ def register(self, fileobj, events, data=None):
+ key = super().register(fileobj, events, data)
+ if events & EVENT_READ:
+ kev = select.kevent(key.fd, select.KQ_FILTER_READ,
+ select.KQ_EV_ADD)
+ self._kqueue.control([kev], 0, 0)
+ if events & EVENT_WRITE:
+ kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
+ select.KQ_EV_ADD)
+ self._kqueue.control([kev], 0, 0)
+ return key
+
+ def unregister(self, fileobj):
+ key = super().unregister(fileobj)
+ if key.events & EVENT_READ:
+ kev = select.kevent(key.fd, select.KQ_FILTER_READ,
+ select.KQ_EV_DELETE)
+ try:
+ self._kqueue.control([kev], 0, 0)
+ except OSError:
+ # This can happen if the FD was closed since it
+ # was registered.
+ pass
+ if key.events & EVENT_WRITE:
+ kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
+ select.KQ_EV_DELETE)
+ try:
+ self._kqueue.control([kev], 0, 0)
+ except OSError:
+ # See comment above.
+ pass
+ return key
+
+ def select(self, timeout=None):
+ timeout = None if timeout is None else max(timeout, 0)
+ max_ev = len(self._fd_to_key)
+ ready = []
+ try:
+ kev_list = self._kqueue.control(None, max_ev, timeout)
+ except InterruptedError:
+ return ready
+ for kev in kev_list:
+ fd = kev.ident
+ flag = kev.filter
+ events = 0
+ if flag == select.KQ_FILTER_READ:
+ events |= EVENT_READ
+ if flag == select.KQ_FILTER_WRITE:
+ events |= EVENT_WRITE
+
+ key = self._key_from_fd(fd)
+ if key:
+ ready.append((key, events & key.events))
+ return ready
+
+ def close(self):
+ self._kqueue.close()
+ super().close()
+
+
+# Choose the best implementation: roughly, epoll|kqueue > poll > select.
+# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
+if 'KqueueSelector' in globals():
+ DefaultSelector = KqueueSelector
+elif 'EpollSelector' in globals():
+ DefaultSelector = EpollSelector
+elif 'PollSelector' in globals():
+ DefaultSelector = PollSelector
+else:
+ DefaultSelector = SelectSelector
diff --git a/Lib/shelve.py b/Lib/shelve.py
index cc1815e..cef580e 100644
--- a/Lib/shelve.py
+++ b/Lib/shelve.py
@@ -61,7 +61,7 @@ from io import BytesIO
import collections
-__all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"]
+__all__ = ["Shelf", "BsdDbShelf", "DbfilenameShelf", "open"]
class _ClosedDict(collections.MutableMapping):
'Marker for a closed dict. Access attempts raise a ValueError.'
@@ -131,6 +131,12 @@ class Shelf(collections.MutableMapping):
except KeyError:
pass
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
def close(self):
self.sync()
try:
@@ -147,6 +153,7 @@ class Shelf(collections.MutableMapping):
def __del__(self):
if not hasattr(self, 'writeback'):
# __init__ didn't succeed, so don't bother closing
+ # see http://bugs.python.org/issue1339007 for details
return
self.close()
diff --git a/Lib/shutil.py b/Lib/shutil.py
index 5a4d4f6..0cd6ec4 100644
--- a/Lib/shutil.py
+++ b/Lib/shutil.py
@@ -39,17 +39,20 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
"ignore_patterns", "chown", "which"]
# disk_usage is added later, if available on the platform
-class Error(EnvironmentError):
+class Error(OSError):
pass
-class SpecialFileError(EnvironmentError):
+class SameFileError(Error):
+ """Raised when source and destination are the same file."""
+
+class SpecialFileError(OSError):
"""Raised when trying to do a kind of operation (e.g. copying) which is
not supported on a special file (e.g. a named pipe)"""
-class ExecError(EnvironmentError):
+class ExecError(OSError):
"""Raised when a command could not be executed"""
-class ReadError(EnvironmentError):
+class ReadError(OSError):
"""Raised when an archive cannot be read"""
class RegistryError(Exception):
@@ -57,11 +60,6 @@ class RegistryError(Exception):
and unpacking registeries fails"""
-try:
- WindowsError
-except NameError:
- WindowsError = None
-
def copyfileobj(fsrc, fdst, length=16*1024):
"""copy data from file-like object fsrc to file-like object fdst"""
while 1:
@@ -90,7 +88,7 @@ def copyfile(src, dst, *, follow_symlinks=True):
"""
if _samefile(src, dst):
- raise Error("`%s` and `%s` are the same file" % (src, dst))
+ raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
@@ -221,6 +219,9 @@ def copy(src, dst, *, follow_symlinks=True):
If follow_symlinks is false, symlinks won't be followed. This
resembles GNU's "cp -P src dst".
+ If source and destination are the same file, a SameFileError will be
+ raised.
+
"""
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
@@ -329,15 +330,13 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
# continue with other files
except Error as err:
errors.extend(err.args[0])
- except EnvironmentError as why:
+ except OSError as why:
errors.append((srcname, dstname, str(why)))
try:
copystat(src, dst)
except OSError as why:
- if WindowsError is not None and isinstance(why, WindowsError):
- # Copying file access times may fail on Windows
- pass
- else:
+ # Copying file access times may fail on Windows
+ if why.winerror is None:
errors.append((src, dst, str(why)))
if errors:
raise Error(errors)
@@ -356,24 +355,24 @@ def _rmtree_unsafe(path, onerror):
names = []
try:
names = os.listdir(path)
- except os.error:
+ except OSError:
onerror(os.listdir, path, sys.exc_info())
for name in names:
fullname = os.path.join(path, name)
try:
mode = os.lstat(fullname).st_mode
- except os.error:
+ except OSError:
mode = 0
if stat.S_ISDIR(mode):
_rmtree_unsafe(fullname, onerror)
else:
try:
os.unlink(fullname)
- except os.error:
+ except OSError:
onerror(os.unlink, fullname, sys.exc_info())
try:
os.rmdir(path)
- except os.error:
+ except OSError:
onerror(os.rmdir, path, sys.exc_info())
# Version using fd-based APIs to protect against races
@@ -464,7 +463,7 @@ def rmtree(path, ignore_errors=False, onerror=None):
_rmtree_safe_fd(fd, path, onerror)
try:
os.rmdir(path)
- except os.error:
+ except OSError:
onerror(os.rmdir, path, sys.exc_info())
else:
try:
diff --git a/Lib/site.py b/Lib/site.py
index 7e09701..ded6539 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -58,17 +58,20 @@ Note that bletch is omitted because it doesn't exist; bar precedes foo
because bar.pth comes alphabetically before foo.pth; and spam is
omitted because it is not mentioned in either path configuration file.
-After these path manipulations, an attempt is made to import a module
+The readline module is also automatically configured to enable
+completion for systems that support it. This can be overriden in
+sitecustomize, usercustomize or PYTHONSTARTUP.
+
+After these operations, an attempt is made to import a module
named sitecustomize, which can perform arbitrary additional
site-specific customizations. If this import fails with an
ImportError exception, it is silently ignored.
-
"""
import sys
import os
-import re
import builtins
+import _sitebuiltins
# Prefixes for site-packages; add additional prefixes like /usr/local here
PREFIXES = [sys.prefix, sys.exec_prefix]
@@ -146,14 +149,14 @@ def addpackage(sitedir, name, known_paths):
and add that to known_paths, or execute it if it starts with 'import '.
"""
if known_paths is None:
- _init_pathinfo()
+ known_paths = _init_pathinfo()
reset = 1
else:
reset = 0
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "r")
- except IOError:
+ except OSError:
return
with f:
for n, line in enumerate(f):
@@ -196,7 +199,7 @@ def addsitedir(sitedir, known_paths=None):
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
- except os.error:
+ except OSError:
return
names = [name for name in names if name.endswith(".pth")]
for name in sorted(names):
@@ -300,9 +303,7 @@ def getsitepackages(prefixes=None):
continue
seen.add(prefix)
- if sys.platform in ('os2emx', 'riscos'):
- sitepackages.append(os.path.join(prefix, "Lib", "site-packages"))
- elif os.sep == '/':
+ if os.sep == '/':
sitepackages.append(os.path.join(prefix, "lib",
"python" + sys.version[:3],
"site-packages"))
@@ -325,27 +326,15 @@ def addsitepackages(known_paths, prefixes=None):
"""Add site-packages (and possibly site-python) to sys.path"""
for sitedir in getsitepackages(prefixes):
if os.path.isdir(sitedir):
+ if "site-python" in sitedir:
+ import warnings
+ warnings.warn('"site-python" directories will not be '
+ 'supported in 3.5 anymore',
+ DeprecationWarning)
addsitedir(sitedir, known_paths)
return known_paths
-def setBEGINLIBPATH():
- """The OS/2 EMX port has optional extension modules that do double duty
- as DLLs (and must use the .DLL file extension) for other extensions.
- The library search path needs to be amended so these will be found
- during module import. Use BEGINLIBPATH so that these are at the start
- of the library search path.
-
- """
- dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload")
- libpath = os.environ['BEGINLIBPATH'].split(';')
- if libpath[-1]:
- libpath.append(dllpath)
- else:
- libpath[-1] = dllpath
- os.environ['BEGINLIBPATH'] = ';'.join(libpath)
-
-
def setquit():
"""Define new builtins 'quit' and 'exit'.
@@ -360,126 +349,94 @@ def setquit():
else:
eof = 'Ctrl-D (i.e. EOF)'
- class Quitter(object):
- def __init__(self, name):
- self.name = name
- def __repr__(self):
- return 'Use %s() or %s to exit' % (self.name, eof)
- def __call__(self, code=None):
- # Shells like IDLE catch the SystemExit, but listen when their
- # stdin wrapper is closed.
- try:
- sys.stdin.close()
- except:
- pass
- raise SystemExit(code)
- builtins.quit = Quitter('quit')
- builtins.exit = Quitter('exit')
-
-
-class _Printer(object):
- """interactive prompt objects for printing the license text, a list of
- contributors and the copyright notice."""
+ builtins.quit = _sitebuiltins.Quitter('quit', eof)
+ builtins.exit = _sitebuiltins.Quitter('exit', eof)
- MAXLINES = 23
-
- def __init__(self, name, data, files=(), dirs=()):
- self.__name = name
- self.__data = data
- self.__files = files
- self.__dirs = dirs
- self.__lines = None
-
- def __setup(self):
- if self.__lines:
- return
- data = None
- for dir in self.__dirs:
- for filename in self.__files:
- filename = os.path.join(dir, filename)
- try:
- fp = open(filename, "r")
- data = fp.read()
- fp.close()
- break
- except IOError:
- pass
- if data:
- break
- if not data:
- data = self.__data
- self.__lines = data.split('\n')
- self.__linecnt = len(self.__lines)
-
- def __repr__(self):
- self.__setup()
- if len(self.__lines) <= self.MAXLINES:
- return "\n".join(self.__lines)
- else:
- return "Type %s() to see the full %s text" % ((self.__name,)*2)
-
- def __call__(self):
- self.__setup()
- prompt = 'Hit Return for more, or q (and Return) to quit: '
- lineno = 0
- while 1:
- try:
- for i in range(lineno, lineno + self.MAXLINES):
- print(self.__lines[i])
- except IndexError:
- break
- else:
- lineno += self.MAXLINES
- key = None
- while key is None:
- key = input(prompt)
- if key not in ('', 'q'):
- key = None
- if key == 'q':
- break
def setcopyright():
"""Set 'copyright' and 'credits' in builtins"""
- builtins.copyright = _Printer("copyright", sys.copyright)
+ builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright)
if sys.platform[:4] == 'java':
- builtins.credits = _Printer(
+ builtins.credits = _sitebuiltins._Printer(
"credits",
"Jython is maintained by the Jython developers (www.jython.org).")
else:
- builtins.credits = _Printer("credits", """\
+ builtins.credits = _sitebuiltins._Printer("credits", """\
Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
for supporting Python development. See www.python.org for more information.""")
- here = os.path.dirname(os.__file__)
- builtins.license = _Printer(
+ files, dirs = [], []
+ # Not all modules are required to have a __file__ attribute. See
+ # PEP 420 for more details.
+ if hasattr(os, '__file__'):
+ here = os.path.dirname(os.__file__)
+ files.extend(["LICENSE.txt", "LICENSE"])
+ dirs.extend([os.path.join(here, os.pardir), here, os.curdir])
+ builtins.license = _sitebuiltins._Printer(
"license",
- "See http://www.python.org/download/releases/%.5s/license/" % sys.version,
- ["LICENSE.txt", "LICENSE"],
- [os.path.join(here, os.pardir), here, os.curdir])
+ "See http://www.python.org/download/releases/%.5s/license" % sys.version,
+ files, dirs)
-class _Helper(object):
- """Define the builtin 'help'.
- This is a wrapper around pydoc.help (with a twist).
+def sethelper():
+ builtins.help = _sitebuiltins._Helper()
+def enablerlcompleter():
+ """Enable default readline configuration on interactive prompts, by
+ registering a sys.__interactivehook__.
+
+ If the readline module can be imported, the hook will set the Tab key
+ as completion key and register ~/.python_history as history file.
+ This can be overriden in the sitecustomize or usercustomize module,
+ or in a PYTHONSTARTUP file.
"""
+ def register_readline():
+ import atexit
+ try:
+ import readline
+ import rlcompleter
+ except ImportError:
+ return
- def __repr__(self):
- return "Type help() for interactive help, " \
- "or help(object) for help about object."
- def __call__(self, *args, **kwds):
- import pydoc
- return pydoc.help(*args, **kwds)
+ # Reading the initialization (config) file may not be enough to set a
+ # completion key, so we set one first and then read the file.
+ readline_doc = getattr(readline, '__doc__', '')
+ if readline_doc is not None and 'libedit' in readline_doc:
+ readline.parse_and_bind('bind ^I rl_complete')
+ else:
+ readline.parse_and_bind('tab: complete')
-def sethelper():
- builtins.help = _Helper()
+ try:
+ readline.read_init_file()
+ except OSError:
+ # An OSError here could have many causes, but the most likely one
+ # is that there's no .inputrc file (or .editrc file in the case of
+ # Mac OS X + libedit) in the expected location. In that case, we
+ # want to ignore the exception.
+ pass
+
+ if readline.get_current_history_length() == 0:
+ # If no history was loaded, default to .python_history.
+ # The guard is necessary to avoid doubling history size at
+ # each interpreter exit when readline was already configured
+ # through a PYTHONSTARTUP hook, see:
+ # http://bugs.python.org/issue5845#msg198636
+ history = os.path.join(os.path.expanduser('~'),
+ '.python_history')
+ try:
+ readline.read_history_file(history)
+ except IOError:
+ pass
+ atexit.register(readline.write_history_file, history)
+
+ sys.__interactivehook__ = register_readline
def aliasmbcs():
"""On Windows, some default encodings are not provided by Python,
while they are always available as "mbcs" in each locale. Make
them usable by aliasing to "mbcs" in such a case."""
if sys.platform == 'win32':
- import locale, codecs
- enc = locale.getdefaultlocale()[1]
+ import _bootlocale, codecs
+ enc = _bootlocale.getpreferredencoding(False)
if enc.startswith('cp'): # "cp***" ?
try:
codecs.lookup(enc)
@@ -488,8 +445,7 @@ def aliasmbcs():
encodings._cache[enc] = encodings._unknown
encodings.aliases.aliases[enc] = 'mbcs'
-
-CONFIG_LINE = re.compile(r'^(?P<key>(\w|[-_])+)\s*=\s*(?P<value>.*)\s*$')
+CONFIG_LINE = r'^(?P<key>(\w|[-_])+)\s*=\s*(?P<value>.*)\s*$'
def venv(known_paths):
global PREFIXES, ENABLE_USER_SITE
@@ -512,12 +468,14 @@ def venv(known_paths):
]
if candidate_confs:
+ import re
+ config_line = re.compile(CONFIG_LINE)
virtual_conf = candidate_confs[0]
system_site = "true"
with open(virtual_conf) as f:
for line in f:
line = line.strip()
- m = CONFIG_LINE.match(line)
+ m = config_line.match(line)
if m:
d = m.groupdict()
key, value = d['key'].lower(), d['value']
@@ -589,11 +547,10 @@ def main():
ENABLE_USER_SITE = check_enableusersite()
known_paths = addusersitepackages(known_paths)
known_paths = addsitepackages(known_paths)
- if sys.platform == 'os2emx':
- setBEGINLIBPATH()
setquit()
setcopyright()
sethelper()
+ enablerlcompleter()
aliasmbcs()
execsitecustomize()
if ENABLE_USER_SITE:
diff --git a/Lib/smtpd.py b/Lib/smtpd.py
index 778d6d6..1fa157a 100755
--- a/Lib/smtpd.py
+++ b/Lib/smtpd.py
@@ -121,8 +121,9 @@ class SMTPChannel(asynchat.async_chat):
})
max_command_size_limit = max(command_size_limits.values())
- def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT):
- asynchat.async_chat.__init__(self, conn)
+ def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT,
+ map=None):
+ asynchat.async_chat.__init__(self, conn, map=map)
self.smtp_server = server
self.conn = conn
self.addr = addr
@@ -137,7 +138,7 @@ class SMTPChannel(asynchat.async_chat):
self.num_bytes = 0
try:
self.peer = conn.getpeername()
- except socket.error as err:
+ except OSError as err:
# a race condition may occur if the other end is closing
# before we can get the peername
self.close()
@@ -576,11 +577,11 @@ class SMTPServer(asyncore.dispatcher):
channel_class = SMTPChannel
def __init__(self, localaddr, remoteaddr,
- data_size_limit=DATA_SIZE_DEFAULT):
+ data_size_limit=DATA_SIZE_DEFAULT, map=None):
self._localaddr = localaddr
self._remoteaddr = remoteaddr
self.data_size_limit = data_size_limit
- asyncore.dispatcher.__init__(self)
+ asyncore.dispatcher.__init__(self, map=map)
try:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
# try to re-use a server port if possible
@@ -597,7 +598,8 @@ class SMTPServer(asyncore.dispatcher):
def handle_accepted(self, conn, addr):
print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM)
- channel = self.channel_class(self, conn, addr, self.data_size_limit)
+ channel = self.channel_class(self, conn, addr, self.data_size_limit,
+ self._map)
# API for "doing something useful with the message"
def process_message(self, peer, mailfrom, rcpttos, data):
@@ -668,7 +670,7 @@ class PureProxy(SMTPServer):
except smtplib.SMTPRecipientsRefused as e:
print('got SMTPRecipientsRefused', file=DEBUGSTREAM)
refused = e.recipients
- except (socket.error, smtplib.SMTPException) as e:
+ except (OSError, smtplib.SMTPException) as e:
print('got', e.__class__, file=DEBUGSTREAM)
# All recipients were refused. If the exception had an associated
# error code, use it. Otherwise,fake it with a non-triggering
@@ -778,7 +780,7 @@ def parseargs():
if opt in ('-h', '--help'):
usage(0)
elif opt in ('-V', '--version'):
- print(__version__, file=sys.stderr)
+ print(__version__)
sys.exit(0)
elif opt in ('-n', '--nosetuid'):
options.setuid = 0
@@ -850,8 +852,7 @@ if __name__ == '__main__':
nobody = pwd.getpwnam('nobody')[2]
try:
os.setuid(nobody)
- except OSError as e:
- if e.errno != errno.EPERM: raise
+ except PermissionError:
print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr)
sys.exit(1)
try:
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
index 57f181b..66b5879 100755
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -67,7 +67,7 @@ _MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3
OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I)
# Exception classes used by this module.
-class SMTPException(Exception):
+class SMTPException(OSError):
"""Base class for all exceptions raised by this module."""
class SMTPServerDisconnected(SMTPException):
@@ -233,6 +233,7 @@ class SMTP:
will be used.
"""
+ self._host = host
self.timeout = timeout
self.esmtp_features = {}
self.source_address = source_address
@@ -312,7 +313,7 @@ class SMTP:
try:
port = int(port)
except ValueError:
- raise socket.error("nonnumeric port")
+ raise OSError("nonnumeric port")
if not port:
port = self.default_port
if self.debuglevel > 0:
@@ -333,7 +334,7 @@ class SMTP:
s = s.encode("ascii")
try:
self.sock.sendall(s)
- except socket.error:
+ except OSError:
self.close()
raise SMTPServerDisconnected('Server not connected')
else:
@@ -366,7 +367,7 @@ class SMTP:
while 1:
try:
line = self.file.readline(_MAXLINE + 1)
- except socket.error as e:
+ except OSError as e:
self.close()
raise SMTPServerDisconnected("Connection unexpectedly closed: "
+ str(e))
@@ -582,7 +583,7 @@ class SMTP:
def encode_cram_md5(challenge, user, password):
challenge = base64.decodebytes(challenge)
response = user + " " + hmac.HMAC(password.encode('ascii'),
- challenge).hexdigest()
+ challenge, 'md5').hexdigest()
return encode_base64(response.encode('ascii'), eol='')
def encode_plain(user, password):
@@ -667,10 +668,12 @@ class SMTP:
if context is not None and certfile is not None:
raise ValueError("context and certfile arguments are mutually "
"exclusive")
- if context is not None:
- self.sock = context.wrap_socket(self.sock)
- else:
- self.sock = ssl.wrap_socket(self.sock, keyfile, certfile)
+ if context is None:
+ context = ssl._create_stdlib_context(certfile=certfile,
+ keyfile=keyfile)
+ server_hostname = self._host if ssl.HAS_SNI else None
+ self.sock = context.wrap_socket(self.sock,
+ server_hostname=server_hostname)
self.file = None
# RFC 3207:
# The client MUST discard any knowledge obtained from
@@ -883,6 +886,9 @@ if _have_ssl:
"exclusive")
self.keyfile = keyfile
self.certfile = certfile
+ if context is None:
+ context = ssl._create_stdlib_context(certfile=certfile,
+ keyfile=keyfile)
self.context = context
SMTP.__init__(self, host, port, local_hostname, timeout,
source_address)
@@ -892,10 +898,9 @@ if _have_ssl:
print('connect:', (host, port), file=stderr)
new_socket = socket.create_connection((host, port), timeout,
self.source_address)
- if self.context is not None:
- new_socket = self.context.wrap_socket(new_socket)
- else:
- new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile)
+ server_hostname = self._host if ssl.HAS_SNI else None
+ new_socket = self.context.wrap_socket(new_socket,
+ server_hostname=server_hostname)
return new_socket
__all__.append("SMTP_SSL")
@@ -937,7 +942,7 @@ class LMTP(SMTP):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.file = None
self.sock.connect(host)
- except socket.error:
+ except OSError:
if self.debuglevel > 0:
print('connect fail:', host, file=stderr)
if self.sock:
diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py
index 9f5dcc9..240e507 100644
--- a/Lib/sndhdr.py
+++ b/Lib/sndhdr.py
@@ -11,7 +11,7 @@ The return tuple contains the following items, in this order:
- number of bits/sample, or 'U' for U-LAW, or 'A' for A-LAW
If the file doesn't have a recognizable type, it returns None.
-If the file can't be opened, IOError is raised.
+If the file can't be opened, OSError is raised.
To compute the total time, divide the number of frames by the
sampling rate (a frame contains a sample for each channel).
@@ -137,14 +137,17 @@ tests.append(test_voc)
def test_wav(h, f):
+ import wave
# 'RIFF' <len> 'WAVE' 'fmt ' <len>
if not h.startswith(b'RIFF') or h[8:12] != b'WAVE' or h[12:16] != b'fmt ':
return None
- style = get_short_le(h[20:22])
- nchannels = get_short_le(h[22:24])
- rate = get_long_le(h[24:28])
- sample_bits = get_short_le(h[34:36])
- return 'wav', rate, nchannels, -1, sample_bits
+ f.seek(0)
+ try:
+ w = wave.openfp(f, 'r')
+ except (EOFError, wave.Error):
+ return None
+ return ('wav', w.getframerate(), w.getnchannels(),
+ w.getnframes(), 8*w.getsampwidth())
tests.append(test_wav)
@@ -230,7 +233,7 @@ def testall(list, recursive, toplevel):
sys.stdout.flush()
try:
print(what(filename))
- except IOError:
+ except OSError:
print('*** not found ***')
if __name__ == '__main__':
diff --git a/Lib/socket.py b/Lib/socket.py
index d4f1b65..6d67b3d 100644
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -48,6 +48,7 @@ import _socket
from _socket import *
import os, sys, io
+from enum import IntEnum
try:
import errno
@@ -60,6 +61,30 @@ EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', 11)
__all__ = ["getfqdn", "create_connection"]
__all__.extend(os._get_exports_list(_socket))
+# Set up the socket.AF_* socket.SOCK_* constants as members of IntEnums for
+# nicer string representations.
+# Note that _socket only knows about the integer values. The public interface
+# in this module understands the enums and translates them back from integers
+# where needed (e.g. .family property of a socket object).
+AddressFamily = IntEnum('AddressFamily',
+ {name: value for name, value in globals().items()
+ if name.isupper() and name.startswith('AF_')})
+globals().update(AddressFamily.__members__)
+
+SocketType = IntEnum('SocketType',
+ {name: value for name, value in globals().items()
+ if name.isupper() and name.startswith('SOCK_')})
+globals().update(SocketType.__members__)
+
+def _intenum_converter(value, enum_klass):
+ """Convert a numeric family value to an IntEnum member.
+
+ If it's not a known member, return the numeric value itself.
+ """
+ try:
+ return enum_klass(value)
+ except ValueError:
+ return value
_realsocket = socket
@@ -91,6 +116,10 @@ class socket(_socket.socket):
__slots__ = ["__weakref__", "_io_refs", "_closed"]
def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None):
+ # For user code address family and type values are IntEnum members, but
+ # for the underlying _socket.socket they're just integers. The
+ # constructor of _socket.socket converts the given argument to an
+ # integer automatically.
_socket.socket.__init__(self, family, type, proto, fileno)
self._io_refs = 0
self._closed = False
@@ -103,13 +132,32 @@ class socket(_socket.socket):
self.close()
def __repr__(self):
- """Wrap __repr__() to reveal the real class name."""
- s = _socket.socket.__repr__(self)
- if s.startswith("<socket object"):
- s = "<%s.%s%s%s" % (self.__class__.__module__,
- self.__class__.__name__,
- getattr(self, '_closed', False) and " [closed] " or "",
- s[7:])
+ """Wrap __repr__() to reveal the real class name and socket
+ address(es).
+ """
+ closed = getattr(self, '_closed', False)
+ s = "<%s.%s%s fd=%i, family=%s, type=%s, proto=%i" \
+ % (self.__class__.__module__,
+ self.__class__.__name__,
+ " [closed]" if closed else "",
+ self.fileno(),
+ self.family,
+ self.type,
+ self.proto)
+ if not closed:
+ try:
+ laddr = self.getsockname()
+ if laddr:
+ s += ", laddr=%s" % str(laddr)
+ except error:
+ pass
+ try:
+ raddr = self.getpeername()
+ if raddr:
+ s += ", raddr=%s" % str(raddr)
+ except error:
+ pass
+ s += '>'
return s
def __getstate__(self):
@@ -118,7 +166,8 @@ class socket(_socket.socket):
def dup(self):
"""dup() -> socket object
- Return a new socket object connected to the same system resource.
+ Duplicate the socket. Return a new socket object connected to the same
+ system resource. The new socket is non-inheritable.
"""
fd = dup(self.fileno())
sock = self.__class__(self.family, self.type, self.proto, fileno=fd)
@@ -210,6 +259,31 @@ class socket(_socket.socket):
self._closed = True
return super().detach()
+ @property
+ def family(self):
+ """Read-only access to the address family for this socket.
+ """
+ return _intenum_converter(super().family, AddressFamily)
+
+ @property
+ def type(self):
+ """Read-only access to the socket type.
+ """
+ return _intenum_converter(super().type, SocketType)
+
+ if os.name == 'nt':
+ def get_inheritable(self):
+ return os.get_handle_inheritable(self.fileno())
+ def set_inheritable(self, inheritable):
+ os.set_handle_inheritable(self.fileno(), inheritable)
+ else:
+ def get_inheritable(self):
+ return os.get_inheritable(self.fileno())
+ def set_inheritable(self, inheritable):
+ os.set_inheritable(self.fileno(), inheritable)
+ get_inheritable.__doc__ = "Get the inheritable flag of the socket"
+ set_inheritable.__doc__ = "Set the inheritable flag of the socket"
+
def fromfd(fd, family, type, proto=0):
""" fromfd(fd, family, type[, proto]) -> socket object
@@ -291,7 +365,7 @@ class SocketIO(io.RawIOBase):
self._checkClosed()
self._checkReadable()
if self._timeout_occurred:
- raise IOError("cannot read from timed out object")
+ raise OSError("cannot read from timed out object")
while True:
try:
return self._sock.recv_into(b)
@@ -435,3 +509,27 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
raise err
else:
raise error("getaddrinfo returns an empty list")
+
+def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
+ """Resolve host and port into list of address info entries.
+
+ Translate the host/port argument into a sequence of 5-tuples that contain
+ all the necessary arguments for creating a socket connected to that service.
+ host is a domain name, a string representation of an IPv4/v6 address or
+ None. port is a string service name such as 'http', a numeric port number or
+ None. By passing None as the value of host and port, you can pass NULL to
+ the underlying C API.
+
+ The family, type and proto arguments can be optionally specified in order to
+ narrow the list of addresses returned. Passing zero as a value for each of
+ these arguments selects the full range of results.
+ """
+ # We override this function since we want to translate the numeric family
+ # and socket type values to enum constants.
+ addrlist = []
+ for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
+ af, socktype, proto, canonname, sa = res
+ addrlist.append((_intenum_converter(af, AddressFamily),
+ _intenum_converter(socktype, SocketType),
+ proto, canonname, sa))
+ return addrlist
diff --git a/Lib/socketserver.py b/Lib/socketserver.py
index 8332fdf..46ee7c5 100644
--- a/Lib/socketserver.py
+++ b/Lib/socketserver.py
@@ -131,7 +131,6 @@ __version__ = "0.4"
import socket
import select
-import sys
import os
import errno
try:
@@ -299,7 +298,7 @@ class BaseServer:
"""
try:
request, client_address = self.get_request()
- except socket.error:
+ except OSError:
return
if self.verify_request(request, client_address):
try:
@@ -479,7 +478,7 @@ class TCPServer(BaseServer):
#explicitly shutdown. socket.close() merely releases
#the socket and waits for GC to perform the actual close.
request.shutdown(socket.SHUT_WR)
- except socket.error:
+ except OSError:
pass #some platforms may raise ENOTCONN here
self.close_request(request)
@@ -532,7 +531,7 @@ class ForkingMixIn:
# children.
try:
pid, status = os.waitpid(0, 0)
- except os.error:
+ except OSError:
pid = None
if pid not in self.active_children: continue
self.active_children.remove(pid)
@@ -545,7 +544,7 @@ class ForkingMixIn:
for child in self.active_children:
try:
pid, status = os.waitpid(child, os.WNOHANG)
- except os.error:
+ except OSError:
pid = None
if not pid: continue
try:
diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py
index b7ec1ad..04649fc 100644
--- a/Lib/sqlite3/test/dbapi.py
+++ b/Lib/sqlite3/test/dbapi.py
@@ -28,6 +28,9 @@ try:
except ImportError:
threading = None
+from test.support import TESTFN, unlink
+
+
class ModuleTests(unittest.TestCase):
def CheckAPILevel(self):
self.assertEqual(sqlite.apilevel, "2.0",
@@ -163,6 +166,21 @@ class ConnectionTests(unittest.TestCase):
with self.assertRaises(AttributeError):
self.cx.in_transaction = True
+ def CheckOpenUri(self):
+ if sqlite.sqlite_version_info < (3, 7, 7):
+ with self.assertRaises(sqlite.NotSupportedError):
+ sqlite.connect(':memory:', uri=True)
+ return
+ self.addCleanup(unlink, TESTFN)
+ with sqlite.connect(TESTFN) as cx:
+ cx.execute('create table test(id integer)')
+ with sqlite.connect('file:' + TESTFN, uri=True) as cx:
+ cx.execute('insert into test(id) values(0)')
+ with sqlite.connect('file:' + TESTFN + '?mode=ro', uri=True) as cx:
+ with self.assertRaises(sqlite.OperationalError):
+ cx.execute('insert into test(id) values(1)')
+
+
class CursorTests(unittest.TestCase):
def setUp(self):
self.cx = sqlite.connect(":memory:")
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
index b984a54..c6860b5 100644
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -10,9 +10,10 @@
"""Internal support module for sre"""
-import _sre, sys
+import _sre
import sre_parse
from sre_constants import *
+from _sre import MAXREPEAT
assert _sre.MAGIC == MAGIC, "SRE module mismatch"
@@ -64,13 +65,6 @@ def _compile(code, pattern, flags):
elif op in REPEATING_CODES:
if flags & SRE_FLAG_TEMPLATE:
raise error("internal: unsupported template operator")
- emit(OPCODES[REPEAT])
- skip = _len(code); emit(0)
- emit(av[0])
- emit(av[1])
- _compile(code, av[2], flags)
- emit(OPCODES[SUCCESS])
- code[skip] = _len(code) - skip
elif _simple(av) and op is not REPEAT:
if op is MAX_REPEAT:
emit(OPCODES[REPEAT_ONE])
@@ -207,158 +201,143 @@ def _compile_charset(charset, flags, code, fixup=None):
def _optimize_charset(charset, fixup):
# internal: optimize character set
out = []
- outappend = out.append
- charmap = [0]*256
- try:
- for op, av in charset:
- if op is NEGATE:
- outappend((op, av))
- elif op is LITERAL:
- charmap[fixup(av)] = 1
- elif op is RANGE:
- for i in range(fixup(av[0]), fixup(av[1])+1):
- charmap[i] = 1
- elif op is CATEGORY:
- # XXX: could append to charmap tail
- return charset # cannot compress
- except IndexError:
- # character set contains unicode characters
- return _optimize_unicode(charset, fixup)
+ tail = []
+ charmap = bytearray(256)
+ for op, av in charset:
+ while True:
+ try:
+ if op is LITERAL:
+ charmap[fixup(av)] = 1
+ elif op is RANGE:
+ for i in range(fixup(av[0]), fixup(av[1])+1):
+ charmap[i] = 1
+ elif op is NEGATE:
+ out.append((op, av))
+ else:
+ tail.append((op, av))
+ except IndexError:
+ if len(charmap) == 256:
+ # character set contains non-UCS1 character codes
+ charmap += b'\0' * 0xff00
+ continue
+ # character set contains non-BMP character codes
+ tail.append((op, av))
+ break
+
# compress character map
- i = p = n = 0
runs = []
- runsappend = runs.append
- for c in charmap:
- if c:
- if n == 0:
- p = i
- n = n + 1
- elif n:
- runsappend((p, n))
- n = 0
- i = i + 1
- if n:
- runsappend((p, n))
- if len(runs) <= 2:
+ q = 0
+ while True:
+ p = charmap.find(1, q)
+ if p < 0:
+ break
+ if len(runs) >= 2:
+ runs = None
+ break
+ q = charmap.find(0, p)
+ if q < 0:
+ runs.append((p, len(charmap)))
+ break
+ runs.append((p, q))
+ if runs is not None:
# use literal/range
- for p, n in runs:
- if n == 1:
- outappend((LITERAL, p))
+ for p, q in runs:
+ if q - p == 1:
+ out.append((LITERAL, p))
else:
- outappend((RANGE, (p, p+n-1)))
+ out.append((RANGE, (p, q - 1)))
+ out += tail
if len(out) < len(charset):
return out
- else:
- # use bitmap
+ return charset
+
+ # use bitmap
+ if len(charmap) == 256:
data = _mk_bitmap(charmap)
- outappend((CHARSET, data))
+ out.append((CHARSET, data))
+ out += tail
return out
- return charset
-def _mk_bitmap(bits):
- data = []
- dataappend = data.append
- if _sre.CODESIZE == 2:
- start = (1, 0)
- else:
- start = (1, 0)
- m, v = start
- for c in bits:
- if c:
- v = v + m
- m = m + m
- if m > MAXCODE:
- dataappend(v)
- m, v = start
- return data
-
-# To represent a big charset, first a bitmap of all characters in the
-# set is constructed. Then, this bitmap is sliced into chunks of 256
-# characters, duplicate chunks are eliminated, and each chunk is
-# given a number. In the compiled expression, the charset is
-# represented by a 32-bit word sequence, consisting of one word for
-# the number of different chunks, a sequence of 256 bytes (64 words)
-# of chunk numbers indexed by their original chunk position, and a
-# sequence of 256-bit chunks (8 words each).
-
-# Compression is normally good: in a typical charset, large ranges of
-# Unicode will be either completely excluded (e.g. if only cyrillic
-# letters are to be matched), or completely included (e.g. if large
-# subranges of Kanji match). These ranges will be represented by
-# chunks of all one-bits or all zero-bits.
-
-# Matching can be also done efficiently: the more significant byte of
-# the Unicode character is an index into the chunk number, and the
-# less significant byte is a bit index in the chunk (just like the
-# CHARSET matching).
-
-# The BIGCHARSET opcode still supports only subsets
-# of the basic multilingual plane; an efficient representation
-# for all of Unicode has not yet been developed. This means,
-# in particular, that negated charsets cannot be represented as
-# bigcharsets.
-
-def _optimize_unicode(charset, fixup):
- try:
- import array
- except ImportError:
- return charset
- charmap = [0]*65536
- negate = 0
- try:
- for op, av in charset:
- if op is NEGATE:
- negate = 1
- elif op is LITERAL:
- charmap[fixup(av)] = 1
- elif op is RANGE:
- for i in range(fixup(av[0]), fixup(av[1])+1):
- charmap[i] = 1
- elif op is CATEGORY:
- # XXX: could expand category
- return charset # cannot compress
- except IndexError:
- # non-BMP characters; XXX now they should work
- return charset
- if negate:
- if sys.maxunicode != 65535:
- # XXX: negation does not work with big charsets
- # XXX2: now they should work, but removing this will make the
- # charmap 17 times bigger
- return charset
- for i in range(65536):
- charmap[i] = not charmap[i]
+ # To represent a big charset, first a bitmap of all characters in the
+ # set is constructed. Then, this bitmap is sliced into chunks of 256
+ # characters, duplicate chunks are eliminated, and each chunk is
+ # given a number. In the compiled expression, the charset is
+ # represented by a 32-bit word sequence, consisting of one word for
+ # the number of different chunks, a sequence of 256 bytes (64 words)
+ # of chunk numbers indexed by their original chunk position, and a
+ # sequence of 256-bit chunks (8 words each).
+
+ # Compression is normally good: in a typical charset, large ranges of
+ # Unicode will be either completely excluded (e.g. if only cyrillic
+ # letters are to be matched), or completely included (e.g. if large
+ # subranges of Kanji match). These ranges will be represented by
+ # chunks of all one-bits or all zero-bits.
+
+ # Matching can be also done efficiently: the more significant byte of
+ # the Unicode character is an index into the chunk number, and the
+ # less significant byte is a bit index in the chunk (just like the
+ # CHARSET matching).
+
+ charmap = bytes(charmap) # should be hashable
comps = {}
- mapping = [0]*256
+ mapping = bytearray(256)
block = 0
- data = []
- for i in range(256):
- chunk = tuple(charmap[i*256:(i+1)*256])
- new = comps.setdefault(chunk, block)
- mapping[i] = new
- if new == block:
- block = block + 1
- data = data + _mk_bitmap(chunk)
- header = [block]
- if _sre.CODESIZE == 2:
- code = 'H'
- else:
- code = 'I'
- # Convert block indices to byte array of 256 bytes
- mapping = array.array('B', mapping).tobytes()
- # Convert byte array to word array
- mapping = array.array(code, mapping)
- assert mapping.itemsize == _sre.CODESIZE
- assert len(mapping) * mapping.itemsize == 256
- header = header + mapping.tolist()
- data[0:0] = header
- return [(BIGCHARSET, data)]
+ data = bytearray()
+ for i in range(0, 65536, 256):
+ chunk = charmap[i: i + 256]
+ if chunk in comps:
+ mapping[i // 256] = comps[chunk]
+ else:
+ mapping[i // 256] = comps[chunk] = block
+ block += 1
+ data += chunk
+ data = _mk_bitmap(data)
+ data[0:0] = [block] + _bytes_to_codes(mapping)
+ out.append((BIGCHARSET, data))
+ out += tail
+ return out
+
+_CODEBITS = _sre.CODESIZE * 8
+_BITS_TRANS = b'0' + b'1' * 255
+def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int):
+ s = bits.translate(_BITS_TRANS)[::-1]
+ return [_int(s[i - _CODEBITS: i], 2)
+ for i in range(len(s), 0, -_CODEBITS)]
+
+def _bytes_to_codes(b):
+ # Convert block indices to word array
+ import array
+ a = array.array('I', b)
+ assert a.itemsize == _sre.CODESIZE
+ assert len(a) * a.itemsize == len(b)
+ return a.tolist()
def _simple(av):
# check if av is a "simple" operator
lo, hi = av[2].getwidth()
return lo == hi == 1 and av[2][0][0] != SUBPATTERN
+def _generate_overlap_table(prefix):
+ """
+ Generate an overlap table for the following prefix.
+ An overlap table is a table of the same size as the prefix which
+ informs about the potential self-overlap for each index in the prefix:
+ - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...]
+ - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with
+ prefix[0:k]
+ """
+ table = [0] * len(prefix)
+ for i in range(1, len(prefix)):
+ idx = table[i - 1]
+ while prefix[i] != prefix[idx]:
+ if idx == 0:
+ table[i] = 0
+ break
+ idx = table[idx - 1]
+ else:
+ table[i] = idx + 1
+ return table
+
def _compile_info(code, pattern, flags):
# internal: compile an info block. in the current version,
# this contains min/max pattern width, and an optional literal
@@ -455,12 +434,7 @@ def _compile_info(code, pattern, flags):
emit(prefix_skip) # skip
code.extend(prefix)
# generate overlap table
- table = [-1] + ([0]*len(prefix))
- for i in range(len(prefix)):
- table[i+1] = table[i]+1
- while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]:
- table[i+1] = table[table[i+1]-1]+1
- code.extend(table[1:]) # don't store first entry
+ code.extend(_generate_overlap_table(prefix))
elif charset:
_compile_charset(charset, flags, code)
code[skip] = len(code) - skip
diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py
index 3fb5eac..23e3516 100644
--- a/Lib/sre_constants.py
+++ b/Lib/sre_constants.py
@@ -15,11 +15,7 @@
MAGIC = 20031017
-try:
- from _sre import MAXREPEAT
-except ImportError:
- import _sre
- MAXREPEAT = _sre.MAXREPEAT = 65535
+from _sre import MAXREPEAT
# SRE standard exception (access as sre.error)
# should this really be here?
@@ -254,6 +250,8 @@ if __name__ == "__main__":
f.write("#define SRE_FLAG_DOTALL %d\n" % SRE_FLAG_DOTALL)
f.write("#define SRE_FLAG_UNICODE %d\n" % SRE_FLAG_UNICODE)
f.write("#define SRE_FLAG_VERBOSE %d\n" % SRE_FLAG_VERBOSE)
+ f.write("#define SRE_FLAG_DEBUG %d\n" % SRE_FLAG_DEBUG)
+ f.write("#define SRE_FLAG_ASCII %d\n" % SRE_FLAG_ASCII)
f.write("#define SRE_INFO_PREFIX %d\n" % SRE_INFO_PREFIX)
f.write("#define SRE_INFO_LITERAL %d\n" % SRE_INFO_LITERAL)
diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py
index 8a77790..399e7b7 100644
--- a/Lib/sre_parse.py
+++ b/Lib/sre_parse.py
@@ -12,9 +12,8 @@
# XXX: show string offset and offending character for all errors
-import sys
-
from sre_constants import *
+from _sre import MAXREPEAT
SPECIAL_CHARS = ".\\[{()*+?^$|"
REPEAT_CHARS = "*+?{"
@@ -768,35 +767,33 @@ def parse_template(source, pattern):
# group references
s = Tokenizer(source)
sget = s.get
- p = []
- a = p.append
- def literal(literal, p=p, pappend=a):
- if p and p[-1][0] is LITERAL:
- p[-1] = LITERAL, p[-1][1] + literal
- else:
- pappend((LITERAL, literal))
- sep = source[:0]
- if isinstance(sep, str):
- makechar = chr
- else:
- makechar = chr
- while 1:
+ groups = []
+ literals = []
+ literal = []
+ lappend = literal.append
+ def addgroup(index):
+ if literal:
+ literals.append(''.join(literal))
+ del literal[:]
+ groups.append((len(literals), index))
+ literals.append(None)
+ while True:
this = sget()
if this is None:
break # end of replacement string
- if this and this[0] == "\\":
+ if this[0] == "\\":
# group
- c = this[1:2]
+ c = this[1]
if c == "g":
name = ""
if s.match("<"):
- while 1:
+ while True:
char = sget()
if char is None:
raise error("unterminated group name")
if char == ">":
break
- name = name + char
+ name += char
if not name:
raise error("missing group name")
try:
@@ -810,50 +807,38 @@ def parse_template(source, pattern):
index = pattern.groupindex[name]
except KeyError:
raise IndexError("unknown group name")
- a((MARK, index))
+ addgroup(index)
elif c == "0":
if s.next in OCTDIGITS:
- this = this + sget()
+ this += sget()
if s.next in OCTDIGITS:
- this = this + sget()
- literal(makechar(int(this[1:], 8) & 0xff))
+ this += sget()
+ lappend(chr(int(this[1:], 8) & 0xff))
elif c in DIGITS:
isoctal = False
if s.next in DIGITS:
- this = this + sget()
+ this += sget()
if (c in OCTDIGITS and this[2] in OCTDIGITS and
s.next in OCTDIGITS):
- this = this + sget()
+ this += sget()
isoctal = True
- literal(makechar(int(this[1:], 8) & 0xff))
+ lappend(chr(int(this[1:], 8) & 0xff))
if not isoctal:
- a((MARK, int(this[1:])))
+ addgroup(int(this[1:]))
else:
try:
- this = makechar(ESCAPES[this][1])
+ this = chr(ESCAPES[this][1])
except KeyError:
pass
- literal(this)
+ lappend(this)
else:
- literal(this)
- # convert template to groups and literals lists
- i = 0
- groups = []
- groupsappend = groups.append
- literals = [None] * len(p)
- if isinstance(source, str):
- encode = lambda x: x
- else:
+ lappend(this)
+ if literal:
+ literals.append(''.join(literal))
+ if not isinstance(source, str):
# The tokenizer implicitly decodes bytes objects as latin-1, we must
# therefore re-encode the final representation.
- encode = lambda x: x.encode('latin-1')
- for c, s in p:
- if c is MARK:
- groupsappend((i, s))
- # literal[i] is already None
- else:
- literals[i] = encode(s)
- i = i + 1
+ literals = [None if s is None else s.encode('latin-1') for s in literals]
return groups, literals
def expand_template(template, match):
diff --git a/Lib/ssl.py b/Lib/ssl.py
index cd8d6b4..d3c18ed 100644
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -52,10 +52,47 @@ PROTOCOL_SSLv2
PROTOCOL_SSLv3
PROTOCOL_SSLv23
PROTOCOL_TLSv1
+PROTOCOL_TLSv1_1
+PROTOCOL_TLSv1_2
+
+The following constants identify various SSL alert message descriptions as per
+http://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-6
+
+ALERT_DESCRIPTION_CLOSE_NOTIFY
+ALERT_DESCRIPTION_UNEXPECTED_MESSAGE
+ALERT_DESCRIPTION_BAD_RECORD_MAC
+ALERT_DESCRIPTION_RECORD_OVERFLOW
+ALERT_DESCRIPTION_DECOMPRESSION_FAILURE
+ALERT_DESCRIPTION_HANDSHAKE_FAILURE
+ALERT_DESCRIPTION_BAD_CERTIFICATE
+ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE
+ALERT_DESCRIPTION_CERTIFICATE_REVOKED
+ALERT_DESCRIPTION_CERTIFICATE_EXPIRED
+ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN
+ALERT_DESCRIPTION_ILLEGAL_PARAMETER
+ALERT_DESCRIPTION_UNKNOWN_CA
+ALERT_DESCRIPTION_ACCESS_DENIED
+ALERT_DESCRIPTION_DECODE_ERROR
+ALERT_DESCRIPTION_DECRYPT_ERROR
+ALERT_DESCRIPTION_PROTOCOL_VERSION
+ALERT_DESCRIPTION_INSUFFICIENT_SECURITY
+ALERT_DESCRIPTION_INTERNAL_ERROR
+ALERT_DESCRIPTION_USER_CANCELLED
+ALERT_DESCRIPTION_NO_RENEGOTIATION
+ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION
+ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+ALERT_DESCRIPTION_UNRECOGNIZED_NAME
+ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE
+ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE
+ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY
"""
import textwrap
import re
+import sys
+import os
+from collections import namedtuple
+from enum import Enum as _Enum
import _ssl # if we can't import it, let the error propagate
@@ -66,35 +103,26 @@ from _ssl import (
SSLSyscallError, SSLEOFError,
)
from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import (
- OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1,
- OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE
- )
-try:
- from _ssl import OP_NO_COMPRESSION
-except ImportError:
- pass
-try:
- from _ssl import OP_SINGLE_ECDH_USE
-except ImportError:
- pass
+from _ssl import (VERIFY_DEFAULT, VERIFY_CRL_CHECK_LEAF, VERIFY_CRL_CHECK_CHAIN,
+ VERIFY_X509_STRICT)
+from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj
from _ssl import RAND_status, RAND_egd, RAND_add, RAND_bytes, RAND_pseudo_bytes
-from _ssl import (
- SSL_ERROR_ZERO_RETURN,
- SSL_ERROR_WANT_READ,
- SSL_ERROR_WANT_WRITE,
- SSL_ERROR_WANT_X509_LOOKUP,
- SSL_ERROR_SYSCALL,
- SSL_ERROR_SSL,
- SSL_ERROR_WANT_CONNECT,
- SSL_ERROR_EOF,
- SSL_ERROR_INVALID_ERROR_CODE,
- )
+
+def _import_symbols(prefix):
+ for n in dir(_ssl):
+ if n.startswith(prefix):
+ globals()[n] = getattr(_ssl, n)
+
+_import_symbols('OP_')
+_import_symbols('ALERT_DESCRIPTION_')
+_import_symbols('SSL_ERROR_')
+
from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN
-from _ssl import (PROTOCOL_SSLv3, PROTOCOL_SSLv23,
- PROTOCOL_TLSv1)
+
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
from _ssl import _OPENSSL_API_VERSION
+
_PROTOCOL_NAMES = {
PROTOCOL_TLSv1: "TLSv1",
PROTOCOL_SSLv23: "SSLv23",
@@ -108,14 +136,25 @@ except ImportError:
else:
_PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
-from socket import getnameinfo as _getnameinfo
-from socket import error as socket_error
+try:
+ from _ssl import PROTOCOL_TLSv1_1, PROTOCOL_TLSv1_2
+except ImportError:
+ pass
+else:
+ _PROTOCOL_NAMES[PROTOCOL_TLSv1_1] = "TLSv1.1"
+ _PROTOCOL_NAMES[PROTOCOL_TLSv1_2] = "TLSv1.2"
+
+if sys.platform == "win32":
+ from _ssl import enum_certificates, enum_crls
+
from socket import socket, AF_INET, SOCK_STREAM, create_connection
from socket import SOL_SOCKET, SO_TYPE
import base64 # for DER-to-PEM translation
-import traceback
import errno
+
+socket_error = OSError # keep that public name in module namespace
+
if _ssl.HAS_TLS_UNIQUE:
CHANNEL_BINDING_TYPES = ['tls-unique']
else:
@@ -123,7 +162,37 @@ else:
# Disable weak or insecure ciphers by default
# (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL')
-_DEFAULT_CIPHERS = 'DEFAULT:!aNULL:!eNULL:!LOW:!EXPORT:!SSLv2'
+# Enable a better set of ciphers by default
+# This list has been explicitly chosen to:
+# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
+# * Prefer ECDHE over DHE for better performance
+# * Prefer any AES-GCM over any AES-CBC for better performance and security
+# * Then Use HIGH cipher suites as a fallback
+# * Then Use 3DES as fallback which is secure but slow
+# * Finally use RC4 as a fallback which is problematic but needed for
+# compatibility some times.
+# * Disable NULL authentication, NULL encryption, and MD5 MACs for security
+# reasons
+_DEFAULT_CIPHERS = (
+ 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
+ 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:ECDH+RC4:'
+ 'DH+RC4:RSA+RC4:!aNULL:!eNULL:!MD5'
+)
+
+# Restricted and more secure ciphers for the server side
+# This list has been explicitly chosen to:
+# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE)
+# * Prefer ECDHE over DHE for better performance
+# * Prefer any AES-GCM over any AES-CBC for better performance and security
+# * Then Use HIGH cipher suites as a fallback
+# * Then Use 3DES as fallback which is secure but slow
+# * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, and RC4 for
+# security reasons
+_RESTRICTED_SERVER_CIPHERS = (
+ 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
+ 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
+ '!eNULL:!MD5:!DSS:!RC4'
+)
class CertificateError(ValueError):
@@ -188,7 +257,9 @@ def match_hostname(cert, hostname):
returns nothing.
"""
if not cert:
- raise ValueError("empty or no certificate")
+ raise ValueError("empty or no certificate, match_hostname needs a "
+ "SSL socket or SSL context with either "
+ "CERT_OPTIONAL or CERT_REQUIRED")
dnsnames = []
san = cert.get('subjectAltName', ())
for key, value in san:
@@ -220,11 +291,58 @@ def match_hostname(cert, hostname):
"subjectAltName fields were found")
+DefaultVerifyPaths = namedtuple("DefaultVerifyPaths",
+ "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env "
+ "openssl_capath")
+
+def get_default_verify_paths():
+ """Return paths to default cafile and capath.
+ """
+ parts = _ssl.get_default_verify_paths()
+
+ # environment vars shadow paths
+ cafile = os.environ.get(parts[0], parts[1])
+ capath = os.environ.get(parts[2], parts[3])
+
+ return DefaultVerifyPaths(cafile if os.path.isfile(cafile) else None,
+ capath if os.path.isdir(capath) else None,
+ *parts)
+
+
+class _ASN1Object(namedtuple("_ASN1Object", "nid shortname longname oid")):
+ """ASN.1 object identifier lookup
+ """
+ __slots__ = ()
+
+ def __new__(cls, oid):
+ return super().__new__(cls, *_txt2obj(oid, name=False))
+
+ @classmethod
+ def fromnid(cls, nid):
+ """Create _ASN1Object from OpenSSL numeric ID
+ """
+ return super().__new__(cls, *_nid2obj(nid))
+
+ @classmethod
+ def fromname(cls, name):
+ """Create _ASN1Object from short name, long name or OID
+ """
+ return super().__new__(cls, *_txt2obj(name, name=True))
+
+
+class Purpose(_ASN1Object, _Enum):
+ """SSLContext purpose flags with X509v3 Extended Key Usage objects
+ """
+ SERVER_AUTH = '1.3.6.1.5.5.7.3.1'
+ CLIENT_AUTH = '1.3.6.1.5.5.7.3.2'
+
+
class SSLContext(_SSLContext):
"""An SSLContext holds various SSL-related configuration options and
data, such as certificates and possibly a private key."""
- __slots__ = ('protocol',)
+ __slots__ = ('protocol', '__weakref__')
+ _windows_cert_stores = ("CA", "ROOT")
def __new__(cls, protocol, *args, **kwargs):
self = _SSLContext.__new__(cls, protocol)
@@ -256,6 +374,112 @@ class SSLContext(_SSLContext):
self._set_npn_protocols(protos)
+ def _load_windows_store_certs(self, storename, purpose):
+ certs = bytearray()
+ for cert, encoding, trust in enum_certificates(storename):
+ # CA certs are never PKCS#7 encoded
+ if encoding == "x509_asn":
+ if trust is True or purpose.oid in trust:
+ certs.extend(cert)
+ self.load_verify_locations(cadata=certs)
+ return certs
+
+ def load_default_certs(self, purpose=Purpose.SERVER_AUTH):
+ if not isinstance(purpose, _ASN1Object):
+ raise TypeError(purpose)
+ if sys.platform == "win32":
+ for storename in self._windows_cert_stores:
+ self._load_windows_store_certs(storename, purpose)
+ else:
+ self.set_default_verify_paths()
+
+
+def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None,
+ capath=None, cadata=None):
+ """Create a SSLContext object with default settings.
+
+ NOTE: The protocol and settings may change anytime without prior
+ deprecation. The values represent a fair balance between maximum
+ compatibility and security.
+ """
+ if not isinstance(purpose, _ASN1Object):
+ raise TypeError(purpose)
+
+ context = SSLContext(PROTOCOL_SSLv23)
+
+ # SSLv2 considered harmful.
+ context.options |= OP_NO_SSLv2
+
+ # SSLv3 has problematic security and is only required for really old
+ # clients such as IE6 on Windows XP
+ context.options |= OP_NO_SSLv3
+
+ # disable compression to prevent CRIME attacks (OpenSSL 1.0+)
+ context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
+
+ if purpose == Purpose.SERVER_AUTH:
+ # verify certs and host name in client mode
+ context.verify_mode = CERT_REQUIRED
+ context.check_hostname = True
+ elif purpose == Purpose.CLIENT_AUTH:
+ # Prefer the server's ciphers by default so that we get stronger
+ # encryption
+ context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0)
+
+ # Use single use keys in order to improve forward secrecy
+ context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0)
+ context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0)
+
+ # disallow ciphers with known vulnerabilities
+ context.set_ciphers(_RESTRICTED_SERVER_CIPHERS)
+
+ if cafile or capath or cadata:
+ context.load_verify_locations(cafile, capath, cadata)
+ elif context.verify_mode != CERT_NONE:
+ # no explicit cafile, capath or cadata but the verify mode is
+ # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
+ # root CA certificates for the given purpose. This may fail silently.
+ context.load_default_certs(purpose)
+ return context
+
+
+def _create_stdlib_context(protocol=PROTOCOL_SSLv23, *, cert_reqs=None,
+ check_hostname=False, purpose=Purpose.SERVER_AUTH,
+ certfile=None, keyfile=None,
+ cafile=None, capath=None, cadata=None):
+ """Create a SSLContext object for Python stdlib modules
+
+ All Python stdlib modules shall use this function to create SSLContext
+ objects in order to keep common settings in one place. The configuration
+ is less restrict than create_default_context()'s to increase backward
+ compatibility.
+ """
+ if not isinstance(purpose, _ASN1Object):
+ raise TypeError(purpose)
+
+ context = SSLContext(protocol)
+ # SSLv2 considered harmful.
+ context.options |= OP_NO_SSLv2
+
+ if cert_reqs is not None:
+ context.verify_mode = cert_reqs
+ context.check_hostname = check_hostname
+
+ if keyfile and not certfile:
+ raise ValueError("certfile must be specified")
+ if certfile or keyfile:
+ context.load_cert_chain(certfile, keyfile)
+
+ # load CA root certs
+ if cafile or capath or cadata:
+ context.load_verify_locations(cafile, capath, cadata)
+ elif context.verify_mode != CERT_NONE:
+ # no explicit cafile, capath or cadata but the verify mode is
+ # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system
+ # root CA certificates for the given purpose. This may fail silently.
+ context.load_default_certs(purpose)
+
+ return context
class SSLSocket(socket):
"""This class implements a subtype of socket.socket that wraps
@@ -272,7 +496,7 @@ class SSLSocket(socket):
_context=None):
if _context:
- self.context = _context
+ self._context = _context
else:
if server_side and not certfile:
raise ValueError("certfile must be specified for server-side "
@@ -281,16 +505,16 @@ class SSLSocket(socket):
raise ValueError("certfile must be specified")
if certfile and not keyfile:
keyfile = certfile
- self.context = SSLContext(ssl_version)
- self.context.verify_mode = cert_reqs
+ self._context = SSLContext(ssl_version)
+ self._context.verify_mode = cert_reqs
if ca_certs:
- self.context.load_verify_locations(ca_certs)
+ self._context.load_verify_locations(ca_certs)
if certfile:
- self.context.load_cert_chain(certfile, keyfile)
+ self._context.load_cert_chain(certfile, keyfile)
if npn_protocols:
- self.context.set_npn_protocols(npn_protocols)
+ self._context.set_npn_protocols(npn_protocols)
if ciphers:
- self.context.set_ciphers(ciphers)
+ self._context.set_ciphers(ciphers)
self.keyfile = keyfile
self.certfile = certfile
self.cert_reqs = cert_reqs
@@ -304,11 +528,17 @@ class SSLSocket(socket):
if server_side and server_hostname:
raise ValueError("server_hostname can only be specified "
"in client mode")
+ if self._context.check_hostname and not server_hostname:
+ if HAS_SNI:
+ raise ValueError("check_hostname requires server_hostname")
+ else:
+ raise ValueError("check_hostname requires server_hostname, "
+ "but it's not supported by your OpenSSL "
+ "library")
self.server_side = server_side
self.server_hostname = server_hostname
self.do_handshake_on_connect = do_handshake_on_connect
self.suppress_ragged_eofs = suppress_ragged_eofs
- connected = False
if sock is not None:
socket.__init__(self,
family=sock.family,
@@ -316,27 +546,29 @@ class SSLSocket(socket):
proto=sock.proto,
fileno=sock.fileno())
self.settimeout(sock.gettimeout())
- # see if it's connected
- try:
- sock.getpeername()
- except socket_error as e:
- if e.errno != errno.ENOTCONN:
- raise
- else:
- connected = True
sock.detach()
elif fileno is not None:
socket.__init__(self, fileno=fileno)
else:
socket.__init__(self, family=family, type=type, proto=proto)
+ # See if we are connected
+ try:
+ self.getpeername()
+ except OSError as e:
+ if e.errno != errno.ENOTCONN:
+ raise
+ connected = False
+ else:
+ connected = True
+
self._closed = False
self._sslobj = None
self._connected = connected
if connected:
# create the SSL object
try:
- self._sslobj = self.context._wrap_socket(self, server_side,
+ self._sslobj = self._context._wrap_socket(self, server_side,
server_hostname)
if do_handshake_on_connect:
timeout = self.gettimeout()
@@ -345,9 +577,18 @@ class SSLSocket(socket):
raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
self.do_handshake()
- except socket_error as x:
+ except (OSError, ValueError):
self.close()
- raise x
+ raise
+
+ @property
+ def context(self):
+ return self._context
+
+ @context.setter
+ def context(self, ctx):
+ self._context = ctx
+ self._sslobj.context = ctx
def dup(self):
raise NotImplemented("Can't dup() %s instances" %
@@ -357,11 +598,21 @@ class SSLSocket(socket):
# raise an exception here if you wish to check for spurious closes
pass
+ def _check_connected(self):
+ if not self._connected:
+ # getpeername() will raise ENOTCONN if the socket is really
+ # not connected; note that we can be connected even without
+ # _connected being set, e.g. if connect() first returned
+ # EAGAIN.
+ self.getpeername()
+
def read(self, len=0, buffer=None):
"""Read up to LEN bytes and return them.
Return zero-length string on EOF."""
self._checkClosed()
+ if not self._sslobj:
+ raise ValueError("Read on closed or unwrapped SSL socket.")
try:
if buffer is not None:
v = self._sslobj.read(len, buffer)
@@ -382,6 +633,8 @@ class SSLSocket(socket):
number of bytes of DATA actually transmitted."""
self._checkClosed()
+ if not self._sslobj:
+ raise ValueError("Write on closed or unwrapped SSL socket.")
return self._sslobj.write(data)
def getpeercert(self, binary_form=False):
@@ -391,6 +644,7 @@ class SSLSocket(socket):
certificate was provided, but not validated."""
self._checkClosed()
+ self._check_connected()
return self._sslobj.peer_certificate(binary_form)
def selected_npn_protocol(self):
@@ -421,18 +675,17 @@ class SSLSocket(socket):
raise ValueError(
"non-zero flags not allowed in calls to send() on %s" %
self.__class__)
- while True:
- try:
- v = self._sslobj.write(data)
- except SSLError as x:
- if x.args[0] == SSL_ERROR_WANT_READ:
- return 0
- elif x.args[0] == SSL_ERROR_WANT_WRITE:
- return 0
- else:
- raise
+ try:
+ v = self._sslobj.write(data)
+ except SSLError as x:
+ if x.args[0] == SSL_ERROR_WANT_READ:
+ return 0
+ elif x.args[0] == SSL_ERROR_WANT_WRITE:
+ return 0
else:
- return v
+ raise
+ else:
+ return v
else:
return socket.send(self, data, flags)
@@ -540,12 +793,11 @@ class SSLSocket(socket):
def _real_close(self):
self._sslobj = None
- # self._closed = True
socket._real_close(self)
def do_handshake(self, block=False):
"""Perform a TLS/SSL handshake."""
-
+ self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
@@ -554,6 +806,12 @@ class SSLSocket(socket):
finally:
self.settimeout(timeout)
+ if self.context.check_hostname:
+ if not self.server_hostname:
+ raise ValueError("check_hostname needs server_hostname "
+ "argument")
+ match_hostname(self.getpeercert(), self.server_hostname)
+
def _real_connect(self, addr, connect_ex):
if self.server_side:
raise ValueError("can't connect in server-side mode")
@@ -569,11 +827,11 @@ class SSLSocket(socket):
rc = None
socket.connect(self, addr)
if not rc:
+ self._connected = True
if self.do_handshake_on_connect:
self.do_handshake()
- self._connected = True
return rc
- except socket_error:
+ except (OSError, ValueError):
self._sslobj = None
raise
@@ -671,15 +929,16 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None):
If 'ssl_version' is specified, use it in the connection attempt."""
host, port = addr
- if (ca_certs is not None):
+ if ca_certs is not None:
cert_reqs = CERT_REQUIRED
else:
cert_reqs = CERT_NONE
- s = create_connection(addr)
- s = wrap_socket(s, ssl_version=ssl_version,
- cert_reqs=cert_reqs, ca_certs=ca_certs)
- dercert = s.getpeercert(True)
- s.close()
+ context = _create_stdlib_context(ssl_version,
+ cert_reqs=cert_reqs,
+ cafile=ca_certs)
+ with create_connection(addr) as sock:
+ with context.wrap_socket(sock) as sslsock:
+ dercert = sslsock.getpeercert(True)
return DER_cert_to_PEM_cert(dercert)
def get_protocol_name(protocol_code):
diff --git a/Lib/stat.py b/Lib/stat.py
index 704adfe..3eecc3e 100644
--- a/Lib/stat.py
+++ b/Lib/stat.py
@@ -147,3 +147,9 @@ def filemode(mode):
else:
perm.append("-")
return "".join(perm)
+
+# If available, use C implementation
+try:
+ from _stat import *
+except ImportError:
+ pass
diff --git a/Lib/statistics.py b/Lib/statistics.py
new file mode 100644
index 0000000..25a26d4
--- /dev/null
+++ b/Lib/statistics.py
@@ -0,0 +1,595 @@
+## Module statistics.py
+##
+## Copyright (c) 2013 Steven D'Aprano <steve+python@pearwood.info>.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+
+
+"""
+Basic statistics module.
+
+This module provides functions for calculating statistics of data, including
+averages, variance, and standard deviation.
+
+Calculating averages
+--------------------
+
+================== =============================================
+Function Description
+================== =============================================
+mean Arithmetic mean (average) of data.
+median Median (middle value) of data.
+median_low Low median of data.
+median_high High median of data.
+median_grouped Median, or 50th percentile, of grouped data.
+mode Mode (most common value) of data.
+================== =============================================
+
+Calculate the arithmetic mean ("the average") of data:
+
+>>> mean([-1.0, 2.5, 3.25, 5.75])
+2.625
+
+
+Calculate the standard median of discrete data:
+
+>>> median([2, 3, 4, 5])
+3.5
+
+
+Calculate the median, or 50th percentile, of data grouped into class intervals
+centred on the data values provided. E.g. if your data points are rounded to
+the nearest whole number:
+
+>>> median_grouped([2, 2, 3, 3, 3, 4]) #doctest: +ELLIPSIS
+2.8333333333...
+
+This should be interpreted in this way: you have two data points in the class
+interval 1.5-2.5, three data points in the class interval 2.5-3.5, and one in
+the class interval 3.5-4.5. The median of these data points is 2.8333...
+
+
+Calculating variability or spread
+---------------------------------
+
+================== =============================================
+Function Description
+================== =============================================
+pvariance Population variance of data.
+variance Sample variance of data.
+pstdev Population standard deviation of data.
+stdev Sample standard deviation of data.
+================== =============================================
+
+Calculate the standard deviation of sample data:
+
+>>> stdev([2.5, 3.25, 5.5, 11.25, 11.75]) #doctest: +ELLIPSIS
+4.38961843444...
+
+If you have previously calculated the mean, you can pass it as the optional
+second argument to the four "spread" functions to avoid recalculating it:
+
+>>> data = [1, 2, 2, 4, 4, 4, 5, 6]
+>>> mu = mean(data)
+>>> pvariance(data, mu)
+2.5
+
+
+Exceptions
+----------
+
+A single exception is defined: StatisticsError is a subclass of ValueError.
+
+"""
+
+__all__ = [ 'StatisticsError',
+ 'pstdev', 'pvariance', 'stdev', 'variance',
+ 'median', 'median_low', 'median_high', 'median_grouped',
+ 'mean', 'mode',
+ ]
+
+
+import collections
+import math
+
+from fractions import Fraction
+from decimal import Decimal
+
+
+# === Exceptions ===
+
+class StatisticsError(ValueError):
+ pass
+
+
+# === Private utilities ===
+
+def _sum(data, start=0):
+ """_sum(data [, start]) -> value
+
+ Return a high-precision sum of the given numeric data. If optional
+ argument ``start`` is given, it is added to the total. If ``data`` is
+ empty, ``start`` (defaulting to 0) is returned.
+
+
+ Examples
+ --------
+
+ >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75)
+ 11.0
+
+ Some sources of round-off error will be avoided:
+
+ >>> _sum([1e50, 1, -1e50] * 1000) # Built-in sum returns zero.
+ 1000.0
+
+ Fractions and Decimals are also supported:
+
+ >>> from fractions import Fraction as F
+ >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)])
+ Fraction(63, 20)
+
+ >>> from decimal import Decimal as D
+ >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")]
+ >>> _sum(data)
+ Decimal('0.6963')
+
+ Mixed types are currently treated as an error, except that int is
+ allowed.
+ """
+ # We fail as soon as we reach a value that is not an int or the type of
+ # the first value which is not an int. E.g. _sum([int, int, float, int])
+ # is okay, but sum([int, int, float, Fraction]) is not.
+ allowed_types = set([int, type(start)])
+ n, d = _exact_ratio(start)
+ partials = {d: n} # map {denominator: sum of numerators}
+ # Micro-optimizations.
+ exact_ratio = _exact_ratio
+ partials_get = partials.get
+ # Add numerators for each denominator.
+ for x in data:
+ _check_type(type(x), allowed_types)
+ n, d = exact_ratio(x)
+ partials[d] = partials_get(d, 0) + n
+ # Find the expected result type. If allowed_types has only one item, it
+ # will be int; if it has two, use the one which isn't int.
+ assert len(allowed_types) in (1, 2)
+ if len(allowed_types) == 1:
+ assert allowed_types.pop() is int
+ T = int
+ else:
+ T = (allowed_types - set([int])).pop()
+ if None in partials:
+ assert issubclass(T, (float, Decimal))
+ assert not math.isfinite(partials[None])
+ return T(partials[None])
+ total = Fraction()
+ for d, n in sorted(partials.items()):
+ total += Fraction(n, d)
+ if issubclass(T, int):
+ assert total.denominator == 1
+ return T(total.numerator)
+ if issubclass(T, Decimal):
+ return T(total.numerator)/total.denominator
+ return T(total)
+
+
+def _check_type(T, allowed):
+ if T not in allowed:
+ if len(allowed) == 1:
+ allowed.add(T)
+ else:
+ types = ', '.join([t.__name__ for t in allowed] + [T.__name__])
+ raise TypeError("unsupported mixed types: %s" % types)
+
+
+def _exact_ratio(x):
+ """Convert Real number x exactly to (numerator, denominator) pair.
+
+ >>> _exact_ratio(0.25)
+ (1, 4)
+
+ x is expected to be an int, Fraction, Decimal or float.
+ """
+ try:
+ try:
+ # int, Fraction
+ return (x.numerator, x.denominator)
+ except AttributeError:
+ # float
+ try:
+ return x.as_integer_ratio()
+ except AttributeError:
+ # Decimal
+ try:
+ return _decimal_to_ratio(x)
+ except AttributeError:
+ msg = "can't convert type '{}' to numerator/denominator"
+ raise TypeError(msg.format(type(x).__name__)) from None
+ except (OverflowError, ValueError):
+ # INF or NAN
+ if __debug__:
+ # Decimal signalling NANs cannot be converted to float :-(
+ if isinstance(x, Decimal):
+ assert not x.is_finite()
+ else:
+ assert not math.isfinite(x)
+ return (x, None)
+
+
+# FIXME This is faster than Fraction.from_decimal, but still too slow.
+def _decimal_to_ratio(d):
+ """Convert Decimal d to exact integer ratio (numerator, denominator).
+
+ >>> from decimal import Decimal
+ >>> _decimal_to_ratio(Decimal("2.6"))
+ (26, 10)
+
+ """
+ sign, digits, exp = d.as_tuple()
+ if exp in ('F', 'n', 'N'): # INF, NAN, sNAN
+ assert not d.is_finite()
+ raise ValueError
+ num = 0
+ for digit in digits:
+ num = num*10 + digit
+ if exp < 0:
+ den = 10**-exp
+ else:
+ num *= 10**exp
+ den = 1
+ if sign:
+ num = -num
+ return (num, den)
+
+
+def _counts(data):
+ # Generate a table of sorted (value, frequency) pairs.
+ table = collections.Counter(iter(data)).most_common()
+ if not table:
+ return table
+ # Extract the values with the highest frequency.
+ maxfreq = table[0][1]
+ for i in range(1, len(table)):
+ if table[i][1] != maxfreq:
+ table = table[:i]
+ break
+ return table
+
+
+# === Measures of central tendency (averages) ===
+
+def mean(data):
+ """Return the sample arithmetic mean of data.
+
+ >>> mean([1, 2, 3, 4, 4])
+ 2.8
+
+ >>> from fractions import Fraction as F
+ >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)])
+ Fraction(13, 21)
+
+ >>> from decimal import Decimal as D
+ >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")])
+ Decimal('0.5625')
+
+ If ``data`` is empty, StatisticsError will be raised.
+ """
+ if iter(data) is data:
+ data = list(data)
+ n = len(data)
+ if n < 1:
+ raise StatisticsError('mean requires at least one data point')
+ return _sum(data)/n
+
+
+# FIXME: investigate ways to calculate medians without sorting? Quickselect?
+def median(data):
+ """Return the median (middle value) of numeric data.
+
+ When the number of data points is odd, return the middle data point.
+ When the number of data points is even, the median is interpolated by
+ taking the average of the two middle values:
+
+ >>> median([1, 3, 5])
+ 3
+ >>> median([1, 3, 5, 7])
+ 4.0
+
+ """
+ data = sorted(data)
+ n = len(data)
+ if n == 0:
+ raise StatisticsError("no median for empty data")
+ if n%2 == 1:
+ return data[n//2]
+ else:
+ i = n//2
+ return (data[i - 1] + data[i])/2
+
+
+def median_low(data):
+ """Return the low median of numeric data.
+
+ When the number of data points is odd, the middle value is returned.
+ When it is even, the smaller of the two middle values is returned.
+
+ >>> median_low([1, 3, 5])
+ 3
+ >>> median_low([1, 3, 5, 7])
+ 3
+
+ """
+ data = sorted(data)
+ n = len(data)
+ if n == 0:
+ raise StatisticsError("no median for empty data")
+ if n%2 == 1:
+ return data[n//2]
+ else:
+ return data[n//2 - 1]
+
+
+def median_high(data):
+ """Return the high median of data.
+
+ When the number of data points is odd, the middle value is returned.
+ When it is even, the larger of the two middle values is returned.
+
+ >>> median_high([1, 3, 5])
+ 3
+ >>> median_high([1, 3, 5, 7])
+ 5
+
+ """
+ data = sorted(data)
+ n = len(data)
+ if n == 0:
+ raise StatisticsError("no median for empty data")
+ return data[n//2]
+
+
+def median_grouped(data, interval=1):
+ """"Return the 50th percentile (median) of grouped continuous data.
+
+ >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5])
+ 3.7
+ >>> median_grouped([52, 52, 53, 54])
+ 52.5
+
+ This calculates the median as the 50th percentile, and should be
+ used when your data is continuous and grouped. In the above example,
+ the values 1, 2, 3, etc. actually represent the midpoint of classes
+ 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in
+ class 3.5-4.5, and interpolation is used to estimate it.
+
+ Optional argument ``interval`` represents the class interval, and
+ defaults to 1. Changing the class interval naturally will change the
+ interpolated 50th percentile value:
+
+ >>> median_grouped([1, 3, 3, 5, 7], interval=1)
+ 3.25
+ >>> median_grouped([1, 3, 3, 5, 7], interval=2)
+ 3.5
+
+ This function does not check whether the data points are at least
+ ``interval`` apart.
+ """
+ data = sorted(data)
+ n = len(data)
+ if n == 0:
+ raise StatisticsError("no median for empty data")
+ elif n == 1:
+ return data[0]
+ # Find the value at the midpoint. Remember this corresponds to the
+ # centre of the class interval.
+ x = data[n//2]
+ for obj in (x, interval):
+ if isinstance(obj, (str, bytes)):
+ raise TypeError('expected number but got %r' % obj)
+ try:
+ L = x - interval/2 # The lower limit of the median interval.
+ except TypeError:
+ # Mixed type. For now we just coerce to float.
+ L = float(x) - float(interval)/2
+ cf = data.index(x) # Number of values below the median interval.
+ # FIXME The following line could be more efficient for big lists.
+ f = data.count(x) # Number of data points in the median interval.
+ return L + interval*(n/2 - cf)/f
+
+
+def mode(data):
+ """Return the most common data point from discrete or nominal data.
+
+ ``mode`` assumes discrete data, and returns a single value. This is the
+ standard treatment of the mode as commonly taught in schools:
+
+ >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
+ 3
+
+ This also works with nominal (non-numeric) data:
+
+ >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
+ 'red'
+
+ If there is not exactly one most common value, ``mode`` will raise
+ StatisticsError.
+ """
+ # Generate a table of sorted (value, frequency) pairs.
+ table = _counts(data)
+ if len(table) == 1:
+ return table[0][0]
+ elif table:
+ raise StatisticsError(
+ 'no unique mode; found %d equally common values' % len(table)
+ )
+ else:
+ raise StatisticsError('no mode for empty data')
+
+
+# === Measures of spread ===
+
+# See http://mathworld.wolfram.com/Variance.html
+# http://mathworld.wolfram.com/SampleVariance.html
+# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+#
+# Under no circumstances use the so-called "computational formula for
+# variance", as that is only suitable for hand calculations with a small
+# amount of low-precision data. It has terrible numeric properties.
+#
+# See a comparison of three computational methods here:
+# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/
+
+def _ss(data, c=None):
+ """Return sum of square deviations of sequence data.
+
+ If ``c`` is None, the mean is calculated in one pass, and the deviations
+ from the mean are calculated in a second pass. Otherwise, deviations are
+ calculated from ``c`` as given. Use the second case with care, as it can
+ lead to garbage results.
+ """
+ if c is None:
+ c = mean(data)
+ ss = _sum((x-c)**2 for x in data)
+ # The following sum should mathematically equal zero, but due to rounding
+ # error may not.
+ ss -= _sum((x-c) for x in data)**2/len(data)
+ assert not ss < 0, 'negative sum of square deviations: %f' % ss
+ return ss
+
+
+def variance(data, xbar=None):
+ """Return the sample variance of data.
+
+ data should be an iterable of Real-valued numbers, with at least two
+ values. The optional argument xbar, if given, should be the mean of
+ the data. If it is missing or None, the mean is automatically calculated.
+
+ Use this function when your data is a sample from a population. To
+ calculate the variance from the entire population, see ``pvariance``.
+
+ Examples:
+
+ >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5]
+ >>> variance(data)
+ 1.3720238095238095
+
+ If you have already calculated the mean of your data, you can pass it as
+ the optional second argument ``xbar`` to avoid recalculating it:
+
+ >>> m = mean(data)
+ >>> variance(data, m)
+ 1.3720238095238095
+
+ This function does not check that ``xbar`` is actually the mean of
+ ``data``. Giving arbitrary values for ``xbar`` may lead to invalid or
+ impossible results.
+
+ Decimals and Fractions are supported:
+
+ >>> from decimal import Decimal as D
+ >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")])
+ Decimal('31.01875')
+
+ >>> from fractions import Fraction as F
+ >>> variance([F(1, 6), F(1, 2), F(5, 3)])
+ Fraction(67, 108)
+
+ """
+ if iter(data) is data:
+ data = list(data)
+ n = len(data)
+ if n < 2:
+ raise StatisticsError('variance requires at least two data points')
+ ss = _ss(data, xbar)
+ return ss/(n-1)
+
+
+def pvariance(data, mu=None):
+ """Return the population variance of ``data``.
+
+ data should be an iterable of Real-valued numbers, with at least one
+ value. The optional argument mu, if given, should be the mean of
+ the data. If it is missing or None, the mean is automatically calculated.
+
+ Use this function to calculate the variance from the entire population.
+ To estimate the variance from a sample, the ``variance`` function is
+ usually a better choice.
+
+ Examples:
+
+ >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25]
+ >>> pvariance(data)
+ 1.25
+
+ If you have already calculated the mean of the data, you can pass it as
+ the optional second argument to avoid recalculating it:
+
+ >>> mu = mean(data)
+ >>> pvariance(data, mu)
+ 1.25
+
+ This function does not check that ``mu`` is actually the mean of ``data``.
+ Giving arbitrary values for ``mu`` may lead to invalid or impossible
+ results.
+
+ Decimals and Fractions are supported:
+
+ >>> from decimal import Decimal as D
+ >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")])
+ Decimal('24.815')
+
+ >>> from fractions import Fraction as F
+ >>> pvariance([F(1, 4), F(5, 4), F(1, 2)])
+ Fraction(13, 72)
+
+ """
+ if iter(data) is data:
+ data = list(data)
+ n = len(data)
+ if n < 1:
+ raise StatisticsError('pvariance requires at least one data point')
+ ss = _ss(data, mu)
+ return ss/n
+
+
+def stdev(data, xbar=None):
+ """Return the square root of the sample variance.
+
+ See ``variance`` for arguments and other details.
+
+ >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75])
+ 1.0810874155219827
+
+ """
+ var = variance(data, xbar)
+ try:
+ return var.sqrt()
+ except AttributeError:
+ return math.sqrt(var)
+
+
+def pstdev(data, mu=None):
+ """Return the square root of the population variance.
+
+ See ``pvariance`` for arguments and other details.
+
+ >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75])
+ 0.986893273527251
+
+ """
+ var = pvariance(data, mu)
+ try:
+ return var.sqrt()
+ except AttributeError:
+ return math.sqrt(var)
diff --git a/Lib/struct.py b/Lib/struct.py
index 9bfc23f..d6bba58 100644
--- a/Lib/struct.py
+++ b/Lib/struct.py
@@ -1,6 +1,7 @@
__all__ = [
# Functions
'calcsize', 'pack', 'pack_into', 'unpack', 'unpack_from',
+ 'iter_unpack',
# Classes
'Struct',
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 0178b08..38f0f18 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -145,11 +145,13 @@ check_call(*popenargs, **kwargs):
getstatusoutput(cmd):
Return (status, output) of executing cmd in a shell.
- Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple
- (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the
- returned output will contain output or error messages. A trailing newline
- is stripped from the output. The exit status for the command can be
- interpreted according to the rules for the C function wait(). Example:
+ Execute the string 'cmd' in a shell with 'check_output' and
+ return a 2-tuple (status, output). Universal newlines mode is used,
+ meaning that the result with be decoded to a string.
+
+ A trailing newline is stripped from the output.
+ The exit status for the command can be interpreted
+ according to the rules for the function 'wait'. Example:
>>> subprocess.getstatusoutput('ls /bin/ls')
(0, '/bin/ls')
@@ -178,6 +180,9 @@ check_output(*popenargs, **kwargs):
>>> output = subprocess.check_output(["ls", "-l", "/dev/null"])
+ There is an additional optional argument, "input", allowing you to
+ pass a string to the subprocess's stdin. If you use this argument
+ you may not also use the Popen constructor's "stdin" argument.
Exceptions
----------
@@ -345,8 +350,6 @@ mswindows = (sys.platform == "win32")
import io
import os
import time
-import traceback
-import gc
import signal
import builtins
import warnings
@@ -398,19 +401,24 @@ if mswindows:
hStdOutput = None
hStdError = None
wShowWindow = 0
- class pywintypes:
- error = IOError
else:
- import select
- _has_poll = hasattr(select, 'poll')
import _posixsubprocess
- _create_pipe = _posixsubprocess.cloexec_pipe
+ import select
+ import selectors
# When select or poll has indicated that the file is writable,
# we can write up to _PIPE_BUF bytes without risk of blocking.
# POSIX defines PIPE_BUF as >= 512.
_PIPE_BUF = getattr(select, 'PIPE_BUF', 512)
+ # poll/select have the advantage of not requiring any extra file
+ # descriptor, contrarily to epoll/kqueue (also, they require a single
+ # syscall).
+ if hasattr(selectors, 'PollSelector'):
+ _PopenSelector = selectors.PollSelector
+ else:
+ _PopenSelector = selectors.SelectSelector
+
__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
"getoutput", "check_output", "CalledProcessError", "DEVNULL"]
@@ -506,6 +514,8 @@ def _args_from_interpreter_flags():
for flag, opt in flag_opt_map.items():
v = getattr(sys.flags, flag)
if v > 0:
+ if flag == 'hash_randomization':
+ v = 1 # Handle specification of an exact seed
args.append('-' + opt * v)
for opt in sys.warnoptions:
args.append('-W' + opt)
@@ -568,14 +578,31 @@ def check_output(*popenargs, timeout=None, **kwargs):
... stderr=STDOUT)
b'ls: non_existent_file: No such file or directory\n'
+ There is an additional optional argument, "input", allowing you to
+ pass a string to the subprocess's stdin. If you use this argument
+ you may not also use the Popen constructor's "stdin" argument, as
+ it too will be used internally. Example:
+
+ >>> check_output(["sed", "-e", "s/foo/bar/"],
+ ... input=b"when in the course of fooman events\n")
+ b'when in the course of barman events\n'
+
If universal_newlines=True is passed, the return value will be a
string rather than bytes.
"""
if 'stdout' in kwargs:
raise ValueError('stdout argument not allowed, it will be overridden.')
+ if 'input' in kwargs:
+ if 'stdin' in kwargs:
+ raise ValueError('stdin and input arguments may not both be used.')
+ inputdata = kwargs['input']
+ del kwargs['input']
+ kwargs['stdin'] = PIPE
+ else:
+ inputdata = None
with Popen(*popenargs, stdout=PIPE, **kwargs) as process:
try:
- output, unused_err = process.communicate(timeout=timeout)
+ output, unused_err = process.communicate(inputdata, timeout=timeout)
except TimeoutExpired:
process.kill()
output, unused_err = process.communicate()
@@ -664,13 +691,15 @@ def list2cmdline(seq):
#
def getstatusoutput(cmd):
- """Return (status, output) of executing cmd in a shell.
+ """ Return (status, output) of executing cmd in a shell.
+
+ Execute the string 'cmd' in a shell with 'check_output' and
+ return a 2-tuple (status, output). Universal newlines mode is used,
+ meaning that the result with be decoded to a string.
- Execute the string 'cmd' in a shell with os.popen() and return a 2-tuple
- (status, output). cmd is actually run as '{ cmd ; } 2>&1', so that the
- returned output will contain output or error messages. A trailing newline
- is stripped from the output. The exit status for the command can be
- interpreted according to the rules for the C function wait(). Example:
+ A trailing newline is stripped from the output.
+ The exit status for the command can be interpreted
+ according to the rules for the function 'wait'. Example:
>>> import subprocess
>>> subprocess.getstatusoutput('ls /bin/ls')
@@ -822,7 +851,7 @@ class Popen(object):
for f in filter(None, (self.stdin, self.stdout, self.stderr)):
try:
f.close()
- except EnvironmentError:
+ except OSError:
pass # Ignore EBADF or other errors.
if not self._closed_child_pipe_fds:
@@ -838,7 +867,7 @@ class Popen(object):
for fd in to_close:
try:
os.close(fd)
- except EnvironmentError:
+ except OSError:
pass
raise
@@ -899,7 +928,7 @@ class Popen(object):
if input:
try:
self.stdin.write(input)
- except IOError as e:
+ except OSError as e:
if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
raise
self.stdin.close()
@@ -1031,23 +1060,6 @@ class Popen(object):
return Handle(h)
- def _find_w9xpopen(self):
- """Find and return absolut path to w9xpopen.exe"""
- w9xpopen = os.path.join(
- os.path.dirname(_winapi.GetModuleFileName(0)),
- "w9xpopen.exe")
- if not os.path.exists(w9xpopen):
- # Eeek - file-not-found - possibly an embedding
- # situation - see if we can locate it in sys.exec_prefix
- w9xpopen = os.path.join(os.path.dirname(sys.base_exec_prefix),
- "w9xpopen.exe")
- if not os.path.exists(w9xpopen):
- raise RuntimeError("Cannot locate w9xpopen.exe, which is "
- "needed for Popen to work with your "
- "shell or platform.")
- return w9xpopen
-
-
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env,
startupinfo, creationflags, shell,
@@ -1076,21 +1088,6 @@ class Popen(object):
startupinfo.wShowWindow = _winapi.SW_HIDE
comspec = os.environ.get("COMSPEC", "cmd.exe")
args = '{} /c "{}"'.format (comspec, args)
- if (_winapi.GetVersion() >= 0x80000000 or
- os.path.basename(comspec).lower() == "command.com"):
- # Win9x, or using command.com on NT. We need to
- # use the w9xpopen intermediate program. For more
- # information, see KB Q150956
- # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
- w9xpopen = self._find_w9xpopen()
- args = '"%s" %s' % (w9xpopen, args)
- # Not passing CREATE_NEW_CONSOLE has been known to
- # cause random failures on win9x. Specifically a
- # dialog: "Your program accessed mem currently in
- # use at xxx" and a hopeful warning about the
- # stability of your system. Cost is Ctrl+C won't
- # kill children.
- creationflags |= _winapi.CREATE_NEW_CONSOLE
# Start the process
try:
@@ -1102,12 +1099,6 @@ class Popen(object):
env,
cwd,
startupinfo)
- except pywintypes.error as e:
- # Translate pywintypes.error to WindowsError, which is
- # a subclass of OSError. FIXME: We should really
- # translate errno using _sys_errlist (or similar), but
- # how can this be done from Python?
- raise WindowsError(*e.args)
finally:
# Child is launched. Close the parent's copy of those pipe
# handles that only the child should have open. You need
@@ -1192,9 +1183,9 @@ class Popen(object):
if input is not None:
try:
self.stdin.write(input)
- except IOError as e:
+ except OSError as e:
if e.errno == errno.EPIPE:
- # ignore pipe full error
+ # communicate() should ignore pipe full error
pass
elif (e.errno == errno.EINVAL
and self.poll() is not None):
@@ -1278,7 +1269,7 @@ class Popen(object):
if stdin is None:
pass
elif stdin == PIPE:
- p2cread, p2cwrite = _create_pipe()
+ p2cread, p2cwrite = os.pipe()
elif stdin == DEVNULL:
p2cread = self._get_devnull()
elif isinstance(stdin, int):
@@ -1290,7 +1281,7 @@ class Popen(object):
if stdout is None:
pass
elif stdout == PIPE:
- c2pread, c2pwrite = _create_pipe()
+ c2pread, c2pwrite = os.pipe()
elif stdout == DEVNULL:
c2pwrite = self._get_devnull()
elif isinstance(stdout, int):
@@ -1302,7 +1293,7 @@ class Popen(object):
if stderr is None:
pass
elif stderr == PIPE:
- errread, errwrite = _create_pipe()
+ errread, errwrite = os.pipe()
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == DEVNULL:
@@ -1354,7 +1345,14 @@ class Popen(object):
# For transferring possible exec failure from child to parent.
# Data format: "exception name:hex errno:description"
# Pickle is not used; it is complex and involves memory allocation.
- errpipe_read, errpipe_write = _create_pipe()
+ errpipe_read, errpipe_write = os.pipe()
+ # errpipe_write must not be in the standard io 0, 1, or 2 fd range.
+ low_fds_to_close = []
+ while errpipe_write < 3:
+ low_fds_to_close.append(errpipe_write)
+ errpipe_write = os.dup(errpipe_write)
+ for low_fd in low_fds_to_close:
+ os.close(low_fd)
try:
try:
# We must avoid complex work that could involve
@@ -1424,13 +1422,13 @@ class Popen(object):
exception_name, hex_errno, err_msg = (
errpipe_data.split(b':', 2))
except ValueError:
- exception_name = b'RuntimeError'
+ exception_name = b'SubprocessError'
hex_errno = b'0'
err_msg = (b'Bad exception data from child: ' +
repr(errpipe_data))
child_exception_type = getattr(
builtins, exception_name.decode('ascii'),
- RuntimeError)
+ SubprocessError)
err_msg = err_msg.decode(errors="surrogatepass")
if issubclass(child_exception_type, OSError) and hex_errno:
errno_num = int(hex_errno, 16)
@@ -1460,11 +1458,11 @@ class Popen(object):
self.returncode = _WEXITSTATUS(sts)
else:
# Should never happen
- raise RuntimeError("Unknown child exit status!")
+ raise SubprocessError("Unknown child exit status!")
def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
- _WNOHANG=os.WNOHANG, _os_error=os.error, _ECHILD=errno.ECHILD):
+ _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
"""Check if child process has terminated. Returns returncode
attribute.
@@ -1477,7 +1475,7 @@ class Popen(object):
pid, sts = _waitpid(self.pid, _WNOHANG)
if pid == self.pid:
self._handle_exitstatus(sts)
- except _os_error as e:
+ except OSError as e:
if _deadstate is not None:
self.returncode = _deadstate
elif e.errno == _ECHILD:
@@ -1551,12 +1549,68 @@ class Popen(object):
if not input:
self.stdin.close()
- if _has_poll:
- stdout, stderr = self._communicate_with_poll(input, endtime,
- orig_timeout)
- else:
- stdout, stderr = self._communicate_with_select(input, endtime,
- orig_timeout)
+ stdout = None
+ stderr = None
+
+ # Only create this mapping if we haven't already.
+ if not self._communication_started:
+ self._fileobj2output = {}
+ if self.stdout:
+ self._fileobj2output[self.stdout] = []
+ if self.stderr:
+ self._fileobj2output[self.stderr] = []
+
+ if self.stdout:
+ stdout = self._fileobj2output[self.stdout]
+ if self.stderr:
+ stderr = self._fileobj2output[self.stderr]
+
+ self._save_input(input)
+
+ if self._input:
+ input_view = memoryview(self._input)
+
+ with _PopenSelector() as selector:
+ if self.stdin and input:
+ selector.register(self.stdin, selectors.EVENT_WRITE)
+ if self.stdout:
+ selector.register(self.stdout, selectors.EVENT_READ)
+ if self.stderr:
+ selector.register(self.stderr, selectors.EVENT_READ)
+
+ while selector.get_map():
+ timeout = self._remaining_time(endtime)
+ if timeout is not None and timeout < 0:
+ raise TimeoutExpired(self.args, orig_timeout)
+
+ ready = selector.select(timeout)
+ self._check_timeout(endtime, orig_timeout)
+
+ # XXX Rewrite these to use non-blocking I/O on the file
+ # objects; they are no longer using C stdio!
+
+ for key, events in ready:
+ if key.fileobj is self.stdin:
+ chunk = input_view[self._input_offset :
+ self._input_offset + _PIPE_BUF]
+ try:
+ self._input_offset += os.write(key.fd, chunk)
+ except OSError as e:
+ if e.errno == errno.EPIPE:
+ selector.unregister(key.fileobj)
+ key.fileobj.close()
+ else:
+ raise
+ else:
+ if self._input_offset >= len(self._input):
+ selector.unregister(key.fileobj)
+ key.fileobj.close()
+ elif key.fileobj in (self.stdout, self.stderr):
+ data = os.read(key.fd, 32768)
+ if not data:
+ selector.unregister(key.fileobj)
+ key.fileobj.close()
+ self._fileobj2output[key.fileobj].append(data)
self.wait(timeout=self._remaining_time(endtime))
@@ -1590,170 +1644,6 @@ class Popen(object):
self._input = self._input.encode(self.stdin.encoding)
- def _communicate_with_poll(self, input, endtime, orig_timeout):
- stdout = None # Return
- stderr = None # Return
-
- if not self._communication_started:
- self._fd2file = {}
-
- poller = select.poll()
- def register_and_append(file_obj, eventmask):
- poller.register(file_obj.fileno(), eventmask)
- self._fd2file[file_obj.fileno()] = file_obj
-
- def close_unregister_and_remove(fd):
- poller.unregister(fd)
- self._fd2file[fd].close()
- self._fd2file.pop(fd)
-
- if self.stdin and input:
- register_and_append(self.stdin, select.POLLOUT)
-
- # Only create this mapping if we haven't already.
- if not self._communication_started:
- self._fd2output = {}
- if self.stdout:
- self._fd2output[self.stdout.fileno()] = []
- if self.stderr:
- self._fd2output[self.stderr.fileno()] = []
-
- select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI
- if self.stdout:
- register_and_append(self.stdout, select_POLLIN_POLLPRI)
- stdout = self._fd2output[self.stdout.fileno()]
- if self.stderr:
- register_and_append(self.stderr, select_POLLIN_POLLPRI)
- stderr = self._fd2output[self.stderr.fileno()]
-
- self._save_input(input)
-
- if self._input:
- input_view = memoryview(self._input)
-
- while self._fd2file:
- timeout = self._remaining_time(endtime)
- if timeout is not None and timeout < 0:
- raise TimeoutExpired(self.args, orig_timeout)
- try:
- ready = poller.poll(timeout)
- except select.error as e:
- if e.args[0] == errno.EINTR:
- continue
- raise
- self._check_timeout(endtime, orig_timeout)
-
- # XXX Rewrite these to use non-blocking I/O on the
- # file objects; they are no longer using C stdio!
-
- for fd, mode in ready:
- if mode & select.POLLOUT:
- chunk = input_view[self._input_offset :
- self._input_offset + _PIPE_BUF]
- try:
- self._input_offset += os.write(fd, chunk)
- except OSError as e:
- if e.errno == errno.EPIPE:
- close_unregister_and_remove(fd)
- else:
- raise
- else:
- if self._input_offset >= len(self._input):
- close_unregister_and_remove(fd)
- elif mode & select_POLLIN_POLLPRI:
- data = os.read(fd, 32768)
- if not data:
- close_unregister_and_remove(fd)
- self._fd2output[fd].append(data)
- else:
- # Ignore hang up or errors.
- close_unregister_and_remove(fd)
-
- return (stdout, stderr)
-
-
- def _communicate_with_select(self, input, endtime, orig_timeout):
- if not self._communication_started:
- self._read_set = []
- self._write_set = []
- if self.stdin and input:
- self._write_set.append(self.stdin)
- if self.stdout:
- self._read_set.append(self.stdout)
- if self.stderr:
- self._read_set.append(self.stderr)
-
- self._save_input(input)
-
- stdout = None # Return
- stderr = None # Return
-
- if self.stdout:
- if not self._communication_started:
- self._stdout_buff = []
- stdout = self._stdout_buff
- if self.stderr:
- if not self._communication_started:
- self._stderr_buff = []
- stderr = self._stderr_buff
-
- while self._read_set or self._write_set:
- timeout = self._remaining_time(endtime)
- if timeout is not None and timeout < 0:
- raise TimeoutExpired(self.args, orig_timeout)
- try:
- (rlist, wlist, xlist) = \
- select.select(self._read_set, self._write_set, [],
- timeout)
- except select.error as e:
- if e.args[0] == errno.EINTR:
- continue
- raise
-
- # According to the docs, returning three empty lists indicates
- # that the timeout expired.
- if not (rlist or wlist or xlist):
- raise TimeoutExpired(self.args, orig_timeout)
- # We also check what time it is ourselves for good measure.
- self._check_timeout(endtime, orig_timeout)
-
- # XXX Rewrite these to use non-blocking I/O on the
- # file objects; they are no longer using C stdio!
-
- if self.stdin in wlist:
- chunk = self._input[self._input_offset :
- self._input_offset + _PIPE_BUF]
- try:
- bytes_written = os.write(self.stdin.fileno(), chunk)
- except OSError as e:
- if e.errno == errno.EPIPE:
- self.stdin.close()
- self._write_set.remove(self.stdin)
- else:
- raise
- else:
- self._input_offset += bytes_written
- if self._input_offset >= len(self._input):
- self.stdin.close()
- self._write_set.remove(self.stdin)
-
- if self.stdout in rlist:
- data = os.read(self.stdout.fileno(), 1024)
- if not data:
- self.stdout.close()
- self._read_set.remove(self.stdout)
- stdout.append(data)
-
- if self.stderr in rlist:
- data = os.read(self.stderr.fileno(), 1024)
- if not data:
- self.stderr.close()
- self._read_set.remove(self.stderr)
- stderr.append(data)
-
- return (stdout, stderr)
-
-
def send_signal(self, sig):
"""Send a signal to the process
"""
diff --git a/Lib/sunau.py b/Lib/sunau.py
index 15f7b03..3c24492 100644
--- a/Lib/sunau.py
+++ b/Lib/sunau.py
@@ -51,7 +51,7 @@ This returns an instance of a class with the following public methods:
getcomptype() -- returns compression type ('NONE' or 'ULAW')
getcompname() -- returns human-readable version of
compression type ('not compressed' matches 'NONE')
- getparams() -- returns a tuple consisting of all of the
+ getparams() -- returns a namedtuple consisting of all of the
above in the above order
getmarkers() -- returns None (for compatibility with the
aifc module)
@@ -103,6 +103,11 @@ The close() method is called automatically when the class instance
is destroyed.
"""
+from collections import namedtuple
+
+_sunau_params = namedtuple('_sunau_params',
+ 'nchannels sampwidth framerate nframes comptype compname')
+
# from <multimedia/audio_filehdr.h>
AUDIO_FILE_MAGIC = 0x2e736e64
AUDIO_FILE_ENCODING_MULAW_8 = 1
@@ -163,6 +168,12 @@ class Au_read:
if self._file:
self.close()
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
def initfp(self, file):
self._file = file
self._soundpos = 0
@@ -246,9 +257,9 @@ class Au_read:
return 'not compressed'
def getparams(self):
- return self.getnchannels(), self.getsampwidth(), \
- self.getframerate(), self.getnframes(), \
- self.getcomptype(), self.getcompname()
+ return _sunau_params(self.getnchannels(), self.getsampwidth(),
+ self.getframerate(), self.getnframes(),
+ self.getcomptype(), self.getcompname())
def getmarkers(self):
return None
@@ -307,6 +318,12 @@ class Au_write:
self.close()
self._file = None
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *args):
+ self.close()
+
def initfp(self, file):
self._file = file
self._framerate = 0
@@ -335,7 +352,7 @@ class Au_write:
def setsampwidth(self, sampwidth):
if self._nframeswritten:
raise Error('cannot change parameters after starting to write')
- if sampwidth not in (1, 2, 4):
+ if sampwidth not in (1, 2, 3, 4):
raise Error('bad sample width')
self._sampwidth = sampwidth
@@ -390,14 +407,16 @@ class Au_write:
self.setcomptype(comptype, compname)
def getparams(self):
- return self.getnchannels(), self.getsampwidth(), \
- self.getframerate(), self.getnframes(), \
- self.getcomptype(), self.getcompname()
+ return _sunau_params(self.getnchannels(), self.getsampwidth(),
+ self.getframerate(), self.getnframes(),
+ self.getcomptype(), self.getcompname())
def tell(self):
return self._nframeswritten
def writeframesraw(self, data):
+ if not isinstance(data, (bytes, bytearray)):
+ data = memoryview(data).cast('B')
self._ensure_header_written()
if self._comptype == 'ULAW':
import audioop
@@ -421,9 +440,9 @@ class Au_write:
self._datalength != self._datawritten:
self._patchheader()
self._file.flush()
+ finally:
if self._opened and self._file:
self._file.close()
- finally:
self._file = None
#
@@ -448,6 +467,9 @@ class Au_write:
elif self._sampwidth == 2:
encoding = AUDIO_FILE_ENCODING_LINEAR_16
self._framesize = 2
+ elif self._sampwidth == 3:
+ encoding = AUDIO_FILE_ENCODING_LINEAR_24
+ self._framesize = 3
elif self._sampwidth == 4:
encoding = AUDIO_FILE_ENCODING_LINEAR_32
self._framesize = 4
diff --git a/Lib/symbol.py b/Lib/symbol.py
index 34143b5..5cf4a65 100755
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -100,7 +100,7 @@ for _name, _value in list(globals().items()):
sym_name[_value] = _name
-def main():
+def _main():
import sys
import token
if len(sys.argv) == 1:
@@ -108,4 +108,4 @@ def main():
token._main()
if __name__ == "__main__":
- main()
+ _main()
diff --git a/Lib/symtable.py b/Lib/symtable.py
index bb27196..e23313b 100644
--- a/Lib/symtable.py
+++ b/Lib/symtable.py
@@ -232,7 +232,8 @@ class Symbol(object):
if __name__ == "__main__":
import os, sys
- src = open(sys.argv[0]).read()
+ with open(sys.argv[0]) as f:
+ src = f.read()
mod = symtable(src, os.path.split(sys.argv[0])[1], "exec")
for ident in mod.get_identifiers():
info = mod.lookup(ident)
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
index 2a2160c..dbf7767 100644
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -1,7 +1,6 @@
"""Access to Python's configuration information."""
import os
-import re
import sys
from os.path import pardir, realpath
@@ -52,25 +51,6 @@ _INSTALL_SCHEMES = {
'scripts': '{base}/Scripts',
'data': '{base}',
},
- 'os2': {
- 'stdlib': '{installed_base}/Lib',
- 'platstdlib': '{base}/Lib',
- 'purelib': '{base}/Lib/site-packages',
- 'platlib': '{base}/Lib/site-packages',
- 'include': '{installed_base}/Include',
- 'platinclude': '{installed_base}/Include',
- 'scripts': '{base}/Scripts',
- 'data': '{base}',
- },
- 'os2_home': {
- 'stdlib': '{userbase}/lib/python{py_version_short}',
- 'platstdlib': '{userbase}/lib/python{py_version_short}',
- 'purelib': '{userbase}/lib/python{py_version_short}/site-packages',
- 'platlib': '{userbase}/lib/python{py_version_short}/site-packages',
- 'include': '{userbase}/include/python{py_version_short}',
- 'scripts': '{userbase}/bin',
- 'data': '{userbase}',
- },
'nt_user': {
'stdlib': '{userbase}/Python{py_version_nodot}',
'platstdlib': '{userbase}/Python{py_version_nodot}',
@@ -210,7 +190,6 @@ def _getuserbase():
def joinuser(*args):
return os.path.expanduser(os.path.join(*args))
- # what about 'os2emx', 'riscos' ?
if os.name == "nt":
base = os.environ.get("APPDATA") or "~"
if env_base:
@@ -242,6 +221,7 @@ def _parse_makefile(filename, vars=None):
"""
# Regexes needed for parsing Makefile (and similar syntaxes,
# like old-style Setup files).
+ import re
_variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
@@ -369,21 +349,21 @@ def _generate_posix_vars():
makefile = get_makefile_filename()
try:
_parse_makefile(makefile, vars)
- except IOError as e:
+ except OSError as e:
msg = "invalid Python installation: unable to open %s" % makefile
if hasattr(e, "strerror"):
msg = msg + " (%s)" % e.strerror
- raise IOError(msg)
+ raise OSError(msg)
# load the installed pyconfig.h:
config_h = get_config_h_filename()
try:
with open(config_h) as f:
parse_config_h(f, vars)
- except IOError as e:
+ except OSError as e:
msg = "invalid Python installation: unable to open %s" % config_h
if hasattr(e, "strerror"):
msg = msg + " (%s)" % e.strerror
- raise IOError(msg)
+ raise OSError(msg)
# On AIX, there are wrong paths to the linker scripts in the Makefile
# -- these paths are relative to the Python source, but when installed
# the scripts are in another directory.
@@ -403,8 +383,8 @@ def _generate_posix_vars():
# get_platform() succeeds.
name = '_sysconfigdata'
if 'darwin' in sys.platform:
- import imp
- module = imp.new_module(name)
+ import types
+ module = types.ModuleType(name)
module.build_time_vars = vars
sys.modules[name] = module
@@ -436,7 +416,6 @@ def _init_non_posix(vars):
vars['LIBDEST'] = get_path('stdlib')
vars['BINLIBDEST'] = get_path('platstdlib')
vars['INCLUDEPY'] = get_path('include')
- vars['SO'] = '.pyd'
vars['EXT_SUFFIX'] = '.pyd'
vars['EXE'] = '.exe'
vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT
@@ -456,6 +435,7 @@ def parse_config_h(fp, vars=None):
"""
if vars is None:
vars = {}
+ import re
define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n")
undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n")
@@ -552,10 +532,14 @@ def get_config_vars(*args):
# sys.abiflags may not be defined on all platforms.
_CONFIG_VARS['abiflags'] = ''
- if os.name in ('nt', 'os2'):
+ if os.name == 'nt':
_init_non_posix(_CONFIG_VARS)
if os.name == 'posix':
_init_posix(_CONFIG_VARS)
+ # For backward compatibility, see issue19555
+ SO = _CONFIG_VARS.get('EXT_SUFFIX')
+ if SO is not None:
+ _CONFIG_VARS['SO'] = SO
# Setting 'userbase' is done below the call to the
# init function to enable using 'get_config_var' in
# the init-function.
@@ -599,6 +583,9 @@ def get_config_var(name):
Equivalent to get_config_vars().get(name)
"""
+ if name == 'SO':
+ import warnings
+ warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2)
return get_config_vars().get(name)
@@ -679,6 +666,7 @@ def get_platform():
return "%s-%s.%s" % (osname, version, release)
elif osname[:6] == "cygwin":
osname = "cygwin"
+ import re
rel_re = re.compile(r'[\d.]+')
m = rel_re.match(release)
if m:
diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py
index 5b9b444..46e0f56 100755
--- a/Lib/tabnanny.py
+++ b/Lib/tabnanny.py
@@ -95,7 +95,7 @@ def check(file):
try:
f = tokenize.open(file)
- except IOError as msg:
+ except OSError as msg:
errprint("%r: I/O Error: %s" % (file, msg))
return
diff --git a/Lib/tarfile.py b/Lib/tarfile.py
index f6d7f79..0a90257 100755
--- a/Lib/tarfile.py
+++ b/Lib/tarfile.py
@@ -56,17 +56,15 @@ except ImportError:
# os.symlink on Windows prior to 6.0 raises NotImplementedError
symlink_exception = (AttributeError, NotImplementedError)
try:
- # WindowsError (1314) will be raised if the caller does not hold the
+ # OSError (winerror=1314) will be raised if the caller does not hold the
# SeCreateSymbolicLinkPrivilege privilege
- symlink_exception += (WindowsError,)
+ symlink_exception += (OSError,)
except NameError:
pass
# from tarfile import *
__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
-from builtins import open as _open # Since 'open' is TarFile.open
-
#---------------------------------------------------------
# tar constants
#---------------------------------------------------------
@@ -140,30 +138,6 @@ PAX_NUMBER_FIELDS = {
}
#---------------------------------------------------------
-# Bits used in the mode field, values in octal.
-#---------------------------------------------------------
-S_IFLNK = 0o120000 # symbolic link
-S_IFREG = 0o100000 # regular file
-S_IFBLK = 0o060000 # block device
-S_IFDIR = 0o040000 # directory
-S_IFCHR = 0o020000 # character device
-S_IFIFO = 0o010000 # fifo
-
-TSUID = 0o4000 # set UID on execution
-TSGID = 0o2000 # set GID on execution
-TSVTX = 0o1000 # reserved
-
-TUREAD = 0o400 # read by owner
-TUWRITE = 0o200 # write by owner
-TUEXEC = 0o100 # execute/search by owner
-TGREAD = 0o040 # read by group
-TGWRITE = 0o020 # write by group
-TGEXEC = 0o010 # execute/search by group
-TOREAD = 0o004 # read by other
-TOWRITE = 0o002 # write by other
-TOEXEC = 0o001 # execute/search by other
-
-#---------------------------------------------------------
# initialization
#---------------------------------------------------------
if os.name in ("nt", "ce"):
@@ -220,7 +194,7 @@ def itn(n, digits=8, format=DEFAULT_FORMAT):
# A 0o200 byte indicates a positive number, a 0o377 byte a negative
# number.
if 0 <= n < 8 ** (digits - 1):
- s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL
+ s = bytes("%0*o" % (digits - 1, int(n)), "ascii") + NUL
elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1):
if n >= 0:
s = bytearray([0o200])
@@ -264,13 +238,13 @@ def copyfileobj(src, dst, length=None):
for b in range(blocks):
buf = src.read(BUFSIZE)
if len(buf) < BUFSIZE:
- raise IOError("end of file reached")
+ raise OSError("end of file reached")
dst.write(buf)
if remainder != 0:
buf = src.read(remainder)
if len(buf) < remainder:
- raise IOError("end of file reached")
+ raise OSError("end of file reached")
dst.write(buf)
return
@@ -405,7 +379,7 @@ class _Stream:
if mode == "r":
self.dbuf = b""
self.cmp = bz2.BZ2Decompressor()
- self.exception = IOError
+ self.exception = OSError
else:
self.cmp = bz2.BZ2Compressor()
@@ -1672,7 +1646,7 @@ class TarFile(object):
try:
t = cls.taropen(name, mode, fileobj, **kwargs)
- except (IOError, EOFError):
+ except (OSError, EOFError):
fileobj.close()
if mode == 'r':
raise ReadError("not a bzip2 file")
@@ -2042,7 +2016,7 @@ class TarFile(object):
try:
self._extract_member(tarinfo, os.path.join(path, tarinfo.name),
set_attrs=set_attrs)
- except EnvironmentError as e:
+ except OSError as e:
if self.errorlevel > 0:
raise
else:
@@ -2231,9 +2205,8 @@ class TarFile(object):
if tarinfo.issym() and hasattr(os, "lchown"):
os.lchown(targetpath, u, g)
else:
- if sys.platform != "os2emx":
- os.chown(targetpath, u, g)
- except EnvironmentError as e:
+ os.chown(targetpath, u, g)
+ except OSError as e:
raise ExtractError("could not change owner")
def chmod(self, tarinfo, targetpath):
@@ -2242,7 +2215,7 @@ class TarFile(object):
if hasattr(os, 'chmod'):
try:
os.chmod(targetpath, tarinfo.mode)
- except EnvironmentError as e:
+ except OSError as e:
raise ExtractError("could not change mode")
def utime(self, tarinfo, targetpath):
@@ -2252,7 +2225,7 @@ class TarFile(object):
return
try:
os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
- except EnvironmentError as e:
+ except OSError as e:
raise ExtractError("could not change modification time")
#--------------------------------------------------------------------------
@@ -2343,9 +2316,9 @@ class TarFile(object):
corresponds to TarFile's mode.
"""
if self.closed:
- raise IOError("%s is closed" % self.__class__.__name__)
+ raise OSError("%s is closed" % self.__class__.__name__)
if mode is not None and self.mode not in mode:
- raise IOError("bad operation for mode %r" % self.mode)
+ raise OSError("bad operation for mode %r" % self.mode)
def _find_link_target(self, tarinfo):
"""Find the target member of a symlink or hardlink member in the
@@ -2449,3 +2422,97 @@ def is_tarfile(name):
bltn_open = open
open = TarFile.open
+
+
+def main():
+ import argparse
+
+ description = 'A simple command line interface for tarfile module.'
+ parser = argparse.ArgumentParser(description=description)
+ parser.add_argument('-v', '--verbose', action='store_true', default=False,
+ help='Verbose output')
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('-l', '--list', metavar='<tarfile>',
+ help='Show listing of a tarfile')
+ group.add_argument('-e', '--extract', nargs='+',
+ metavar=('<tarfile>', '<output_dir>'),
+ help='Extract tarfile into target dir')
+ group.add_argument('-c', '--create', nargs='+',
+ metavar=('<name>', '<file>'),
+ help='Create tarfile from sources')
+ group.add_argument('-t', '--test', metavar='<tarfile>',
+ help='Test if a tarfile is valid')
+ args = parser.parse_args()
+
+ if args.test:
+ src = args.test
+ if is_tarfile(src):
+ with open(src, 'r') as tar:
+ tar.getmembers()
+ print(tar.getmembers(), file=sys.stderr)
+ if args.verbose:
+ print('{!r} is a tar archive.'.format(src))
+ else:
+ parser.exit(1, '{!r} is not a tar archive.\n'.format(src))
+
+ elif args.list:
+ src = args.list
+ if is_tarfile(src):
+ with TarFile.open(src, 'r:*') as tf:
+ tf.list(verbose=args.verbose)
+ else:
+ parser.exit(1, '{!r} is not a tar archive.\n'.format(src))
+
+ elif args.extract:
+ if len(args.extract) == 1:
+ src = args.extract[0]
+ curdir = os.curdir
+ elif len(args.extract) == 2:
+ src, curdir = args.extract
+ else:
+ parser.exit(1, parser.format_help())
+
+ if is_tarfile(src):
+ with TarFile.open(src, 'r:*') as tf:
+ tf.extractall(path=curdir)
+ if args.verbose:
+ if curdir == '.':
+ msg = '{!r} file is extracted.'.format(src)
+ else:
+ msg = ('{!r} file is extracted '
+ 'into {!r} directory.').format(src, curdir)
+ print(msg)
+ else:
+ parser.exit(1, '{!r} is not a tar archive.\n'.format(src))
+
+ elif args.create:
+ tar_name = args.create.pop(0)
+ _, ext = os.path.splitext(tar_name)
+ compressions = {
+ # gz
+ 'gz': 'gz',
+ 'tgz': 'gz',
+ # xz
+ 'xz': 'xz',
+ 'txz': 'xz',
+ # bz2
+ 'bz2': 'bz2',
+ 'tbz': 'bz2',
+ 'tbz2': 'bz2',
+ 'tb2': 'bz2',
+ }
+ tar_mode = 'w:' + compressions[ext] if ext in compressions else 'w'
+ tar_files = args.create
+
+ with TarFile.open(tar_name, tar_mode) as tf:
+ for file_name in tar_files:
+ tf.add(file_name)
+
+ if args.verbose:
+ print('{!r} file created.'.format(tar_name))
+
+ else:
+ parser.exit(1, parser.format_help())
+
+if __name__ == '__main__':
+ main()
diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py
index d49d4f4..0cacac8 100644
--- a/Lib/telnetlib.py
+++ b/Lib/telnetlib.py
@@ -17,13 +17,12 @@ guido Guido van Rossum pts/2 <Dec 2 11:10> snag.cnri.reston..
Note that read_all() won't read until eof -- it just reads some data
-- but it guarantees to read at least one byte unless EOF is hit.
-It is possible to pass a Telnet object to select.select() in order to
-wait until more data is available. Note that in this case,
-read_eager() may return b'' even if there was data on the socket,
-because the protocol negotiation may have eaten the data. This is why
-EOFError is needed in some cases to distinguish between "no data" and
-"connection closed" (since the socket also appears ready for reading
-when it is closed).
+It is possible to pass a Telnet object to a selector in order to wait until
+more data is available. Note that in this case, read_eager() may return b''
+even if there was data on the socket, because the protocol negotiation may have
+eaten the data. This is why EOFError is needed in some cases to distinguish
+between "no data" and "connection closed" (since the socket also appears ready
+for reading when it is closed).
To do:
- option negotiation
@@ -34,10 +33,9 @@ To do:
# Imported modules
-import errno
import sys
import socket
-import select
+import selectors
try:
from time import monotonic as _time
except ImportError:
@@ -134,6 +132,15 @@ PRAGMA_HEARTBEAT = bytes([140]) # TELOPT PRAGMA HEARTBEAT
EXOPL = bytes([255]) # Extended-Options-List
NOOPT = bytes([0])
+
+# poll/select have the advantage of not requiring any extra file descriptor,
+# contrarily to epoll/kqueue (also, they require a single syscall).
+if hasattr(selectors, 'PollSelector'):
+ _TelnetSelector = selectors.PollSelector
+else:
+ _TelnetSelector = selectors.SelectSelector
+
+
class Telnet:
"""Telnet interface class.
@@ -210,7 +217,6 @@ class Telnet:
self.sb = 0 # flag for SB and SE sequence.
self.sbdataq = b''
self.option_callback = None
- self._has_poll = hasattr(select, 'poll')
if host is not None:
self.open(host, port, timeout)
@@ -277,7 +283,7 @@ class Telnet:
"""Write a string to the socket, doubling any IAC characters.
Can block if the connection is blocked. May raise
- socket.error if the connection is closed.
+ OSError if the connection is closed.
"""
if IAC in buffer:
@@ -293,61 +299,6 @@ class Telnet:
is closed and no cooked data is available.
"""
- if self._has_poll:
- return self._read_until_with_poll(match, timeout)
- else:
- return self._read_until_with_select(match, timeout)
-
- def _read_until_with_poll(self, match, timeout):
- """Read until a given string is encountered or until timeout.
-
- This method uses select.poll() to implement the timeout.
- """
- n = len(match)
- call_timeout = timeout
- if timeout is not None:
- time_start = _time()
- self.process_rawq()
- i = self.cookedq.find(match)
- if i < 0:
- poller = select.poll()
- poll_in_or_priority_flags = select.POLLIN | select.POLLPRI
- poller.register(self, poll_in_or_priority_flags)
- while i < 0 and not self.eof:
- try:
- ready = poller.poll(None if timeout is None
- else 1000 * call_timeout)
- except select.error as e:
- if e.errno == errno.EINTR:
- if timeout is not None:
- elapsed = _time() - time_start
- call_timeout = timeout-elapsed
- continue
- raise
- for fd, mode in ready:
- if mode & poll_in_or_priority_flags:
- i = max(0, len(self.cookedq)-n)
- self.fill_rawq()
- self.process_rawq()
- i = self.cookedq.find(match, i)
- if timeout is not None:
- elapsed = _time() - time_start
- if elapsed >= timeout:
- break
- call_timeout = timeout-elapsed
- poller.unregister(self)
- if i >= 0:
- i = i + n
- buf = self.cookedq[:i]
- self.cookedq = self.cookedq[i:]
- return buf
- return self.read_very_lazy()
-
- def _read_until_with_select(self, match, timeout=None):
- """Read until a given string is encountered or until timeout.
-
- The timeout is implemented using select.select().
- """
n = len(match)
self.process_rawq()
i = self.cookedq.find(match)
@@ -356,26 +307,25 @@ class Telnet:
buf = self.cookedq[:i]
self.cookedq = self.cookedq[i:]
return buf
- s_reply = ([self], [], [])
- s_args = s_reply
if timeout is not None:
- s_args = s_args + (timeout,)
- time_start = _time()
- while not self.eof and select.select(*s_args) == s_reply:
- i = max(0, len(self.cookedq)-n)
- self.fill_rawq()
- self.process_rawq()
- i = self.cookedq.find(match, i)
- if i >= 0:
- i = i+n
- buf = self.cookedq[:i]
- self.cookedq = self.cookedq[i:]
- return buf
- if timeout is not None:
- elapsed = _time() - time_start
- if elapsed >= timeout:
- break
- s_args = s_reply + (timeout-elapsed,)
+ deadline = _time() + timeout
+ with _TelnetSelector() as selector:
+ selector.register(self, selectors.EVENT_READ)
+ while not self.eof:
+ if selector.select(timeout):
+ i = max(0, len(self.cookedq)-n)
+ self.fill_rawq()
+ self.process_rawq()
+ i = self.cookedq.find(match, i)
+ if i >= 0:
+ i = i+n
+ buf = self.cookedq[:i]
+ self.cookedq = self.cookedq[i:]
+ return buf
+ if timeout is not None:
+ timeout = deadline - _time()
+ if timeout < 0:
+ break
return self.read_very_lazy()
def read_all(self):
@@ -580,29 +530,35 @@ class Telnet:
def sock_avail(self):
"""Test whether data is available on the socket."""
- return select.select([self], [], [], 0) == ([self], [], [])
+ with _TelnetSelector() as selector:
+ selector.register(self, selectors.EVENT_READ)
+ return bool(selector.select(0))
def interact(self):
"""Interaction function, emulates a very dumb telnet client."""
if sys.platform == "win32":
self.mt_interact()
return
- while 1:
- rfd, wfd, xfd = select.select([self, sys.stdin], [], [])
- if self in rfd:
- try:
- text = self.read_eager()
- except EOFError:
- print('*** Connection closed by remote host ***')
- break
- if text:
- sys.stdout.write(text.decode('ascii'))
- sys.stdout.flush()
- if sys.stdin in rfd:
- line = sys.stdin.readline().encode('ascii')
- if not line:
- break
- self.write(line)
+ with _TelnetSelector() as selector:
+ selector.register(self, selectors.EVENT_READ)
+ selector.register(sys.stdin, selectors.EVENT_READ)
+
+ while True:
+ for key, events in selector.select():
+ if key.fileobj is self:
+ try:
+ text = self.read_eager()
+ except EOFError:
+ print('*** Connection closed by remote host ***')
+ return
+ if text:
+ sys.stdout.write(text.decode('ascii'))
+ sys.stdout.flush()
+ elif key.fileobj is sys.stdin:
+ line = sys.stdin.readline().encode('ascii')
+ if not line:
+ return
+ self.write(line)
def mt_interact(self):
"""Multithreaded version of interact()."""
@@ -649,79 +605,6 @@ class Telnet:
results are undeterministic, and may depend on the I/O timing.
"""
- if self._has_poll:
- return self._expect_with_poll(list, timeout)
- else:
- return self._expect_with_select(list, timeout)
-
- def _expect_with_poll(self, expect_list, timeout=None):
- """Read until one from a list of a regular expressions matches.
-
- This method uses select.poll() to implement the timeout.
- """
- re = None
- expect_list = expect_list[:]
- indices = range(len(expect_list))
- for i in indices:
- if not hasattr(expect_list[i], "search"):
- if not re: import re
- expect_list[i] = re.compile(expect_list[i])
- call_timeout = timeout
- if timeout is not None:
- time_start = _time()
- self.process_rawq()
- m = None
- for i in indices:
- m = expect_list[i].search(self.cookedq)
- if m:
- e = m.end()
- text = self.cookedq[:e]
- self.cookedq = self.cookedq[e:]
- break
- if not m:
- poller = select.poll()
- poll_in_or_priority_flags = select.POLLIN | select.POLLPRI
- poller.register(self, poll_in_or_priority_flags)
- while not m and not self.eof:
- try:
- ready = poller.poll(None if timeout is None
- else 1000 * call_timeout)
- except select.error as e:
- if e.errno == errno.EINTR:
- if timeout is not None:
- elapsed = _time() - time_start
- call_timeout = timeout-elapsed
- continue
- raise
- for fd, mode in ready:
- if mode & poll_in_or_priority_flags:
- self.fill_rawq()
- self.process_rawq()
- for i in indices:
- m = expect_list[i].search(self.cookedq)
- if m:
- e = m.end()
- text = self.cookedq[:e]
- self.cookedq = self.cookedq[e:]
- break
- if timeout is not None:
- elapsed = _time() - time_start
- if elapsed >= timeout:
- break
- call_timeout = timeout-elapsed
- poller.unregister(self)
- if m:
- return (i, m, text)
- text = self.read_very_lazy()
- if not text and self.eof:
- raise EOFError
- return (-1, None, text)
-
- def _expect_with_select(self, list, timeout=None):
- """Read until one from a list of a regular expressions matches.
-
- The timeout is implemented using select.select().
- """
re = None
list = list[:]
indices = range(len(list))
@@ -730,27 +613,27 @@ class Telnet:
if not re: import re
list[i] = re.compile(list[i])
if timeout is not None:
- time_start = _time()
- while 1:
- self.process_rawq()
- for i in indices:
- m = list[i].search(self.cookedq)
- if m:
- e = m.end()
- text = self.cookedq[:e]
- self.cookedq = self.cookedq[e:]
- return (i, m, text)
- if self.eof:
- break
- if timeout is not None:
- elapsed = _time() - time_start
- if elapsed >= timeout:
- break
- s_args = ([self.fileno()], [], [], timeout-elapsed)
- r, w, x = select.select(*s_args)
- if not r:
- break
- self.fill_rawq()
+ deadline = _time() + timeout
+ with _TelnetSelector() as selector:
+ selector.register(self, selectors.EVENT_READ)
+ while not self.eof:
+ self.process_rawq()
+ for i in indices:
+ m = list[i].search(self.cookedq)
+ if m:
+ e = m.end()
+ text = self.cookedq[:e]
+ self.cookedq = self.cookedq[e:]
+ return (i, m, text)
+ if timeout is not None:
+ ready = selector.select(timeout)
+ timeout = deadline - _time()
+ if not ready:
+ if timeout < 0:
+ break
+ else:
+ continue
+ self.fill_rawq()
text = self.read_very_lazy()
if not text and self.eof:
raise EOFError
diff --git a/Lib/tempfile.py b/Lib/tempfile.py
index 6bc842f..f0e25fc 100644
--- a/Lib/tempfile.py
+++ b/Lib/tempfile.py
@@ -27,7 +27,6 @@ __all__ = [
# Imports.
-import atexit as _atexit
import functools as _functools
import warnings as _warnings
import io as _io
@@ -35,23 +34,7 @@ import os as _os
import shutil as _shutil
import errno as _errno
from random import Random as _Random
-
-try:
- import fcntl as _fcntl
-except ImportError:
- def _set_cloexec(fd):
- pass
-else:
- def _set_cloexec(fd):
- try:
- flags = _fcntl.fcntl(fd, _fcntl.F_GETFD, 0)
- except OSError:
- pass
- else:
- # flags read successfully, modify
- flags |= _fcntl.FD_CLOEXEC
- _fcntl.fcntl(fd, _fcntl.F_SETFD, flags)
-
+import weakref as _weakref
try:
import _thread
@@ -60,8 +43,6 @@ except ImportError:
_allocate_lock = _thread.allocate_lock
_text_openflags = _os.O_RDWR | _os.O_CREAT | _os.O_EXCL
-if hasattr(_os, 'O_NOINHERIT'):
- _text_openflags |= _os.O_NOINHERIT
if hasattr(_os, 'O_NOFOLLOW'):
_text_openflags |= _os.O_NOFOLLOW
@@ -90,8 +71,8 @@ else:
# Fallback. All we need is something that raises OSError if the
# file doesn't exist.
def _stat(fn):
- f = open(fn)
- f.close()
+ fd = _os.open(fn, _os.O_RDONLY)
+ _os.close(fd)
def _exists(fn):
try:
@@ -125,7 +106,7 @@ class _RandomNameSequence:
def __next__(self):
c = self.characters
choose = self.rng.choice
- letters = [choose(c) for dummy in "123456"]
+ letters = [choose(c) for dummy in range(8)]
return ''.join(letters)
def _candidate_tempdir_list():
@@ -167,7 +148,7 @@ def _get_default_tempdir():
for dir in dirlist:
if dir != _os.curdir:
- dir = _os.path.normcase(_os.path.abspath(dir))
+ dir = _os.path.abspath(dir)
# Try only a few names per directory.
for seq in range(100):
name = next(namer)
@@ -217,7 +198,6 @@ def _mkstemp_inner(dir, pre, suf, flags):
file = _os.path.join(dir, pre + name + suf)
try:
fd = _os.open(file, flags, 0o600)
- _set_cloexec(fd)
return (fd, _os.path.abspath(file))
except FileExistsError:
continue # try again
@@ -356,8 +336,7 @@ class _TemporaryFileCloser:
underlying file object, without adding a __del__ method to the
temporary file."""
- # Set here since __del__ checks it
- file = None
+ file = None # Set here since __del__ checks it
close_called = False
def __init__(self, file, name, delete=True):
@@ -479,10 +458,14 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None,
flags |= _os.O_TEMPORARY
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
- file = _io.open(fd, mode, buffering=buffering,
- newline=newline, encoding=encoding)
+ try:
+ file = _io.open(fd, mode, buffering=buffering,
+ newline=newline, encoding=encoding)
- return _TemporaryFileWrapper(file, name, delete)
+ return _TemporaryFileWrapper(file, name, delete)
+ except Exception:
+ _os.close(fd)
+ raise
if _os.name != 'posix' or _os.sys.platform == 'cygwin':
# On non-POSIX and Cygwin systems, assume that we cannot unlink a file
@@ -682,10 +665,21 @@ class TemporaryDirectory(object):
# Handle mkdtemp raising an exception
name = None
+ _finalizer = None
_closed = False
def __init__(self, suffix="", prefix=template, dir=None):
self.name = mkdtemp(suffix, prefix, dir)
+ self._finalizer = _weakref.finalize(
+ self, self._cleanup, self.name,
+ warn_message="Implicitly cleaning up {!r}".format(self))
+
+ @classmethod
+ def _cleanup(cls, name, warn_message=None):
+ _shutil.rmtree(name)
+ if warn_message is not None:
+ _warnings.warn(warn_message, ResourceWarning)
+
def __repr__(self):
return "<{} {!r}>".format(self.__class__.__name__, self.name)
@@ -693,53 +687,12 @@ class TemporaryDirectory(object):
def __enter__(self):
return self.name
- def cleanup(self, _warn=False, _warnings=_warnings):
- if self.name and not self._closed:
- try:
- _shutil.rmtree(self.name)
- except (TypeError, AttributeError) as ex:
- if "None" not in '%s' % (ex,):
- raise
- self._rmtree(self.name)
- self._closed = True
- if _warn and _warnings.warn:
- try:
- _warnings.warn("Implicitly cleaning up {!r}".format(self),
- ResourceWarning)
- except:
- if _is_running:
- raise
- # Don't raise an exception if modules needed for emitting
- # a warning are already cleaned in shutdown process.
-
def __exit__(self, exc, value, tb):
self.cleanup()
- def __del__(self):
- # Issue a ResourceWarning if implicit cleanup needed
- self.cleanup(_warn=True)
-
- def _rmtree(self, path, _OSError=OSError, _sep=_os.path.sep,
- _listdir=_os.listdir, _remove=_os.remove, _rmdir=_os.rmdir):
- # Essentially a stripped down version of shutil.rmtree. We can't
- # use globals because they may be None'ed out at shutdown.
- if not isinstance(path, str):
- _sep = _sep.encode()
- try:
- for name in _listdir(path):
- fullname = path + _sep + name
- try:
- _remove(fullname)
- except _OSError:
- self._rmtree(fullname)
- _rmdir(path)
- except _OSError:
- pass
-
-_is_running = True
-
-def _on_shutdown():
- global _is_running
- _is_running = False
-
-_atexit.register(_on_shutdown)
+ def cleanup(self):
+ if self._finalizer is not None:
+ self._finalizer.detach()
+ if self.name is not None and not self._closed:
+ _shutil.rmtree(self.name)
+ self._closed = True
diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py
index ce5615b..d5fbe15 100644
--- a/Lib/test/__main__.py
+++ b/Lib/test/__main__.py
@@ -1,13 +1,3 @@
-from test import regrtest, support
+from test import regrtest
-
-TEMPDIR, TESTCWD = regrtest._make_temp_dir_for_build(regrtest.TEMPDIR)
-regrtest.TEMPDIR = TEMPDIR
-regrtest.TESTCWD = TESTCWD
-
-# Run the tests in a context manager that temporary changes the CWD to a
-# temporary and writable directory. If it's not possible to create or
-# change the CWD, the original CWD will be used. The original CWD is
-# available from support.SAVEDCWD.
-with support.temp_cwd(TESTCWD, quiet=True):
- regrtest.main()
+regrtest.main_in_temp_cwd()
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
index aa66db4..44d6c71 100644
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/_test_multiprocessing.py
@@ -41,7 +41,7 @@ from multiprocessing import util
try:
from multiprocessing import reduction
- HAS_REDUCTION = True
+ HAS_REDUCTION = reduction.HAVE_SEND_HANDLE
except ImportError:
HAS_REDUCTION = False
@@ -97,6 +97,9 @@ try:
except:
MAXFD = 256
+# To speed up tests when using the forkserver, we can preload these:
+PRELOAD = ['__main__', 'test.test_multiprocessing_forkserver']
+
#
# Some tests require ctypes
#
@@ -292,17 +295,22 @@ class _TestProcess(BaseTestCase):
self.assertTimingAlmostEqual(join.elapsed, 0.0)
self.assertEqual(p.is_alive(), True)
+ # XXX maybe terminating too soon causes the problems on Gentoo...
+ time.sleep(1)
+
p.terminate()
if hasattr(signal, 'alarm'):
+ # On the Gentoo buildbot waitpid() often seems to block forever.
+ # We use alarm() to interrupt it if it blocks for too long.
def handler(*args):
raise RuntimeError('join took too long: %s' % p)
old_handler = signal.signal(signal.SIGALRM, handler)
try:
signal.alarm(10)
self.assertEqual(join(), None)
- signal.alarm(0)
finally:
+ signal.alarm(0)
signal.signal(signal.SIGALRM, old_handler)
else:
self.assertEqual(join(), None)
@@ -340,7 +348,6 @@ class _TestProcess(BaseTestCase):
@classmethod
def _test_recursion(cls, wconn, id):
- from multiprocessing import forking
wconn.send(id)
if len(id) < 2:
for i in range(2):
@@ -388,7 +395,7 @@ class _TestProcess(BaseTestCase):
self.assertFalse(wait_for_handle(sentinel, timeout=0.0))
event.set()
p.join()
- self.assertTrue(wait_for_handle(sentinel, timeout=DELTA))
+ self.assertTrue(wait_for_handle(sentinel, timeout=1))
#
#
@@ -688,9 +695,6 @@ class _TestQueue(BaseTestCase):
def test_task_done(self):
queue = self.JoinableQueue()
- if sys.version_info < (2, 5) and not hasattr(queue, 'task_done'):
- self.skipTest("requires 'queue.task_done()' method")
-
workers = [self.Process(target=self._test_task_done, args=(queue,))
for i in range(4)]
@@ -1777,6 +1781,46 @@ class _TestPool(BaseTestCase):
self.assertEqual(r.get(), expected)
self.assertRaises(ValueError, p.map_async, sqr, L)
+ @classmethod
+ def _test_traceback(cls):
+ raise RuntimeError(123) # some comment
+
+ def test_traceback(self):
+ # We want ensure that the traceback from the child process is
+ # contained in the traceback raised in the main process.
+ if self.TYPE == 'processes':
+ with self.Pool(1) as p:
+ try:
+ p.apply(self._test_traceback)
+ except Exception as e:
+ exc = e
+ else:
+ raise AssertionError('expected RuntimeError')
+ self.assertIs(type(exc), RuntimeError)
+ self.assertEqual(exc.args, (123,))
+ cause = exc.__cause__
+ self.assertIs(type(cause), multiprocessing.pool.RemoteTraceback)
+ self.assertIn('raise RuntimeError(123) # some comment', cause.tb)
+
+ with test.support.captured_stderr() as f1:
+ try:
+ raise exc
+ except RuntimeError:
+ sys.excepthook(*sys.exc_info())
+ self.assertIn('raise RuntimeError(123) # some comment',
+ f1.getvalue())
+
+ @classmethod
+ def _test_wrapped_exception(cls):
+ raise RuntimeError('foo')
+
+ def test_wrapped_exception(self):
+ # Issue #20980: Should not wrap exception when using thread pool
+ with self.Pool(1) as p:
+ with self.assertRaises(RuntimeError):
+ p.apply(self._test_wrapped_exception)
+
+
def raising():
raise KeyError("key")
@@ -2484,7 +2528,7 @@ class _TestPicklingConnections(BaseTestCase):
@classmethod
def tearDownClass(cls):
- from multiprocessing.reduction import resource_sharer
+ from multiprocessing import resource_sharer
resource_sharer.stop(timeout=5)
@classmethod
@@ -2798,30 +2842,40 @@ class _TestFinalize(BaseTestCase):
# Test that from ... import * works for each module
#
-class _TestImportStar(BaseTestCase):
+class _TestImportStar(unittest.TestCase):
- ALLOWED_TYPES = ('processes',)
+ def get_module_names(self):
+ import glob
+ folder = os.path.dirname(multiprocessing.__file__)
+ pattern = os.path.join(folder, '*.py')
+ files = glob.glob(pattern)
+ modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files]
+ modules = ['multiprocessing.' + m for m in modules]
+ modules.remove('multiprocessing.__init__')
+ modules.append('multiprocessing')
+ return modules
def test_import(self):
- modules = [
- 'multiprocessing', 'multiprocessing.connection',
- 'multiprocessing.heap', 'multiprocessing.managers',
- 'multiprocessing.pool', 'multiprocessing.process',
- 'multiprocessing.synchronize', 'multiprocessing.util'
- ]
-
- if HAS_REDUCTION:
- modules.append('multiprocessing.reduction')
+ modules = self.get_module_names()
+ if sys.platform == 'win32':
+ modules.remove('multiprocessing.popen_fork')
+ modules.remove('multiprocessing.popen_forkserver')
+ modules.remove('multiprocessing.popen_spawn_posix')
+ else:
+ modules.remove('multiprocessing.popen_spawn_win32')
+ if not HAS_REDUCTION:
+ modules.remove('multiprocessing.popen_forkserver')
- if c_int is not None:
+ if c_int is None:
# This module requires _ctypes
- modules.append('multiprocessing.sharedctypes')
+ modules.remove('multiprocessing.sharedctypes')
for name in modules:
__import__(name)
mod = sys.modules[name]
+ self.assertTrue(hasattr(mod, '__all__'), name)
- for attr in getattr(mod, '__all__', ()):
+ for attr in mod.__all__:
self.assertTrue(
hasattr(mod, attr),
'%r does not have attribute %r' % (mod, attr)
@@ -2904,7 +2958,7 @@ class _TestPollEintr(BaseTestCase):
@classmethod
def _killer(cls, pid):
- time.sleep(0.5)
+ time.sleep(0.1)
os.kill(pid, signal.SIGUSR1)
@unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
@@ -2917,12 +2971,14 @@ class _TestPollEintr(BaseTestCase):
try:
killer = self.Process(target=self._killer, args=(pid,))
killer.start()
- p = self.Process(target=time.sleep, args=(1,))
- p.start()
- p.join()
+ try:
+ p = self.Process(target=time.sleep, args=(2,))
+ p.start()
+ p.join()
+ finally:
+ killer.join()
self.assertTrue(got_signal[0])
self.assertEqual(p.exitcode, 0)
- killer.join()
finally:
signal.signal(signal.SIGUSR1, oldhandler)
@@ -2935,8 +2991,11 @@ class TestInvalidHandle(unittest.TestCase):
@unittest.skipIf(WIN32, "skipped on Windows")
def test_invalid_handles(self):
conn = multiprocessing.connection.Connection(44977608)
+ # check that poll() doesn't crash
try:
- self.assertRaises((ValueError, OSError), conn.poll)
+ conn.poll()
+ except (ValueError, OSError):
+ pass
finally:
# Hack private attribute _handle to avoid printing an error
# in conn.__del__
@@ -2944,131 +3003,6 @@ class TestInvalidHandle(unittest.TestCase):
self.assertRaises((ValueError, OSError),
multiprocessing.connection.Connection, -1)
-#
-# Functions used to create test cases from the base ones in this module
-#
-
-def create_test_cases(Mixin, type):
- result = {}
- glob = globals()
- Type = type.capitalize()
- ALL_TYPES = {'processes', 'threads', 'manager'}
-
- for name in list(glob.keys()):
- if name.startswith('_Test'):
- base = glob[name]
- assert set(base.ALLOWED_TYPES) <= ALL_TYPES, set(base.ALLOWED_TYPES)
- if type in base.ALLOWED_TYPES:
- newname = 'With' + Type + name[1:]
- class Temp(base, Mixin, unittest.TestCase):
- pass
- result[newname] = Temp
- Temp.__name__ = Temp.__qualname__ = newname
- Temp.__module__ = Mixin.__module__
- return result
-
-#
-# Create test cases
-#
-
-class ProcessesMixin(object):
- TYPE = 'processes'
- Process = multiprocessing.Process
- connection = multiprocessing.connection
- current_process = staticmethod(multiprocessing.current_process)
- active_children = staticmethod(multiprocessing.active_children)
- Pool = staticmethod(multiprocessing.Pool)
- Pipe = staticmethod(multiprocessing.Pipe)
- Queue = staticmethod(multiprocessing.Queue)
- JoinableQueue = staticmethod(multiprocessing.JoinableQueue)
- Lock = staticmethod(multiprocessing.Lock)
- RLock = staticmethod(multiprocessing.RLock)
- Semaphore = staticmethod(multiprocessing.Semaphore)
- BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore)
- Condition = staticmethod(multiprocessing.Condition)
- Event = staticmethod(multiprocessing.Event)
- Barrier = staticmethod(multiprocessing.Barrier)
- Value = staticmethod(multiprocessing.Value)
- Array = staticmethod(multiprocessing.Array)
- RawValue = staticmethod(multiprocessing.RawValue)
- RawArray = staticmethod(multiprocessing.RawArray)
-
-testcases_processes = create_test_cases(ProcessesMixin, type='processes')
-globals().update(testcases_processes)
-
-
-class ManagerMixin(object):
- TYPE = 'manager'
- Process = multiprocessing.Process
- Queue = property(operator.attrgetter('manager.Queue'))
- JoinableQueue = property(operator.attrgetter('manager.JoinableQueue'))
- Lock = property(operator.attrgetter('manager.Lock'))
- RLock = property(operator.attrgetter('manager.RLock'))
- Semaphore = property(operator.attrgetter('manager.Semaphore'))
- BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore'))
- Condition = property(operator.attrgetter('manager.Condition'))
- Event = property(operator.attrgetter('manager.Event'))
- Barrier = property(operator.attrgetter('manager.Barrier'))
- Value = property(operator.attrgetter('manager.Value'))
- Array = property(operator.attrgetter('manager.Array'))
- list = property(operator.attrgetter('manager.list'))
- dict = property(operator.attrgetter('manager.dict'))
- Namespace = property(operator.attrgetter('manager.Namespace'))
-
- @classmethod
- def Pool(cls, *args, **kwds):
- return cls.manager.Pool(*args, **kwds)
-
- @classmethod
- def setUpClass(cls):
- cls.manager = multiprocessing.Manager()
-
- @classmethod
- def tearDownClass(cls):
- # only the manager process should be returned by active_children()
- # but this can take a bit on slow machines, so wait a few seconds
- # if there are other children too (see #17395)
- t = 0.01
- while len(multiprocessing.active_children()) > 1 and t < 5:
- time.sleep(t)
- t *= 2
- gc.collect() # do garbage collection
- if cls.manager._number_of_objects() != 0:
- # This is not really an error since some tests do not
- # ensure that all processes which hold a reference to a
- # managed object have been joined.
- print('Shared objects which still exist at manager shutdown:')
- print(cls.manager._debug_info())
- cls.manager.shutdown()
- cls.manager.join()
- cls.manager = None
-
-testcases_manager = create_test_cases(ManagerMixin, type='manager')
-globals().update(testcases_manager)
-
-
-class ThreadsMixin(object):
- TYPE = 'threads'
- Process = multiprocessing.dummy.Process
- connection = multiprocessing.dummy.connection
- current_process = staticmethod(multiprocessing.dummy.current_process)
- active_children = staticmethod(multiprocessing.dummy.active_children)
- Pool = staticmethod(multiprocessing.Pool)
- Pipe = staticmethod(multiprocessing.dummy.Pipe)
- Queue = staticmethod(multiprocessing.dummy.Queue)
- JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue)
- Lock = staticmethod(multiprocessing.dummy.Lock)
- RLock = staticmethod(multiprocessing.dummy.RLock)
- Semaphore = staticmethod(multiprocessing.dummy.Semaphore)
- BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore)
- Condition = staticmethod(multiprocessing.dummy.Condition)
- Event = staticmethod(multiprocessing.dummy.Event)
- Barrier = staticmethod(multiprocessing.dummy.Barrier)
- Value = staticmethod(multiprocessing.dummy.Value)
- Array = staticmethod(multiprocessing.dummy.Array)
-
-testcases_threads = create_test_cases(ThreadsMixin, type='threads')
-globals().update(testcases_threads)
class OtherTest(unittest.TestCase):
@@ -3418,7 +3352,7 @@ class TestFlags(unittest.TestCase):
def test_flags(self):
import json, subprocess
# start child process using unusual flags
- prog = ('from test.test_multiprocessing import TestFlags; ' +
+ prog = ('from test._test_multiprocessing import TestFlags; ' +
'TestFlags.run_in_child()')
data = subprocess.check_output(
[sys.executable, '-E', '-S', '-O', '-c', prog])
@@ -3465,13 +3399,14 @@ class TestTimeouts(unittest.TestCase):
class TestNoForkBomb(unittest.TestCase):
def test_noforkbomb(self):
+ sm = multiprocessing.get_start_method()
name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
- if WIN32:
- rc, out, err = test.script_helper.assert_python_failure(name)
+ if sm != 'fork':
+ rc, out, err = test.script_helper.assert_python_failure(name, sm)
self.assertEqual('', out.decode('ascii'))
self.assertIn('RuntimeError', err.decode('ascii'))
else:
- rc, out, err = test.script_helper.assert_python_ok(name)
+ rc, out, err = test.script_helper.assert_python_ok(name, sm)
self.assertEqual('123', out.decode('ascii').rstrip())
self.assertEqual('', err.decode('ascii'))
@@ -3489,7 +3424,8 @@ class TestForkAwareThreadLock(unittest.TestCase):
if n > 1:
p = multiprocessing.Process(target=cls.child, args=(n-1, conn))
p.start()
- p.join()
+ conn.close()
+ p.join(timeout=5)
else:
conn.send(len(util._afterfork_registry))
conn.close()
@@ -3500,11 +3436,78 @@ class TestForkAwareThreadLock(unittest.TestCase):
old_size = len(util._afterfork_registry)
p = multiprocessing.Process(target=self.child, args=(5, w))
p.start()
+ w.close()
new_size = r.recv()
- p.join()
+ p.join(timeout=5)
self.assertLessEqual(new_size, old_size)
#
+# Check that non-forked child processes do not inherit unneeded fds/handles
+#
+
+class TestCloseFds(unittest.TestCase):
+
+ def get_high_socket_fd(self):
+ if WIN32:
+ # The child process will not have any socket handles, so
+ # calling socket.fromfd() should produce WSAENOTSOCK even
+ # if there is a handle of the same number.
+ return socket.socket().detach()
+ else:
+ # We want to produce a socket with an fd high enough that a
+ # freshly created child process will not have any fds as high.
+ fd = socket.socket().detach()
+ to_close = []
+ while fd < 50:
+ to_close.append(fd)
+ fd = os.dup(fd)
+ for x in to_close:
+ os.close(x)
+ return fd
+
+ def close(self, fd):
+ if WIN32:
+ socket.socket(fileno=fd).close()
+ else:
+ os.close(fd)
+
+ @classmethod
+ def _test_closefds(cls, conn, fd):
+ try:
+ s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
+ except Exception as e:
+ conn.send(e)
+ else:
+ s.close()
+ conn.send(None)
+
+ def test_closefd(self):
+ if not HAS_REDUCTION:
+ raise unittest.SkipTest('requires fd pickling')
+
+ reader, writer = multiprocessing.Pipe()
+ fd = self.get_high_socket_fd()
+ try:
+ p = multiprocessing.Process(target=self._test_closefds,
+ args=(writer, fd))
+ p.start()
+ writer.close()
+ e = reader.recv()
+ p.join(timeout=5)
+ finally:
+ self.close(fd)
+ writer.close()
+ reader.close()
+
+ if multiprocessing.get_start_method() == 'fork':
+ self.assertIs(e, None)
+ else:
+ WSAENOTSOCK = 10038
+ self.assertIsInstance(e, OSError)
+ self.assertTrue(e.errno == errno.EBADF or
+ e.winerror == WSAENOTSOCK, e)
+
+#
# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
#
@@ -3548,10 +3551,10 @@ class TestIgnoreEINTR(unittest.TestCase):
def handler(signum, frame):
pass
signal.signal(signal.SIGUSR1, handler)
- l = multiprocessing.connection.Listener()
- conn.send(l.address)
- a = l.accept()
- a.send('welcome')
+ with multiprocessing.connection.Listener() as l:
+ conn.send(l.address)
+ a = l.accept()
+ a.send('welcome')
@unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
def test_ignore_listener(self):
@@ -3572,26 +3575,267 @@ class TestIgnoreEINTR(unittest.TestCase):
finally:
conn.close()
+class TestStartMethod(unittest.TestCase):
+ @classmethod
+ def _check_context(cls, conn):
+ conn.send(multiprocessing.get_start_method())
+
+ def check_context(self, ctx):
+ r, w = ctx.Pipe(duplex=False)
+ p = ctx.Process(target=self._check_context, args=(w,))
+ p.start()
+ w.close()
+ child_method = r.recv()
+ r.close()
+ p.join()
+ self.assertEqual(child_method, ctx.get_start_method())
+
+ def test_context(self):
+ for method in ('fork', 'spawn', 'forkserver'):
+ try:
+ ctx = multiprocessing.get_context(method)
+ except ValueError:
+ continue
+ self.assertEqual(ctx.get_start_method(), method)
+ self.assertIs(ctx.get_context(), ctx)
+ self.assertRaises(ValueError, ctx.set_start_method, 'spawn')
+ self.assertRaises(ValueError, ctx.set_start_method, None)
+ self.check_context(ctx)
+
+ def test_set_get(self):
+ multiprocessing.set_forkserver_preload(PRELOAD)
+ count = 0
+ old_method = multiprocessing.get_start_method()
+ try:
+ for method in ('fork', 'spawn', 'forkserver'):
+ try:
+ multiprocessing.set_start_method(method, force=True)
+ except ValueError:
+ continue
+ self.assertEqual(multiprocessing.get_start_method(), method)
+ ctx = multiprocessing.get_context()
+ self.assertEqual(ctx.get_start_method(), method)
+ self.assertTrue(type(ctx).__name__.lower().startswith(method))
+ self.assertTrue(
+ ctx.Process.__name__.lower().startswith(method))
+ self.check_context(multiprocessing)
+ count += 1
+ finally:
+ multiprocessing.set_start_method(old_method, force=True)
+ self.assertGreaterEqual(count, 1)
+
+ def test_get_all(self):
+ methods = multiprocessing.get_all_start_methods()
+ if sys.platform == 'win32':
+ self.assertEqual(methods, ['spawn'])
+ else:
+ self.assertTrue(methods == ['fork', 'spawn'] or
+ methods == ['fork', 'spawn', 'forkserver'])
+
+#
+# Check that killing process does not leak named semaphores
+#
+
+@unittest.skipIf(sys.platform == "win32",
+ "test semantics don't make sense on Windows")
+class TestSemaphoreTracker(unittest.TestCase):
+ def test_semaphore_tracker(self):
+ import subprocess
+ cmd = '''if 1:
+ import multiprocessing as mp, time, os
+ mp.set_start_method("spawn")
+ lock1 = mp.Lock()
+ lock2 = mp.Lock()
+ os.write(%d, lock1._semlock.name.encode("ascii") + b"\\n")
+ os.write(%d, lock2._semlock.name.encode("ascii") + b"\\n")
+ time.sleep(10)
+ '''
+ r, w = os.pipe()
+ p = subprocess.Popen([sys.executable,
+ '-c', cmd % (w, w)],
+ pass_fds=[w],
+ stderr=subprocess.PIPE)
+ os.close(w)
+ with open(r, 'rb', closefd=True) as f:
+ name1 = f.readline().rstrip().decode('ascii')
+ name2 = f.readline().rstrip().decode('ascii')
+ _multiprocessing.sem_unlink(name1)
+ p.terminate()
+ p.wait()
+ time.sleep(2.0)
+ with self.assertRaises(OSError) as ctx:
+ _multiprocessing.sem_unlink(name2)
+ # docs say it should be ENOENT, but OSX seems to give EINVAL
+ self.assertIn(ctx.exception.errno, (errno.ENOENT, errno.EINVAL))
+ err = p.stderr.read().decode('utf-8')
+ p.stderr.close()
+ expected = 'semaphore_tracker: There appear to be 2 leaked semaphores'
+ self.assertRegex(err, expected)
+ self.assertRegex(err, 'semaphore_tracker: %r: \[Errno' % name1)
+
#
-#
+# Mixins
#
-def setUpModule():
- if sys.platform.startswith("linux"):
- try:
- lock = multiprocessing.RLock()
- except OSError:
- raise unittest.SkipTest("OSError raises on RLock creation, "
- "see issue 3111!")
- check_enough_semaphores()
- util.get_temp_dir() # creates temp directory for use by all processes
- multiprocessing.get_logger().setLevel(LOG_LEVEL)
+class ProcessesMixin(object):
+ TYPE = 'processes'
+ Process = multiprocessing.Process
+ connection = multiprocessing.connection
+ current_process = staticmethod(multiprocessing.current_process)
+ active_children = staticmethod(multiprocessing.active_children)
+ Pool = staticmethod(multiprocessing.Pool)
+ Pipe = staticmethod(multiprocessing.Pipe)
+ Queue = staticmethod(multiprocessing.Queue)
+ JoinableQueue = staticmethod(multiprocessing.JoinableQueue)
+ Lock = staticmethod(multiprocessing.Lock)
+ RLock = staticmethod(multiprocessing.RLock)
+ Semaphore = staticmethod(multiprocessing.Semaphore)
+ BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore)
+ Condition = staticmethod(multiprocessing.Condition)
+ Event = staticmethod(multiprocessing.Event)
+ Barrier = staticmethod(multiprocessing.Barrier)
+ Value = staticmethod(multiprocessing.Value)
+ Array = staticmethod(multiprocessing.Array)
+ RawValue = staticmethod(multiprocessing.RawValue)
+ RawArray = staticmethod(multiprocessing.RawArray)
+
+
+class ManagerMixin(object):
+ TYPE = 'manager'
+ Process = multiprocessing.Process
+ Queue = property(operator.attrgetter('manager.Queue'))
+ JoinableQueue = property(operator.attrgetter('manager.JoinableQueue'))
+ Lock = property(operator.attrgetter('manager.Lock'))
+ RLock = property(operator.attrgetter('manager.RLock'))
+ Semaphore = property(operator.attrgetter('manager.Semaphore'))
+ BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore'))
+ Condition = property(operator.attrgetter('manager.Condition'))
+ Event = property(operator.attrgetter('manager.Event'))
+ Barrier = property(operator.attrgetter('manager.Barrier'))
+ Value = property(operator.attrgetter('manager.Value'))
+ Array = property(operator.attrgetter('manager.Array'))
+ list = property(operator.attrgetter('manager.list'))
+ dict = property(operator.attrgetter('manager.dict'))
+ Namespace = property(operator.attrgetter('manager.Namespace'))
+
+ @classmethod
+ def Pool(cls, *args, **kwds):
+ return cls.manager.Pool(*args, **kwds)
+
+ @classmethod
+ def setUpClass(cls):
+ cls.manager = multiprocessing.Manager()
+
+ @classmethod
+ def tearDownClass(cls):
+ # only the manager process should be returned by active_children()
+ # but this can take a bit on slow machines, so wait a few seconds
+ # if there are other children too (see #17395)
+ t = 0.01
+ while len(multiprocessing.active_children()) > 1 and t < 5:
+ time.sleep(t)
+ t *= 2
+ gc.collect() # do garbage collection
+ if cls.manager._number_of_objects() != 0:
+ # This is not really an error since some tests do not
+ # ensure that all processes which hold a reference to a
+ # managed object have been joined.
+ print('Shared objects which still exist at manager shutdown:')
+ print(cls.manager._debug_info())
+ cls.manager.shutdown()
+ cls.manager.join()
+ cls.manager = None
-def tearDownModule():
- # pause a bit so we don't get warning about dangling threads/processes
- time.sleep(0.5)
+class ThreadsMixin(object):
+ TYPE = 'threads'
+ Process = multiprocessing.dummy.Process
+ connection = multiprocessing.dummy.connection
+ current_process = staticmethod(multiprocessing.dummy.current_process)
+ active_children = staticmethod(multiprocessing.dummy.active_children)
+ Pool = staticmethod(multiprocessing.Pool)
+ Pipe = staticmethod(multiprocessing.dummy.Pipe)
+ Queue = staticmethod(multiprocessing.dummy.Queue)
+ JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue)
+ Lock = staticmethod(multiprocessing.dummy.Lock)
+ RLock = staticmethod(multiprocessing.dummy.RLock)
+ Semaphore = staticmethod(multiprocessing.dummy.Semaphore)
+ BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore)
+ Condition = staticmethod(multiprocessing.dummy.Condition)
+ Event = staticmethod(multiprocessing.dummy.Event)
+ Barrier = staticmethod(multiprocessing.dummy.Barrier)
+ Value = staticmethod(multiprocessing.dummy.Value)
+ Array = staticmethod(multiprocessing.dummy.Array)
+#
+# Functions used to create test cases from the base ones in this module
+#
+
+def install_tests_in_module_dict(remote_globs, start_method):
+ __module__ = remote_globs['__name__']
+ local_globs = globals()
+ ALL_TYPES = {'processes', 'threads', 'manager'}
-if __name__ == '__main__':
- unittest.main()
+ for name, base in local_globs.items():
+ if not isinstance(base, type):
+ continue
+ if issubclass(base, BaseTestCase):
+ if base is BaseTestCase:
+ continue
+ assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES
+ for type_ in base.ALLOWED_TYPES:
+ newname = 'With' + type_.capitalize() + name[1:]
+ Mixin = local_globs[type_.capitalize() + 'Mixin']
+ class Temp(base, Mixin, unittest.TestCase):
+ pass
+ Temp.__name__ = Temp.__qualname__ = newname
+ Temp.__module__ = __module__
+ remote_globs[newname] = Temp
+ elif issubclass(base, unittest.TestCase):
+ class Temp(base, object):
+ pass
+ Temp.__name__ = Temp.__qualname__ = name
+ Temp.__module__ = __module__
+ remote_globs[name] = Temp
+
+ dangling = [None, None]
+ old_start_method = [None]
+
+ def setUpModule():
+ multiprocessing.set_forkserver_preload(PRELOAD)
+ multiprocessing.process._cleanup()
+ dangling[0] = multiprocessing.process._dangling.copy()
+ dangling[1] = threading._dangling.copy()
+ old_start_method[0] = multiprocessing.get_start_method(allow_none=True)
+ try:
+ multiprocessing.set_start_method(start_method, force=True)
+ except ValueError:
+ raise unittest.SkipTest(start_method +
+ ' start method not supported')
+
+ if sys.platform.startswith("linux"):
+ try:
+ lock = multiprocessing.RLock()
+ except OSError:
+ raise unittest.SkipTest("OSError raises on RLock creation, "
+ "see issue 3111!")
+ check_enough_semaphores()
+ util.get_temp_dir() # creates temp directory
+ multiprocessing.get_logger().setLevel(LOG_LEVEL)
+
+ def tearDownModule():
+ multiprocessing.set_start_method(old_start_method[0], force=True)
+ # pause a bit so we don't get warning about dangling threads/processes
+ time.sleep(0.5)
+ multiprocessing.process._cleanup()
+ gc.collect()
+ tmp = set(multiprocessing.process._dangling) - set(dangling[0])
+ if tmp:
+ print('Dangling processes:', tmp, file=sys.stderr)
+ del tmp
+ tmp = set(threading._dangling) - set(dangling[1])
+ if tmp:
+ print('Dangling threads:', tmp, file=sys.stderr)
+
+ remote_globs['setUpModule'] = setUpModule
+ remote_globs['tearDownModule'] = tearDownModule
diff --git a/Lib/test/audiodata/pluck-pcm24.au b/Lib/test/audiodata/pluck-pcm24.au
new file mode 100644
index 0000000..0bb2304
--- /dev/null
+++ b/Lib/test/audiodata/pluck-pcm24.au
Binary files differ
diff --git a/Lib/test/audiotests.py b/Lib/test/audiotests.py
index 6ff5bb8..b7497ce 100644
--- a/Lib/test/audiotests.py
+++ b/Lib/test/audiotests.py
@@ -12,24 +12,6 @@ class UnseekableIO(io.FileIO):
def seek(self, *args, **kwargs):
raise io.UnsupportedOperation
-def byteswap2(data):
- a = array.array('h')
- a.frombytes(data)
- a.byteswap()
- return a.tobytes()
-
-def byteswap3(data):
- ba = bytearray(data)
- ba[::3] = data[2::3]
- ba[2::3] = data[::3]
- return bytes(ba)
-
-def byteswap4(data):
- a = array.array('i')
- a.frombytes(data)
- a.byteswap()
- return a.tobytes()
-
class AudioTests:
close_fd = False
@@ -56,6 +38,12 @@ class AudioTests:
params = f.getparams()
self.assertEqual(params,
(nchannels, sampwidth, framerate, nframes, comptype, compname))
+ self.assertEqual(params.nchannels, nchannels)
+ self.assertEqual(params.sampwidth, sampwidth)
+ self.assertEqual(params.framerate, framerate)
+ self.assertEqual(params.nframes, nframes)
+ self.assertEqual(params.comptype, comptype)
+ self.assertEqual(params.compname, compname)
dump = pickle.dumps(params)
self.assertEqual(pickle.loads(dump), params)
@@ -72,15 +60,12 @@ class AudioWriteTests(AudioTests):
return f
def check_file(self, testfile, nframes, frames):
- f = self.module.open(testfile, 'rb')
- try:
+ with self.module.open(testfile, 'rb') as f:
self.assertEqual(f.getnchannels(), self.nchannels)
self.assertEqual(f.getsampwidth(), self.sampwidth)
self.assertEqual(f.getframerate(), self.framerate)
self.assertEqual(f.getnframes(), nframes)
self.assertEqual(f.readframes(nframes), frames)
- finally:
- f.close()
def test_write_params(self):
f = self.create_file(TESTFN)
@@ -90,6 +75,53 @@ class AudioWriteTests(AudioTests):
self.nframes, self.comptype, self.compname)
f.close()
+ def test_write_context_manager_calls_close(self):
+ # Close checks for a minimum header and will raise an error
+ # if it is not set, so this proves that close is called.
+ with self.assertRaises(self.module.Error):
+ with self.module.open(TESTFN, 'wb'):
+ pass
+ with self.assertRaises(self.module.Error):
+ with open(TESTFN, 'wb') as testfile:
+ with self.module.open(testfile):
+ pass
+
+ def test_context_manager_with_open_file(self):
+ with open(TESTFN, 'wb') as testfile:
+ with self.module.open(testfile) as f:
+ f.setnchannels(self.nchannels)
+ f.setsampwidth(self.sampwidth)
+ f.setframerate(self.framerate)
+ f.setcomptype(self.comptype, self.compname)
+ self.assertEqual(testfile.closed, self.close_fd)
+ with open(TESTFN, 'rb') as testfile:
+ with self.module.open(testfile) as f:
+ self.assertFalse(f.getfp().closed)
+ params = f.getparams()
+ self.assertEqual(params.nchannels, self.nchannels)
+ self.assertEqual(params.sampwidth, self.sampwidth)
+ self.assertEqual(params.framerate, self.framerate)
+ if not self.close_fd:
+ self.assertIsNone(f.getfp())
+ self.assertEqual(testfile.closed, self.close_fd)
+
+ def test_context_manager_with_filename(self):
+ # If the file doesn't get closed, this test won't fail, but it will
+ # produce a resource leak warning.
+ with self.module.open(TESTFN, 'wb') as f:
+ f.setnchannels(self.nchannels)
+ f.setsampwidth(self.sampwidth)
+ f.setframerate(self.framerate)
+ f.setcomptype(self.comptype, self.compname)
+ with self.module.open(TESTFN) as f:
+ self.assertFalse(f.getfp().closed)
+ params = f.getparams()
+ self.assertEqual(params.nchannels, self.nchannels)
+ self.assertEqual(params.sampwidth, self.sampwidth)
+ self.assertEqual(params.framerate, self.framerate)
+ if not self.close_fd:
+ self.assertIsNone(f.getfp())
+
def test_write(self):
f = self.create_file(TESTFN)
f.setnframes(self.nframes)
@@ -98,6 +130,30 @@ class AudioWriteTests(AudioTests):
self.check_file(TESTFN, self.nframes, self.frames)
+ def test_write_bytearray(self):
+ f = self.create_file(TESTFN)
+ f.setnframes(self.nframes)
+ f.writeframes(bytearray(self.frames))
+ f.close()
+
+ self.check_file(TESTFN, self.nframes, self.frames)
+
+ def test_write_array(self):
+ f = self.create_file(TESTFN)
+ f.setnframes(self.nframes)
+ f.writeframes(array.array('h', self.frames))
+ f.close()
+
+ self.check_file(TESTFN, self.nframes, self.frames)
+
+ def test_write_memoryview(self):
+ f = self.create_file(TESTFN)
+ f.setnframes(self.nframes)
+ f.writeframes(memoryview(self.frames))
+ f.close()
+
+ self.check_file(TESTFN, self.nframes, self.frames)
+
def test_incompleted_write(self):
with open(TESTFN, 'wb') as testfile:
testfile.write(b'ababagalamaga')
@@ -137,20 +193,18 @@ class AudioWriteTests(AudioTests):
self.check_file(testfile, self.nframes, self.frames)
def test_unseekable_read(self):
- f = self.create_file(TESTFN)
- f.setnframes(self.nframes)
- f.writeframes(self.frames)
- f.close()
+ with self.create_file(TESTFN) as f:
+ f.setnframes(self.nframes)
+ f.writeframes(self.frames)
with UnseekableIO(TESTFN, 'rb') as testfile:
self.check_file(testfile, self.nframes, self.frames)
def test_unseekable_write(self):
with UnseekableIO(TESTFN, 'wb') as testfile:
- f = self.create_file(testfile)
- f.setnframes(self.nframes)
- f.writeframes(self.frames)
- f.close()
+ with self.create_file(testfile) as f:
+ f.setnframes(self.nframes)
+ f.writeframes(self.frames)
self.check_file(TESTFN, self.nframes, self.frames)
@@ -267,12 +321,9 @@ class AudioTestsWithSourceFile(AudioTests):
with open(TESTFN, 'rb') as testfile:
self.assertEqual(testfile.read(13), b'ababagalamaga')
- f = self.module.open(testfile, 'rb')
- try:
+ with self.module.open(testfile, 'rb') as f:
self.assertEqual(f.getnchannels(), self.nchannels)
self.assertEqual(f.getsampwidth(), self.sampwidth)
self.assertEqual(f.getframerate(), self.framerate)
self.assertEqual(f.getnframes(), self.sndfilenframes)
self.assertEqual(f.readframes(self.nframes), self.frames)
- finally:
- f.close()
diff --git a/Lib/test/badsyntax_future10.py b/Lib/test/badsyntax_future10.py
new file mode 100644
index 0000000..fa5ab67
--- /dev/null
+++ b/Lib/test/badsyntax_future10.py
@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+"spam, bar, blah"
+from __future__ import print_function
diff --git a/Lib/test/buffer_tests.py b/Lib/test/buffer_tests.py
index cf54c28..0a62940 100644
--- a/Lib/test/buffer_tests.py
+++ b/Lib/test/buffer_tests.py
@@ -160,14 +160,20 @@ class MixinBytesBufferCommonTests(object):
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8))
self.assertEqual(b'abc\rab def\ng hi',
self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(4))
+ self.assertEqual(b'abc\r\nab def\ng hi',
+ self.marshal(b'abc\r\nab\tdef\ng\thi').expandtabs())
+ self.assertEqual(b'abc\r\nab def\ng hi',
+ self.marshal(b'abc\r\nab\tdef\ng\thi').expandtabs(8))
self.assertEqual(b'abc\r\nab def\ng hi',
self.marshal(b'abc\r\nab\tdef\ng\thi').expandtabs(4))
- self.assertEqual(b'abc\rab def\ng hi',
- self.marshal(b'abc\rab\tdef\ng\thi').expandtabs())
- self.assertEqual(b'abc\rab def\ng hi',
- self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8))
self.assertEqual(b'abc\r\nab\r\ndef\ng\r\nhi',
- self.marshal(b'abc\r\nab\r\ndef\ng\r\nhi').expandtabs(4))
+ self.marshal(b'abc\r\nab\r\ndef\ng\r\nhi').expandtabs(4))
+ # check keyword args
+ self.assertEqual(b'abc\rab def\ng hi',
+ self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(tabsize=8))
+ self.assertEqual(b'abc\rab def\ng hi',
+ self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(tabsize=4))
+
self.assertEqual(b' a\n b', self.marshal(b' \ta\n\tb').expandtabs(1))
self.assertRaises(TypeError, self.marshal(b'hello').expandtabs, 42, 42)
diff --git a/Lib/test/bytecode_helper.py b/Lib/test/bytecode_helper.py
new file mode 100644
index 0000000..58b4209
--- /dev/null
+++ b/Lib/test/bytecode_helper.py
@@ -0,0 +1,41 @@
+"""bytecode_helper - support tools for testing correct bytecode generation"""
+
+import unittest
+import dis
+import io
+
+_UNSPECIFIED = object()
+
+class BytecodeTestCase(unittest.TestCase):
+ """Custom assertion methods for inspecting bytecode."""
+
+ def get_disassembly_as_string(self, co):
+ s = io.StringIO()
+ dis.dis(co, file=s)
+ return s.getvalue()
+
+ def assertInBytecode(self, x, opname, argval=_UNSPECIFIED):
+ """Returns instr if op is found, otherwise throws AssertionError"""
+ for instr in dis.get_instructions(x):
+ if instr.opname == opname:
+ if argval is _UNSPECIFIED or instr.argval == argval:
+ return instr
+ disassembly = self.get_disassembly_as_string(x)
+ if argval is _UNSPECIFIED:
+ msg = '%s not found in bytecode:\n%s' % (opname, disassembly)
+ else:
+ msg = '(%s,%r) not found in bytecode:\n%s'
+ msg = msg % (opname, argval, disassembly)
+ self.fail(msg)
+
+ def assertNotInBytecode(self, x, opname, argval=_UNSPECIFIED):
+ """Throws AssertionError if op is found"""
+ for instr in dis.get_instructions(x):
+ if instr.opname == opname:
+ disassembly = self.get_disassembly_as_string(co)
+ if opargval is _UNSPECIFIED:
+ msg = '%s occurs in bytecode:\n%s' % (opname, disassembly)
+ elif instr.argval == argval:
+ msg = '(%s,%r) occurs in bytecode:\n%s'
+ msg = msg % (opname, argval, disassembly)
+ self.fail(msg)
diff --git a/Lib/test/coding20731.py b/Lib/test/coding20731.py
index ca4962e..b0e227a 100644
--- a/Lib/test/coding20731.py
+++ b/Lib/test/coding20731.py
@@ -1,4 +1,4 @@
-#coding:latin1
-
-
-
+#coding:latin1
+
+
+
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index ab55476..3f3c60a 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -619,6 +619,10 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase):
eq(td(hours=-.2/us_per_hour), td(0))
eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1))
+ # Test for a patch in Issue 8860
+ eq(td(microseconds=0.5), 0.5*td(microseconds=1.0))
+ eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution)
+
def test_massive_normalization(self):
td = timedelta(microseconds=-1)
self.assertEqual((td.days, td.seconds, td.microseconds),
diff --git a/Lib/test/final_a.py b/Lib/test/final_a.py
new file mode 100644
index 0000000..390ee88
--- /dev/null
+++ b/Lib/test/final_a.py
@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.final_b
+
+x = 'a'
+
+class C:
+ def __del__(self):
+ # Inspect module globals and builtins
+ print("x =", x)
+ print("final_b.x =", test.final_b.x)
+ print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+ print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()
diff --git a/Lib/test/final_b.py b/Lib/test/final_b.py
new file mode 100644
index 0000000..7228d82
--- /dev/null
+++ b/Lib/test/final_b.py
@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.final_a
+
+x = 'b'
+
+class C:
+ def __del__(self):
+ # Inspect module globals and builtins
+ print("x =", x)
+ print("final_a.x =", test.final_a.x)
+ print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+ print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()
diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py
index 88527df..19b54ec 100644
--- a/Lib/test/fork_wait.py
+++ b/Lib/test/fork_wait.py
@@ -28,7 +28,7 @@ class ForkWait(unittest.TestCase):
self.alive[id] = os.getpid()
try:
time.sleep(SHORTSLEEP)
- except IOError:
+ except OSError:
pass
def wait_impl(self, cpid):
diff --git a/Lib/test/keycert3.pem b/Lib/test/keycert3.pem
new file mode 100644
index 0000000..5bfa62c
--- /dev/null
+++ b/Lib/test/keycert3.pem
@@ -0,0 +1,73 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP
+jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM
+9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ
+aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe
+yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j
+y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+
+AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW
+5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL
+9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9
+1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT
+DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh
+1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m
+JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3
+RnJdHOMXWem7/w==
+-----END PRIVATE KEY-----
+Certificate:
+ Data:
+ Version: 1 (0x0)
+ Serial Number: 12723342612721443281 (0xb09264b1f2da21d1)
+ Signature Algorithm: sha1WithRSAEncryption
+ Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Validity
+ Not Before: Jan 4 19:47:07 2013 GMT
+ Not After : Nov 13 19:47:07 2022 GMT
+ Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost
+ Subject Public Key Info:
+ Public Key Algorithm: rsaEncryption
+ Public-Key: (1024 bit)
+ Modulus:
+ 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d:
+ 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb:
+ c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99:
+ 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c:
+ f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93:
+ 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23:
+ f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5:
+ af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6:
+ 21:82:a5:3c:88:e5:be:1b:b1
+ Exponent: 65537 (0x10001)
+ Signature Algorithm: sha1WithRSAEncryption
+ 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a:
+ e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93:
+ f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13:
+ e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92:
+ d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59:
+ 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8:
+ ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1:
+ 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75:
+ 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96:
+ 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48:
+ 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a:
+ f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6:
+ 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41:
+ a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb:
+ fc:a9:94:71
+-----BEGIN CERTIFICATE-----
+MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY
+WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV
+BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3
+WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV
+BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv
+c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C
+tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola
+N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1
+TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR
+iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG
+xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo
+5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv
+mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF
+YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh
+2EJ36/yplHE=
+-----END CERTIFICATE-----
diff --git a/Lib/test/keycert4.pem b/Lib/test/keycert4.pem
new file mode 100644
index 0000000..53355c8
--- /dev/null
+++ b/Lib/test/keycert4.pem
@@ -0,0 +1,73 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv
+L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2
+NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1
+L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L
+pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de
+R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9
+myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT
+drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS
+Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx
+i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK
+Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu
+JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN
++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/
+e83Gq6ffLVfKNQ==
+-----END PRIVATE KEY-----
+Certificate:
+ Data:
+ Version: 1 (0x0)
+ Serial Number: 12723342612721443282 (0xb09264b1f2da21d2)
+ Signature Algorithm: sha1WithRSAEncryption
+ Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Validity
+ Not Before: Jan 4 19:47:07 2013 GMT
+ Not After : Nov 13 19:47:07 2022 GMT
+ Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname
+ Subject Public Key Info:
+ Public Key Algorithm: rsaEncryption
+ Public-Key: (1024 bit)
+ Modulus:
+ 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14:
+ 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9:
+ cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a:
+ b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76:
+ 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7:
+ 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a:
+ d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79:
+ 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd:
+ 81:7e:bd:1b:ae:0b:5d:c6:39
+ Exponent: 65537 (0x10001)
+ Signature Algorithm: sha1WithRSAEncryption
+ ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0:
+ 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f:
+ 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56:
+ 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15:
+ 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1:
+ 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb:
+ 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e:
+ e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18:
+ d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1:
+ af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53:
+ 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68:
+ 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da:
+ 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92:
+ 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa:
+ 49:12:1e:ce
+-----BEGIN CERTIFICATE-----
+MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY
+WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV
+BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3
+WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV
+BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z
+dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU
+aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0
+ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ
+hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v
+xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338
+Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP
+XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0
+UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz
+aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb
+oF+6ufu6+kkSHs4=
+-----END CERTIFICATE-----
diff --git a/Lib/test/leakers/test_gestalt.py b/Lib/test/leakers/test_gestalt.py
deleted file mode 100644
index e0081c1..0000000
--- a/Lib/test/leakers/test_gestalt.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import sys
-
-if sys.platform != 'darwin':
- raise ValueError("This test only leaks on Mac OS X")
-
-def leak():
- # taken from platform._mac_ver_lookup()
- from gestalt import gestalt
- import MacOS
-
- try:
- gestalt('sysu')
- except MacOS.Error:
- pass
diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py
index bfbf44e..1cbcea2 100644
--- a/Lib/test/lock_tests.py
+++ b/Lib/test/lock_tests.py
@@ -80,6 +80,11 @@ class BaseLockTests(BaseTestCase):
lock = self.locktype()
del lock
+ def test_repr(self):
+ lock = self.locktype()
+ repr(lock)
+ del lock
+
def test_acquire_destroy(self):
lock = self.locktype()
lock.acquire()
@@ -413,6 +418,17 @@ class ConditionTests(BaseTestCase):
self.assertRaises(RuntimeError, cond.notify)
def _check_notify(self, cond):
+ # Note that this test is sensitive to timing. If the worker threads
+ # don't execute in a timely fashion, the main thread may think they
+ # are further along then they are. The main thread therefore issues
+ # _wait() statements to try to make sure that it doesn't race ahead
+ # of the workers.
+ # Secondly, this test assumes that condition variables are not subject
+ # to spurious wakeups. The absence of spurious wakeups is an implementation
+ # detail of Condition Cariables in current CPython, but in general, not
+ # a guaranteed property of condition variables as a programming
+ # construct. In particular, it is possible that this can no longer
+ # be conveniently guaranteed should their implementation ever change.
N = 5
results1 = []
results2 = []
@@ -440,6 +456,9 @@ class ConditionTests(BaseTestCase):
_wait()
self.assertEqual(results1, [(True, 1)] * 3)
self.assertEqual(results2, [])
+ # first wait, to ensure all workers settle into cond.wait() before
+ # we continue. See issue #8799
+ _wait()
# Notify 5 threads: they might be in their first or second wait
cond.acquire()
cond.notify(5)
@@ -450,6 +469,7 @@ class ConditionTests(BaseTestCase):
_wait()
self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2)
self.assertEqual(results2, [(True, 2)] * 3)
+ _wait() # make sure all workers settle into cond.wait()
# Notify all threads: they are all in their second wait
cond.acquire()
cond.notify_all()
diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py
index 48d2e57..4251d55 100644
--- a/Lib/test/make_ssl_certs.py
+++ b/Lib/test/make_ssl_certs.py
@@ -2,6 +2,7 @@
and friends."""
import os
+import shutil
import sys
import tempfile
from subprocess import *
@@ -20,11 +21,54 @@ req_template = """
[req_x509_extensions]
subjectAltName = DNS:{hostname}
+
+ [ ca ]
+ default_ca = CA_default
+
+ [ CA_default ]
+ dir = cadir
+ database = $dir/index.txt
+ crlnumber = $dir/crl.txt
+ default_md = sha1
+ default_days = 3600
+ default_crl_days = 3600
+ certificate = pycacert.pem
+ private_key = pycakey.pem
+ serial = $dir/serial
+ RANDFILE = $dir/.rand
+
+ policy = policy_match
+
+ [ policy_match ]
+ countryName = match
+ stateOrProvinceName = optional
+ organizationName = match
+ organizationalUnitName = optional
+ commonName = supplied
+ emailAddress = optional
+
+ [ policy_anything ]
+ countryName = optional
+ stateOrProvinceName = optional
+ localityName = optional
+ organizationName = optional
+ organizationalUnitName = optional
+ commonName = supplied
+ emailAddress = optional
+
+
+ [ v3_ca ]
+
+ subjectKeyIdentifier=hash
+ authorityKeyIdentifier=keyid:always,issuer
+ basicConstraints = CA:true
+
"""
here = os.path.abspath(os.path.dirname(__file__))
-def make_cert_key(hostname):
+def make_cert_key(hostname, sign=False):
+ print("creating cert for " + hostname)
tempnames = []
for i in range(3):
with tempfile.NamedTemporaryFile(delete=False) as f:
@@ -33,10 +77,25 @@ def make_cert_key(hostname):
try:
with open(req_file, 'w') as f:
f.write(req_template.format(hostname=hostname))
- args = ['req', '-new', '-days', '3650', '-nodes', '-x509',
+ args = ['req', '-new', '-days', '3650', '-nodes',
'-newkey', 'rsa:1024', '-keyout', key_file,
- '-out', cert_file, '-config', req_file]
+ '-config', req_file]
+ if sign:
+ with tempfile.NamedTemporaryFile(delete=False) as f:
+ tempnames.append(f.name)
+ reqfile = f.name
+ args += ['-out', reqfile ]
+
+ else:
+ args += ['-x509', '-out', cert_file ]
check_call(['openssl'] + args)
+
+ if sign:
+ args = ['ca', '-config', req_file, '-out', cert_file, '-outdir', 'cadir',
+ '-policy', 'policy_anything', '-batch', '-infiles', reqfile ]
+ check_call(['openssl'] + args)
+
+
with open(cert_file, 'r') as f:
cert = f.read()
with open(key_file, 'r') as f:
@@ -46,6 +105,36 @@ def make_cert_key(hostname):
for name in tempnames:
os.remove(name)
+TMP_CADIR = 'cadir'
+
+def unmake_ca():
+ shutil.rmtree(TMP_CADIR)
+
+def make_ca():
+ os.mkdir(TMP_CADIR)
+ with open(os.path.join('cadir','index.txt'),'a+') as f:
+ pass # empty file
+ with open(os.path.join('cadir','crl.txt'),'a+') as f:
+ r.write("00")
+ with open(os.path.join('cadir','index.txt.attr'),'w+') as f:
+ f.write('unique_subject = no')
+
+ with tempfile.NamedTemporaryFile("w") as t:
+ t.write(req_template.format(hostname='our-ca-server'))
+ t.flush()
+ with tempfile.NamedTemporaryFile() as f:
+ args = ['req', '-new', '-days', '3650', '-extensions', 'v3_ca', '-nodes',
+ '-newkey', 'rsa:2048', '-keyout', 'pycakey.pem',
+ '-out', f.name,
+ '-subj', '/C=XY/L=Castle Anthrax/O=Python Software Foundation CA/CN=our-ca-server']
+ check_call(['openssl'] + args)
+ args = ['ca', '-config', t.name, '-create_serial',
+ '-out', 'pycacert.pem', '-batch', '-outdir', TMP_CADIR,
+ '-keyfile', 'pycakey.pem', '-days', '3650',
+ '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name ]
+ check_call(['openssl'] + args)
+ args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl']
+ check_call(['openssl'] + args)
if __name__ == '__main__':
os.chdir(here)
@@ -54,11 +143,34 @@ if __name__ == '__main__':
f.write(cert)
with open('ssl_key.pem', 'w') as f:
f.write(key)
+ print("password protecting ssl_key.pem in ssl_key.passwd.pem")
+ check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass'])
+ check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass'])
+
with open('keycert.pem', 'w') as f:
f.write(key)
f.write(cert)
+
+ with open('keycert.passwd.pem', 'a+') as f:
+ f.write(cert)
+
# For certificate matching tests
+ make_ca()
cert, key = make_cert_key('fakehostname')
with open('keycert2.pem', 'w') as f:
f.write(key)
f.write(cert)
+
+ cert, key = make_cert_key('localhost', True)
+ with open('keycert3.pem', 'w') as f:
+ f.write(key)
+ f.write(cert)
+
+ cert, key = make_cert_key('fakehostname', True)
+ with open('keycert4.pem', 'w') as f:
+ f.write(key)
+ f.write(cert)
+
+ unmake_ca()
+ print("\n\nPlease change the values in test_ssl.py, test_parse_cert function related to notAfter,notBefore and serialNumber")
+ check_call(['openssl','x509','-in','keycert.pem','-dates','-serial','-noout'])
diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py
index 861bfb2..e36724f 100644
--- a/Lib/test/mock_socket.py
+++ b/Lib/test/mock_socket.py
@@ -145,12 +145,8 @@ def gethostbyname(name):
return ""
-class gaierror(Exception):
- pass
-
-
-class error(Exception):
- pass
+gaierror = socket_module.gaierror
+error = socket_module.error
# Constants
diff --git a/Lib/test/mp_fork_bomb.py b/Lib/test/mp_fork_bomb.py
index 908afe3..017e010 100644
--- a/Lib/test/mp_fork_bomb.py
+++ b/Lib/test/mp_fork_bomb.py
@@ -7,6 +7,11 @@ def foo():
# correctly on Windows. However, we should get a RuntimeError rather
# than the Windows equivalent of a fork bomb.
+if len(sys.argv) > 1:
+ multiprocessing.set_start_method(sys.argv[1])
+else:
+ multiprocessing.set_start_method('spawn')
+
p = multiprocessing.Process(target=foo)
p.start()
p.join()
diff --git a/Lib/test/multibytecodec_support.py b/Lib/test/multibytecodec_support.py
index 14fea3e..51ca3bd 100644
--- a/Lib/test/multibytecodec_support.py
+++ b/Lib/test/multibytecodec_support.py
@@ -281,7 +281,7 @@ class TestBase_Mapping(unittest.TestCase):
unittest.TestCase.__init__(self, *args, **kw)
try:
self.open_mapping_file().close() # test it to report the error early
- except (IOError, HTTPException):
+ except (OSError, HTTPException):
self.skipTest("Could not retrieve "+self.mapfileurl)
def open_mapping_file(self):
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 4d59bde..a2bea60 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1,9 +1,11 @@
+import copyreg
import io
-import unittest
import pickle
import pickletools
+import random
+import struct
import sys
-import copyreg
+import unittest
import weakref
from http.cookies import SimpleCookie
@@ -93,6 +95,9 @@ class E(C):
def __getinitargs__(self):
return ()
+class H(object):
+ pass
+
import __main__
__main__.C = C
C.__module__ = "__main__"
@@ -100,6 +105,8 @@ __main__.D = D
D.__module__ = "__main__"
__main__.E = E
E.__module__ = "__main__"
+__main__.H = H
+H.__module__ = "__main__"
class myint(int):
def __init__(self, x):
@@ -491,31 +498,52 @@ def create_data():
x.append(5)
return x
+
class AbstractPickleTests(unittest.TestCase):
# Subclass must define self.dumps, self.loads.
+ optimized = False
+
_testdata = create_data()
def setUp(self):
pass
+ def assert_is_copy(self, obj, objcopy, msg=None):
+ """Utility method to verify if two objects are copies of each others.
+ """
+ if msg is None:
+ msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
+ self.assertEqual(obj, objcopy, msg=msg)
+ self.assertIs(type(obj), type(objcopy), msg=msg)
+ if hasattr(obj, '__dict__'):
+ self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
+ self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
+ if hasattr(obj, '__slots__'):
+ self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
+ for slot in obj.__slots__:
+ self.assertEqual(
+ hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
+ self.assertEqual(getattr(obj, slot, None),
+ getattr(objcopy, slot, None), msg=msg)
+
def test_misc(self):
# test various datatypes not tested by testdata
for proto in protocols:
x = myint(4)
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
x = (1, ())
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
x = initarg(1, x)
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
# XXX test __reduce__ protocol?
@@ -524,16 +552,16 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(expected, proto)
got = self.loads(s)
- self.assertEqual(expected, got)
+ self.assert_is_copy(expected, got)
def test_load_from_data0(self):
- self.assertEqual(self._testdata, self.loads(DATA0))
+ self.assert_is_copy(self._testdata, self.loads(DATA0))
def test_load_from_data1(self):
- self.assertEqual(self._testdata, self.loads(DATA1))
+ self.assert_is_copy(self._testdata, self.loads(DATA1))
def test_load_from_data2(self):
- self.assertEqual(self._testdata, self.loads(DATA2))
+ self.assert_is_copy(self._testdata, self.loads(DATA2))
def test_load_classic_instance(self):
# See issue5180. Test loading 2.x pickles that
@@ -555,7 +583,7 @@ class AbstractPickleTests(unittest.TestCase):
b"X\n"
b"p0\n"
b"(dp1\nb.").replace(b'X', xname)
- self.assertEqual(X(*args), self.loads(pickle0))
+ self.assert_is_copy(X(*args), self.loads(pickle0))
# Protocol 1 (binary mode pickle)
"""
@@ -572,7 +600,7 @@ class AbstractPickleTests(unittest.TestCase):
pickle1 = (b'(c__main__\n'
b'X\n'
b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
- self.assertEqual(X(*args), self.loads(pickle1))
+ self.assert_is_copy(X(*args), self.loads(pickle1))
# Protocol 2 (pickle2 = b'\x80\x02' + pickle1)
"""
@@ -590,7 +618,7 @@ class AbstractPickleTests(unittest.TestCase):
pickle2 = (b'\x80\x02(c__main__\n'
b'X\n'
b'q\x00oq\x01}q\x02b.').replace(b'X', xname)
- self.assertEqual(X(*args), self.loads(pickle2))
+ self.assert_is_copy(X(*args), self.loads(pickle2))
# There are gratuitous differences between pickles produced by
# pickle and cPickle, largely because cPickle starts PUT indices at
@@ -615,6 +643,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(l, proto)
x = self.loads(s)
+ self.assertIsInstance(x, list)
self.assertEqual(len(x), 1)
self.assertTrue(x is x[0])
@@ -624,6 +653,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(t, proto)
x = self.loads(s)
+ self.assertIsInstance(x, tuple)
self.assertEqual(len(x), 1)
self.assertEqual(len(x[0]), 1)
self.assertTrue(x is x[0][0])
@@ -634,15 +664,39 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(d, proto)
x = self.loads(s)
+ self.assertIsInstance(x, dict)
self.assertEqual(list(x.keys()), [1])
self.assertTrue(x[1] is x)
+ def test_recursive_set(self):
+ h = H()
+ y = set({h})
+ h.attr = y
+ for proto in protocols:
+ s = self.dumps(y, proto)
+ x = self.loads(s)
+ self.assertIsInstance(x, set)
+ self.assertIs(list(x)[0].attr, x)
+ self.assertEqual(len(x), 1)
+
+ def test_recursive_frozenset(self):
+ h = H()
+ y = frozenset({h})
+ h.attr = y
+ for proto in protocols:
+ s = self.dumps(y, proto)
+ x = self.loads(s)
+ self.assertIsInstance(x, frozenset)
+ self.assertIs(list(x)[0].attr, x)
+ self.assertEqual(len(x), 1)
+
def test_recursive_inst(self):
i = C()
i.attr = i
for proto in protocols:
s = self.dumps(i, proto)
x = self.loads(s)
+ self.assertIsInstance(x, C)
self.assertEqual(dir(x), dir(i))
self.assertIs(x.attr, x)
@@ -655,6 +709,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(l, proto)
x = self.loads(s)
+ self.assertIsInstance(x, list)
self.assertEqual(len(x), 1)
self.assertEqual(dir(x[0]), dir(i))
self.assertEqual(list(x[0].attr.keys()), [1])
@@ -662,31 +717,8 @@ class AbstractPickleTests(unittest.TestCase):
def test_get(self):
self.assertRaises(KeyError, self.loads, b'g0\np0')
- self.assertEqual(self.loads(b'((Kdtp0\nh\x00l.))'), [(100,), (100,)])
-
- def test_insecure_strings(self):
- # XXX Some of these tests are temporarily disabled
- insecure = [b"abc", b"2 + 2", # not quoted
- ## b"'abc' + 'def'", # not a single quoted string
- b"'abc", # quote is not closed
- b"'abc\"", # open quote and close quote don't match
- b"'abc' ?", # junk after close quote
- b"'\\'", # trailing backslash
- # Variations on issue #17710
- b"'",
- b'"',
- b"' ",
- b"' ",
- b"' ",
- b"' ",
- b'" ',
- # some tests of the quoting rules
- ## b"'abc\"\''",
- ## b"'\\\\a\'\'\'\\\'\\\\\''",
- ]
- for b in insecure:
- buf = b"S" + b + b"\012p0\012."
- self.assertRaises(ValueError, self.loads, buf)
+ self.assert_is_copy([(100,), (100,)],
+ self.loads(b'((Kdtp0\nh\x00l.))'))
def test_unicode(self):
endcases = ['', '<\\u>', '<\\\u1234>', '<\n>',
@@ -697,26 +729,26 @@ class AbstractPickleTests(unittest.TestCase):
for u in endcases:
p = self.dumps(u, proto)
u2 = self.loads(p)
- self.assertEqual(u2, u)
+ self.assert_is_copy(u, u2)
def test_unicode_high_plane(self):
t = '\U00012345'
for proto in protocols:
p = self.dumps(t, proto)
t2 = self.loads(p)
- self.assertEqual(t2, t)
+ self.assert_is_copy(t, t2)
def test_bytes(self):
for proto in protocols:
for s in b'', b'xyz', b'xyz'*100:
p = self.dumps(s, proto)
- self.assertEqual(self.loads(p), s)
+ self.assert_is_copy(s, self.loads(p))
for s in [bytes([i]) for i in range(256)]:
p = self.dumps(s, proto)
- self.assertEqual(self.loads(p), s)
+ self.assert_is_copy(s, self.loads(p))
for s in [bytes([i, i]) for i in range(256)]:
p = self.dumps(s, proto)
- self.assertEqual(self.loads(p), s)
+ self.assert_is_copy(s, self.loads(p))
def test_ints(self):
import sys
@@ -726,14 +758,14 @@ class AbstractPickleTests(unittest.TestCase):
for expected in (-n, n):
s = self.dumps(expected, proto)
n2 = self.loads(s)
- self.assertEqual(expected, n2)
+ self.assert_is_copy(expected, n2)
n = n >> 1
def test_maxint64(self):
maxint64 = (1 << 63) - 1
data = b'I' + str(maxint64).encode("ascii") + b'\n.'
got = self.loads(data)
- self.assertEqual(got, maxint64)
+ self.assert_is_copy(maxint64, got)
# Try too with a bogus literal.
data = b'I' + str(maxint64).encode("ascii") + b'JUNK\n.'
@@ -748,7 +780,7 @@ class AbstractPickleTests(unittest.TestCase):
for n in npos, -npos:
pickle = self.dumps(n, proto)
got = self.loads(pickle)
- self.assertEqual(n, got)
+ self.assert_is_copy(n, got)
# Try a monster. This is quadratic-time in protos 0 & 1, so don't
# bother with those.
nbase = int("deadbeeffeedface", 16)
@@ -756,6 +788,10 @@ class AbstractPickleTests(unittest.TestCase):
for n in nbase, -nbase:
p = self.dumps(n, 2)
got = self.loads(p)
+ # assert_is_copy is very expensive here as it precomputes
+ # a failure message by computing the repr() of n and got,
+ # we just do the check ourselves.
+ self.assertIs(type(got), int)
self.assertEqual(n, got)
def test_float(self):
@@ -766,7 +802,7 @@ class AbstractPickleTests(unittest.TestCase):
for value in test_values:
pickle = self.dumps(value, proto)
got = self.loads(pickle)
- self.assertEqual(value, got)
+ self.assert_is_copy(value, got)
@run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
def test_float_format(self):
@@ -774,10 +810,18 @@ class AbstractPickleTests(unittest.TestCase):
self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
def test_reduce(self):
- pass
+ for proto in protocols:
+ inst = AAA()
+ dumped = self.dumps(inst, proto)
+ loaded = self.loads(dumped)
+ self.assertEqual(loaded, REDUCE_A)
def test_getinitargs(self):
- pass
+ for proto in protocols:
+ inst = initarg(1, 2)
+ dumped = self.dumps(inst, proto)
+ loaded = self.loads(dumped)
+ self.assert_is_copy(inst, loaded)
def test_pop_empty_stack(self):
# Test issue7455
@@ -798,6 +842,7 @@ class AbstractPickleTests(unittest.TestCase):
s = self.dumps(a, proto)
b = self.loads(s)
self.assertEqual(a, b)
+ self.assertIs(type(a), type(b))
def test_structseq(self):
import time
@@ -807,29 +852,29 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(t, proto)
u = self.loads(s)
- self.assertEqual(t, u)
+ self.assert_is_copy(t, u)
if hasattr(os, "stat"):
t = os.stat(os.curdir)
s = self.dumps(t, proto)
u = self.loads(s)
- self.assertEqual(t, u)
+ self.assert_is_copy(t, u)
if hasattr(os, "statvfs"):
t = os.statvfs(os.curdir)
s = self.dumps(t, proto)
u = self.loads(s)
- self.assertEqual(t, u)
+ self.assert_is_copy(t, u)
def test_ellipsis(self):
for proto in protocols:
s = self.dumps(..., proto)
u = self.loads(s)
- self.assertEqual(..., u)
+ self.assertIs(..., u)
def test_notimplemented(self):
for proto in protocols:
s = self.dumps(NotImplemented, proto)
u = self.loads(s)
- self.assertEqual(NotImplemented, u)
+ self.assertIs(NotImplemented, u)
def test_singleton_types(self):
# Issue #6477: Test that types of built-in singletons can be pickled.
@@ -843,21 +888,21 @@ class AbstractPickleTests(unittest.TestCase):
# Tests for protocol 2
def test_proto(self):
- build_none = pickle.NONE + pickle.STOP
for proto in protocols:
- expected = build_none
+ pickled = self.dumps(None, proto)
if proto >= 2:
- expected = pickle.PROTO + bytes([proto]) + expected
- p = self.dumps(None, proto)
- self.assertEqual(p, expected)
+ proto_header = pickle.PROTO + bytes([proto])
+ self.assertTrue(pickled.startswith(proto_header))
+ else:
+ self.assertEqual(count_opcode(pickle.PROTO, pickled), 0)
oob = protocols[-1] + 1 # a future protocol
+ build_none = pickle.NONE + pickle.STOP
badpickle = pickle.PROTO + bytes([oob]) + build_none
try:
self.loads(badpickle)
- except ValueError as detail:
- self.assertTrue(str(detail).startswith(
- "unsupported pickle protocol"))
+ except ValueError as err:
+ self.assertIn("unsupported pickle protocol", str(err))
else:
self.fail("expected bad protocol number to raise ValueError")
@@ -866,7 +911,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
self.assertEqual(opcode_in_pickle(pickle.LONG1, s), proto >= 2)
def test_long4(self):
@@ -874,7 +919,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
self.assertEqual(opcode_in_pickle(pickle.LONG4, s), proto >= 2)
def test_short_tuples(self):
@@ -912,9 +957,9 @@ class AbstractPickleTests(unittest.TestCase):
for x in a, b, c, d, e:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y, (proto, x, s, y))
- expected = expected_opcode[proto, len(x)]
- self.assertEqual(opcode_in_pickle(expected, s), True)
+ self.assert_is_copy(x, y)
+ expected = expected_opcode[min(proto, 3), len(x)]
+ self.assertTrue(opcode_in_pickle(expected, s))
def test_singletons(self):
# Map (proto, singleton) to expected opcode.
@@ -938,8 +983,8 @@ class AbstractPickleTests(unittest.TestCase):
s = self.dumps(x, proto)
y = self.loads(s)
self.assertTrue(x is y, (proto, x, s, y))
- expected = expected_opcode[proto, x]
- self.assertEqual(opcode_in_pickle(expected, s), True)
+ expected = expected_opcode[min(proto, 3), x]
+ self.assertTrue(opcode_in_pickle(expected, s))
def test_newobj_tuple(self):
x = MyTuple([1, 2, 3])
@@ -948,8 +993,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(tuple(x), tuple(y))
- self.assertEqual(x.__dict__, y.__dict__)
+ self.assert_is_copy(x, y)
def test_newobj_list(self):
x = MyList([1, 2, 3])
@@ -958,8 +1002,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(list(x), list(y))
- self.assertEqual(x.__dict__, y.__dict__)
+ self.assert_is_copy(x, y)
def test_newobj_generic(self):
for proto in protocols:
@@ -970,6 +1013,7 @@ class AbstractPickleTests(unittest.TestCase):
s = self.dumps(x, proto)
y = self.loads(s)
detail = (proto, C, B, x, y, type(y))
+ self.assert_is_copy(x, y) # XXX revisit
self.assertEqual(B(x), B(y), detail)
self.assertEqual(x.__dict__, y.__dict__, detail)
@@ -1008,11 +1052,10 @@ class AbstractPickleTests(unittest.TestCase):
s1 = self.dumps(x, 1)
self.assertIn(__name__.encode("utf-8"), s1)
self.assertIn(b"MyList", s1)
- self.assertEqual(opcode_in_pickle(opcode, s1), False)
+ self.assertFalse(opcode_in_pickle(opcode, s1))
y = self.loads(s1)
- self.assertEqual(list(x), list(y))
- self.assertEqual(x.__dict__, y.__dict__)
+ self.assert_is_copy(x, y)
# Dump using protocol 2 for test.
s2 = self.dumps(x, 2)
@@ -1021,9 +1064,7 @@ class AbstractPickleTests(unittest.TestCase):
self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2))
y = self.loads(s2)
- self.assertEqual(list(x), list(y))
- self.assertEqual(x.__dict__, y.__dict__)
-
+ self.assert_is_copy(x, y)
finally:
e.restore()
@@ -1047,7 +1088,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
num_appends = count_opcode(pickle.APPENDS, s)
self.assertEqual(num_appends, proto > 0)
@@ -1056,7 +1097,7 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
num_appends = count_opcode(pickle.APPENDS, s)
if proto == 0:
self.assertEqual(num_appends, 0)
@@ -1070,7 +1111,7 @@ class AbstractPickleTests(unittest.TestCase):
s = self.dumps(x, proto)
self.assertIsInstance(s, bytes_types)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
num_setitems = count_opcode(pickle.SETITEMS, s)
self.assertEqual(num_setitems, proto > 0)
@@ -1079,22 +1120,49 @@ class AbstractPickleTests(unittest.TestCase):
for proto in protocols:
s = self.dumps(x, proto)
y = self.loads(s)
- self.assertEqual(x, y)
+ self.assert_is_copy(x, y)
num_setitems = count_opcode(pickle.SETITEMS, s)
if proto == 0:
self.assertEqual(num_setitems, 0)
else:
self.assertTrue(num_setitems >= 2)
+ def test_set_chunking(self):
+ n = 10 # too small to chunk
+ x = set(range(n))
+ for proto in protocols:
+ s = self.dumps(x, proto)
+ y = self.loads(s)
+ self.assert_is_copy(x, y)
+ num_additems = count_opcode(pickle.ADDITEMS, s)
+ if proto < 4:
+ self.assertEqual(num_additems, 0)
+ else:
+ self.assertEqual(num_additems, 1)
+
+ n = 2500 # expect at least two chunks when proto >= 4
+ x = set(range(n))
+ for proto in protocols:
+ s = self.dumps(x, proto)
+ y = self.loads(s)
+ self.assert_is_copy(x, y)
+ num_additems = count_opcode(pickle.ADDITEMS, s)
+ if proto < 4:
+ self.assertEqual(num_additems, 0)
+ else:
+ self.assertGreaterEqual(num_additems, 2)
+
def test_simple_newobj(self):
x = object.__new__(SimpleNewObj) # avoid __init__
x.abc = 666
for proto in protocols:
s = self.dumps(x, proto)
- self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), proto >= 2)
+ self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s),
+ 2 <= proto < 4)
+ self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
+ proto >= 4)
y = self.loads(s) # will raise TypeError if __init__ called
- self.assertEqual(y.abc, 666)
- self.assertEqual(x.__dict__, y.__dict__)
+ self.assert_is_copy(x, y)
def test_newobj_list_slots(self):
x = SlotList([1, 2, 3])
@@ -1102,10 +1170,7 @@ class AbstractPickleTests(unittest.TestCase):
x.bar = "hello"
s = self.dumps(x, 2)
y = self.loads(s)
- self.assertEqual(list(x), list(y))
- self.assertEqual(x.__dict__, y.__dict__)
- self.assertEqual(x.foo, y.foo)
- self.assertEqual(x.bar, y.bar)
+ self.assert_is_copy(x, y)
def test_reduce_overrides_default_reduce_ex(self):
for proto in protocols:
@@ -1154,11 +1219,10 @@ class AbstractPickleTests(unittest.TestCase):
@no_tracing
def test_bad_getattr(self):
+ # Issue #3514: crash when there is an infinite loop in __getattr__
x = BadGetattr()
- for proto in 0, 1:
+ for proto in protocols:
self.assertRaises(RuntimeError, self.dumps, x, proto)
- # protocol 2 don't raise a RuntimeError.
- d = self.dumps(x, 2)
def test_reduce_bad_iterator(self):
# Issue4176: crash when 4th and 5th items of __reduce__()
@@ -1191,11 +1255,10 @@ class AbstractPickleTests(unittest.TestCase):
obj = [dict(large_dict), dict(large_dict), dict(large_dict)]
for proto in protocols:
- dumped = self.dumps(obj, proto)
- loaded = self.loads(dumped)
- self.assertEqual(loaded, obj,
- "Failed protocol %d: %r != %r"
- % (proto, obj, loaded))
+ with self.subTest(proto=proto):
+ dumped = self.dumps(obj, proto)
+ loaded = self.loads(dumped)
+ self.assert_is_copy(obj, loaded)
def test_attribute_name_interning(self):
# Test that attribute names of pickled objects are interned when
@@ -1248,6 +1311,35 @@ class AbstractPickleTests(unittest.TestCase):
dumped = self.dumps(set([3]), 2)
self.assertEqual(dumped, DATA6)
+ def test_load_python2_str_as_bytes(self):
+ # From Python 2: pickle.dumps('a\x00\xa0', protocol=0)
+ self.assertEqual(self.loads(b"S'a\\x00\\xa0'\n.",
+ encoding="bytes"), b'a\x00\xa0')
+ # From Python 2: pickle.dumps('a\x00\xa0', protocol=1)
+ self.assertEqual(self.loads(b'U\x03a\x00\xa0.',
+ encoding="bytes"), b'a\x00\xa0')
+ # From Python 2: pickle.dumps('a\x00\xa0', protocol=2)
+ self.assertEqual(self.loads(b'\x80\x02U\x03a\x00\xa0.',
+ encoding="bytes"), b'a\x00\xa0')
+
+ def test_load_python2_unicode_as_str(self):
+ # From Python 2: pickle.dumps(u'π', protocol=0)
+ self.assertEqual(self.loads(b'V\\u03c0\n.',
+ encoding='bytes'), 'π')
+ # From Python 2: pickle.dumps(u'π', protocol=1)
+ self.assertEqual(self.loads(b'X\x02\x00\x00\x00\xcf\x80.',
+ encoding="bytes"), 'π')
+ # From Python 2: pickle.dumps(u'π', protocol=2)
+ self.assertEqual(self.loads(b'\x80\x02X\x02\x00\x00\x00\xcf\x80.',
+ encoding="bytes"), 'π')
+
+ def test_load_long_python2_str_as_bytes(self):
+ # From Python 2: pickle.dumps('x' * 300, protocol=1)
+ self.assertEqual(self.loads(pickle.BINSTRING +
+ struct.pack("<I", 300) +
+ b'x' * 300 + pickle.STOP,
+ encoding='bytes'), b'x' * 300)
+
def test_large_pickles(self):
# Test the correctness of internal buffering routines when handling
# large data.
@@ -1266,11 +1358,14 @@ class AbstractPickleTests(unittest.TestCase):
def test_int_pickling_efficiency(self):
# Test compacity of int representation (see issue #12744)
for proto in protocols:
- sizes = [len(self.dumps(2**n, proto)) for n in range(70)]
- # the size function is monotonic
- self.assertEqual(sorted(sizes), sizes)
- if proto >= 2:
- self.assertLessEqual(sizes[-1], 14)
+ with self.subTest(proto=proto):
+ pickles = [self.dumps(2**n, proto) for n in range(70)]
+ sizes = list(map(len, pickles))
+ # the size function is monotonic
+ self.assertEqual(sorted(sizes), sizes)
+ if proto >= 2:
+ for p in pickles:
+ self.assertFalse(opcode_in_pickle(pickle.LONG, p))
def check_negative_32b_binXXX(self, dumped):
if sys.maxsize > 2**32:
@@ -1301,6 +1396,35 @@ class AbstractPickleTests(unittest.TestCase):
dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.'
self.assertRaises(ValueError, self.loads, dumped)
+ def test_badly_escaped_string(self):
+ self.assertRaises(ValueError, self.loads, b"S'\\'\n.")
+
+ def test_badly_quoted_string(self):
+ # Issue #17710
+ badpickles = [b"S'\n.",
+ b'S"\n.',
+ b'S\' \n.',
+ b'S" \n.',
+ b'S\'"\n.',
+ b'S"\'\n.',
+ b"S' ' \n.",
+ b'S" " \n.',
+ b"S ''\n.",
+ b'S ""\n.',
+ b'S \n.',
+ b'S\n.',
+ b'S.']
+ for p in badpickles:
+ self.assertRaises(pickle.UnpicklingError, self.loads, p)
+
+ def test_correctly_quoted_string(self):
+ goodpickles = [(b"S''\n.", ''),
+ (b'S""\n.', ''),
+ (b'S"\\n"\n.', '\n'),
+ (b"S'\\n'\n.", '\n')]
+ for p, expected in goodpickles:
+ self.assertEqual(self.loads(p), expected)
+
def _check_pickling_with_opcode(self, obj, opcode, proto):
pickled = self.dumps(obj, proto)
self.assertTrue(opcode_in_pickle(opcode, pickled))
@@ -1324,6 +1448,194 @@ class AbstractPickleTests(unittest.TestCase):
else:
self._check_pickling_with_opcode(obj, pickle.SETITEMS, proto)
+ # Exercise framing (proto >= 4) for significant workloads
+
+ FRAME_SIZE_TARGET = 64 * 1024
+
+ def check_frame_opcodes(self, pickled):
+ """
+ Check the arguments of FRAME opcodes in a protocol 4+ pickle.
+ """
+ frame_opcode_size = 9
+ last_arg = last_pos = None
+ for op, arg, pos in pickletools.genops(pickled):
+ if op.name != 'FRAME':
+ continue
+ if last_pos is not None:
+ # The previous frame's size should be equal to the number
+ # of bytes up to the current frame.
+ frame_size = pos - last_pos - frame_opcode_size
+ self.assertEqual(frame_size, last_arg)
+ last_arg, last_pos = arg, pos
+ # The last frame's size should be equal to the number of bytes up
+ # to the pickle's end.
+ frame_size = len(pickled) - last_pos - frame_opcode_size
+ self.assertEqual(frame_size, last_arg)
+
+ def test_framing_many_objects(self):
+ obj = list(range(10**5))
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ with self.subTest(proto=proto):
+ pickled = self.dumps(obj, proto)
+ unpickled = self.loads(pickled)
+ self.assertEqual(obj, unpickled)
+ bytes_per_frame = (len(pickled) /
+ count_opcode(pickle.FRAME, pickled))
+ self.assertGreater(bytes_per_frame,
+ self.FRAME_SIZE_TARGET / 2)
+ self.assertLessEqual(bytes_per_frame,
+ self.FRAME_SIZE_TARGET * 1)
+ self.check_frame_opcodes(pickled)
+
+ def test_framing_large_objects(self):
+ N = 1024 * 1024
+ obj = [b'x' * N, b'y' * N, b'z' * N]
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ with self.subTest(proto=proto):
+ pickled = self.dumps(obj, proto)
+ unpickled = self.loads(pickled)
+ self.assertEqual(obj, unpickled)
+ n_frames = count_opcode(pickle.FRAME, pickled)
+ self.assertGreaterEqual(n_frames, len(obj))
+ self.check_frame_opcodes(pickled)
+
+ def test_optional_frames(self):
+ if pickle.HIGHEST_PROTOCOL < 4:
+ return
+
+ def remove_frames(pickled, keep_frame=None):
+ """Remove frame opcodes from the given pickle."""
+ frame_starts = []
+ # 1 byte for the opcode and 8 for the argument
+ frame_opcode_size = 9
+ for opcode, _, pos in pickletools.genops(pickled):
+ if opcode.name == 'FRAME':
+ frame_starts.append(pos)
+
+ newpickle = bytearray()
+ last_frame_end = 0
+ for i, pos in enumerate(frame_starts):
+ if keep_frame and keep_frame(i):
+ continue
+ newpickle += pickled[last_frame_end:pos]
+ last_frame_end = pos + frame_opcode_size
+ newpickle += pickled[last_frame_end:]
+ return newpickle
+
+ frame_size = self.FRAME_SIZE_TARGET
+ num_frames = 20
+ obj = [bytes([i]) * frame_size for i in range(num_frames)]
+
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ pickled = self.dumps(obj, proto)
+
+ frameless_pickle = remove_frames(pickled)
+ self.assertEqual(count_opcode(pickle.FRAME, frameless_pickle), 0)
+ self.assertEqual(obj, self.loads(frameless_pickle))
+
+ some_frames_pickle = remove_frames(pickled, lambda i: i % 2)
+ self.assertLess(count_opcode(pickle.FRAME, some_frames_pickle),
+ count_opcode(pickle.FRAME, pickled))
+ self.assertEqual(obj, self.loads(some_frames_pickle))
+
+ def test_nested_names(self):
+ global Nested
+ class Nested:
+ class A:
+ class B:
+ class C:
+ pass
+
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for obj in [Nested.A, Nested.A.B, Nested.A.B.C]:
+ with self.subTest(proto=proto, obj=obj):
+ unpickled = self.loads(self.dumps(obj, proto))
+ self.assertIs(obj, unpickled)
+
+ def test_py_methods(self):
+ global PyMethodsTest
+ class PyMethodsTest:
+ @staticmethod
+ def cheese():
+ return "cheese"
+ @classmethod
+ def wine(cls):
+ assert cls is PyMethodsTest
+ return "wine"
+ def biscuits(self):
+ assert isinstance(self, PyMethodsTest)
+ return "biscuits"
+ class Nested:
+ "Nested class"
+ @staticmethod
+ def ketchup():
+ return "ketchup"
+ @classmethod
+ def maple(cls):
+ assert cls is PyMethodsTest.Nested
+ return "maple"
+ def pie(self):
+ assert isinstance(self, PyMethodsTest.Nested)
+ return "pie"
+
+ py_methods = (
+ PyMethodsTest.cheese,
+ PyMethodsTest.wine,
+ PyMethodsTest().biscuits,
+ PyMethodsTest.Nested.ketchup,
+ PyMethodsTest.Nested.maple,
+ PyMethodsTest.Nested().pie
+ )
+ py_unbound_methods = (
+ (PyMethodsTest.biscuits, PyMethodsTest),
+ (PyMethodsTest.Nested.pie, PyMethodsTest.Nested)
+ )
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for method in py_methods:
+ with self.subTest(proto=proto, method=method):
+ unpickled = self.loads(self.dumps(method, proto))
+ self.assertEqual(method(), unpickled())
+ for method, cls in py_unbound_methods:
+ obj = cls()
+ with self.subTest(proto=proto, method=method):
+ unpickled = self.loads(self.dumps(method, proto))
+ self.assertEqual(method(obj), unpickled(obj))
+
+ def test_c_methods(self):
+ global Subclass
+ class Subclass(tuple):
+ class Nested(str):
+ pass
+
+ c_methods = (
+ # bound built-in method
+ ("abcd".index, ("c",)),
+ # unbound built-in method
+ (str.index, ("abcd", "c")),
+ # bound "slot" method
+ ([1, 2, 3].__len__, ()),
+ # unbound "slot" method
+ (list.__len__, ([1, 2, 3],)),
+ # bound "coexist" method
+ ({1, 2}.__contains__, (2,)),
+ # unbound "coexist" method
+ (set.__contains__, ({1, 2}, 2)),
+ # built-in class method
+ (dict.fromkeys, (("a", 1), ("b", 2))),
+ # built-in static method
+ (bytearray.maketrans, (b"abc", b"xyz")),
+ # subclass methods
+ (Subclass([1,2,2]).count, (2,)),
+ (Subclass.count, (Subclass([1,2,2]), 2)),
+ (Subclass.Nested("sweet").count, ("e",)),
+ (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")),
+ )
+ for proto in range(4, pickle.HIGHEST_PROTOCOL + 1):
+ for method, args in c_methods:
+ with self.subTest(proto=proto, method=method):
+ unpickled = self.loads(self.dumps(method, proto))
+ self.assertEqual(method(*args), unpickled(*args))
+
class BigmemPickleTests(unittest.TestCase):
@@ -1336,8 +1648,9 @@ class BigmemPickleTests(unittest.TestCase):
for proto in protocols:
if proto < 2:
continue
- with self.assertRaises((ValueError, OverflowError)):
- self.dumps(data, protocol=proto)
+ with self.subTest(proto=proto):
+ with self.assertRaises((ValueError, OverflowError)):
+ self.dumps(data, protocol=proto)
finally:
data = None
@@ -1352,24 +1665,44 @@ class BigmemPickleTests(unittest.TestCase):
for proto in protocols:
if proto < 3:
continue
- try:
- pickled = self.dumps(data, protocol=proto)
- self.assertTrue(b"abcd" in pickled[:15])
- self.assertTrue(b"abcd" in pickled[-15:])
- finally:
- pickled = None
+ with self.subTest(proto=proto):
+ try:
+ pickled = self.dumps(data, protocol=proto)
+ header = (pickle.BINBYTES +
+ struct.pack("<I", len(data)))
+ data_start = pickled.index(data)
+ self.assertEqual(
+ header,
+ pickled[data_start-len(header):data_start])
+ finally:
+ pickled = None
finally:
data = None
@bigmemtest(size=_4G, memuse=2.5, dry_run=False)
def test_huge_bytes_64b(self, size):
- data = b"a" * size
+ data = b"acbd" * (size // 4)
try:
for proto in protocols:
if proto < 3:
continue
- with self.assertRaises((ValueError, OverflowError)):
- self.dumps(data, protocol=proto)
+ with self.subTest(proto=proto):
+ if proto == 3:
+ # Protocol 3 does not support large bytes objects.
+ # Verify that we do not crash when processing one.
+ with self.assertRaises((ValueError, OverflowError)):
+ self.dumps(data, protocol=proto)
+ continue
+ try:
+ pickled = self.dumps(data, protocol=proto)
+ header = (pickle.BINBYTES8 +
+ struct.pack("<Q", len(data)))
+ data_start = pickled.index(data)
+ self.assertEqual(
+ header,
+ pickled[data_start-len(header):data_start])
+ finally:
+ pickled = None
finally:
data = None
@@ -1381,27 +1714,52 @@ class BigmemPickleTests(unittest.TestCase):
data = "abcd" * (size // 4)
try:
for proto in protocols:
- try:
- pickled = self.dumps(data, protocol=proto)
- self.assertTrue(b"abcd" in pickled[:15])
- self.assertTrue(b"abcd" in pickled[-15:])
- finally:
- pickled = None
+ if proto == 0:
+ continue
+ with self.subTest(proto=proto):
+ try:
+ pickled = self.dumps(data, protocol=proto)
+ header = (pickle.BINUNICODE +
+ struct.pack("<I", len(data)))
+ data_start = pickled.index(b'abcd')
+ self.assertEqual(
+ header,
+ pickled[data_start-len(header):data_start])
+ self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
+ pickled.index(b"abcd")), len(data))
+ finally:
+ pickled = None
finally:
data = None
- # BINUNICODE (protocols 1, 2 and 3) cannot carry more than
- # 2**32 - 1 bytes of utf-8 encoded unicode.
+ # BINUNICODE (protocols 1, 2 and 3) cannot carry more than 2**32 - 1 bytes
+ # of utf-8 encoded unicode. BINUNICODE8 (protocol 4) supports these huge
+ # unicode strings however.
@bigmemtest(size=_4G, memuse=8, dry_run=False)
def test_huge_str_64b(self, size):
- data = "a" * size
+ data = "abcd" * (size // 4)
try:
for proto in protocols:
if proto == 0:
continue
- with self.assertRaises((ValueError, OverflowError)):
- self.dumps(data, protocol=proto)
+ with self.subTest(proto=proto):
+ if proto < 4:
+ with self.assertRaises((ValueError, OverflowError)):
+ self.dumps(data, protocol=proto)
+ continue
+ try:
+ pickled = self.dumps(data, protocol=proto)
+ header = (pickle.BINUNICODE8 +
+ struct.pack("<Q", len(data)))
+ data_start = pickled.index(b'abcd')
+ self.assertEqual(
+ header,
+ pickled[data_start-len(header):data_start])
+ self.assertEqual((pickled.rindex(b"abcd") + len(b"abcd") -
+ pickled.index(b"abcd")), len(data))
+ finally:
+ pickled = None
finally:
data = None
@@ -1445,8 +1803,8 @@ class REX_five(object):
return object.__reduce__(self)
class REX_six(object):
- """This class is used to check the 4th argument (list iterator) of the reduce
- protocol.
+ """This class is used to check the 4th argument (list iterator) of
+ the reduce protocol.
"""
def __init__(self, items=None):
self.items = items if items is not None else []
@@ -1458,8 +1816,8 @@ class REX_six(object):
return type(self), (), None, iter(self.items), None
class REX_seven(object):
- """This class is used to check the 5th argument (dict iterator) of the reduce
- protocol.
+ """This class is used to check the 5th argument (dict iterator) of
+ the reduce protocol.
"""
def __init__(self, table=None):
self.table = table if table is not None else {}
@@ -1497,10 +1855,16 @@ class MyList(list):
class MyDict(dict):
sample = {"a": 1, "b": 2}
+class MySet(set):
+ sample = {"a", "b"}
+
+class MyFrozenSet(frozenset):
+ sample = frozenset({"a", "b"})
+
myclasses = [MyInt, MyFloat,
MyComplex,
MyStr, MyUnicode,
- MyTuple, MyList, MyDict]
+ MyTuple, MyList, MyDict, MySet, MyFrozenSet]
class SlotList(MyList):
@@ -1510,6 +1874,8 @@ class SimpleNewObj(object):
def __init__(self, a, b, c):
# raise an error, to make sure this isn't called
raise TypeError("SimpleNewObj.__init__() didn't expect to get called")
+ def __eq__(self, other):
+ return self.__dict__ == other.__dict__
class BadGetattr:
def __getattr__(self, key):
@@ -1546,7 +1912,7 @@ class AbstractPickleModuleTests(unittest.TestCase):
def test_highest_protocol(self):
# Of course this needs to be changed when HIGHEST_PROTOCOL changes.
- self.assertEqual(pickle.HIGHEST_PROTOCOL, 3)
+ self.assertEqual(pickle.HIGHEST_PROTOCOL, 4)
def test_callapi(self):
f = io.BytesIO()
@@ -1731,22 +2097,23 @@ class AbstractPicklerUnpicklerObjectTests(unittest.TestCase):
def _check_multiple_unpicklings(self, ioclass):
for proto in protocols:
- data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
- f = ioclass()
- pickler = self.pickler_class(f, protocol=proto)
- pickler.dump(data1)
- pickled = f.getvalue()
-
- N = 5
- f = ioclass(pickled * N)
- unpickler = self.unpickler_class(f)
- for i in range(N):
- if f.seekable():
- pos = f.tell()
- self.assertEqual(unpickler.load(), data1)
- if f.seekable():
- self.assertEqual(f.tell(), pos + len(pickled))
- self.assertRaises(EOFError, unpickler.load)
+ with self.subTest(proto=proto):
+ data1 = [(x, str(x)) for x in range(2000)] + [b"abcde", len]
+ f = ioclass()
+ pickler = self.pickler_class(f, protocol=proto)
+ pickler.dump(data1)
+ pickled = f.getvalue()
+
+ N = 5
+ f = ioclass(pickled * N)
+ unpickler = self.unpickler_class(f)
+ for i in range(N):
+ if f.seekable():
+ pos = f.tell()
+ self.assertEqual(unpickler.load(), data1)
+ if f.seekable():
+ self.assertEqual(f.tell(), pos + len(pickled))
+ self.assertRaises(EOFError, unpickler.load)
def test_multiple_unpicklings_seekable(self):
self._check_multiple_unpicklings(io.BytesIO)
diff --git a/Lib/test/pycacert.pem b/Lib/test/pycacert.pem
new file mode 100644
index 0000000..09b1f3e
--- /dev/null
+++ b/Lib/test/pycacert.pem
@@ -0,0 +1,78 @@
+Certificate:
+ Data:
+ Version: 3 (0x2)
+ Serial Number: 12723342612721443280 (0xb09264b1f2da21d0)
+ Signature Algorithm: sha1WithRSAEncryption
+ Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Validity
+ Not Before: Jan 4 19:47:07 2013 GMT
+ Not After : Jan 2 19:47:07 2023 GMT
+ Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Subject Public Key Info:
+ Public Key Algorithm: rsaEncryption
+ Public-Key: (2048 bit)
+ Modulus:
+ 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2:
+ 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4:
+ e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f:
+ e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f:
+ 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf:
+ 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d:
+ a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3:
+ e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4:
+ 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf:
+ 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c:
+ e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6:
+ c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a:
+ cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01:
+ 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87:
+ 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f:
+ 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14:
+ e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4:
+ c5:4d
+ Exponent: 65537 (0x10001)
+ X509v3 extensions:
+ X509v3 Subject Key Identifier:
+ BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B
+ X509v3 Authority Key Identifier:
+ keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B
+
+ X509v3 Basic Constraints:
+ CA:TRUE
+ Signature Algorithm: sha1WithRSAEncryption
+ 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6:
+ 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d:
+ a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95:
+ 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17:
+ 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c:
+ 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4:
+ fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7:
+ 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24:
+ 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33:
+ 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61:
+ ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f:
+ 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64:
+ b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb:
+ 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3:
+ 5e:58:c8:9e
+-----BEGIN CERTIFICATE-----
+MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV
+BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW
+MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx
+OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV
+q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/
+AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA
+Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni
+0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx
+6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w
+HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2
+2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB
+AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4
+QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1
+Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O
+JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR
+f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf
+9mmvtk57HVjsO6lTo15YyJ4=
+-----END CERTIFICATE-----
diff --git a/Lib/test/pycakey.pem b/Lib/test/pycakey.pem
new file mode 100644
index 0000000..fc6effe
--- /dev/null
+++ b/Lib/test/pycakey.pem
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9
+K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc
+nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd
+LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g
+uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB
+uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu
+Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE
+ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls
+jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu
+xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h
+6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm
+J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy
+tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i
+EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4
+wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv
+mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope
+LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT
+C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f
+HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU
+iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm
+OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G
+D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE
+bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt
+/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv
+kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP
+UuvXsaPph7GTqPuy4Kab12YC
+-----END PRIVATE KEY-----
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index ae62c6e..c1c831f 100755
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1,11 +1,18 @@
#! /usr/bin/env python3
"""
-Usage:
+Script to run Python regression tests.
+Run this script with -h or --help for documentation.
+"""
+
+USAGE = """\
python -m test [options] [test_name1 [test_name2 ...]]
python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
+"""
+DESCRIPTION = """\
+Run Python regression tests.
If no arguments or options are provided, finds all files matching
the pattern "test_*" in the Lib/test subdirectory and runs
@@ -15,63 +22,10 @@ For more rigorous testing, it is useful to use the following
command line:
python -E -Wd -m test [options] [test_name1 ...]
+"""
-
-Options:
-
--h/--help -- print this text and exit
---timeout TIMEOUT
- -- dump the traceback and exit if a test takes more
- than TIMEOUT seconds; disabled if TIMEOUT is negative
- or equals to zero
---wait -- wait for user input, e.g., allow a debugger to be attached
-
-Verbosity
-
--v/--verbose -- run tests in verbose mode with output to stdout
--w/--verbose2 -- re-run failed tests in verbose mode
--W/--verbose3 -- display test output on failure
--d/--debug -- print traceback for failed tests
--q/--quiet -- no output unless one or more tests fail
--o/--slow -- print the slowest 10 tests
- --header -- print header with interpreter info
-
-Selecting tests
-
--r/--randomize -- randomize test execution order (see below)
- --randseed -- pass a random seed to reproduce a previous random run
--f/--fromfile -- read names of tests to run from a file (see below)
--x/--exclude -- arguments are tests to *exclude*
--s/--single -- single step through a set of tests (see below)
--m/--match PAT -- match test cases and methods with glob pattern PAT
--G/--failfast -- fail as soon as a test fails (only with -v or -W)
--u/--use RES1,RES2,...
- -- specify which special resource intensive tests to run
--M/--memlimit LIMIT
- -- run very large memory-consuming tests
- --testdir DIR
- -- execute test files in the specified directory (instead
- of the Python stdlib test suite)
-
-Special runs
-
--l/--findleaks -- if GC is available detect tests that leak memory
--L/--runleaks -- run the leaks(1) command just before exit
--R/--huntrleaks RUNCOUNTS
- -- search for reference leaks (needs debug build, v. slow)
--j/--multiprocess PROCESSES
- -- run PROCESSES processes at once
--T/--coverage -- turn on code coverage tracing using the trace module
--D/--coverdir DIRECTORY
- -- Directory where coverage files are put
--N/--nocoverdir -- Put coverage files alongside modules
--t/--threshold THRESHOLD
- -- call gc.set_threshold(THRESHOLD)
--n/--nowindows -- suppress error message boxes on Windows
--F/--forever -- run the specified tests in a loop, until an error happens
-
-
-Additional Option Details:
+EPILOG = """\
+Additional option details:
-r randomizes test execution order. You can use --randseed=int to provide a
int seed value for the randomizer; this is useful for reproducing troublesome
@@ -168,11 +122,12 @@ option '-uall,-gui'.
# We import importlib *ASAP* in order to test #15386
import importlib
+import argparse
import builtins
import faulthandler
-import getopt
import io
import json
+import locale
import logging
import os
import platform
@@ -194,7 +149,7 @@ try:
except ImportError:
threading = None
try:
- import multiprocessing.process
+ import _multiprocessing, multiprocessing.process
except ImportError:
multiprocessing = None
@@ -246,20 +201,262 @@ from test import support
RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network',
'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
-TEMPDIR = os.path.abspath(tempfile.gettempdir())
-
-def usage(msg):
- print(msg, file=sys.stderr)
- print("Use --help for usage", file=sys.stderr)
- sys.exit(2)
-
+# When tests are run from the Python build directory, it is best practice
+# to keep the test files in a subfolder. This eases the cleanup of leftover
+# files using the "make distclean" command.
+if sysconfig.is_python_build():
+ TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build')
+else:
+ TEMPDIR = tempfile.gettempdir()
+TEMPDIR = os.path.abspath(TEMPDIR)
+
+class _ArgParser(argparse.ArgumentParser):
+
+ def error(self, message):
+ super().error(message + "\nPass -h or --help for complete help.")
+
+def _create_parser():
+ # Set prog to prevent the uninformative "__main__.py" from displaying in
+ # error messages when using "python -m test ...".
+ parser = _ArgParser(prog='regrtest.py',
+ usage=USAGE,
+ description=DESCRIPTION,
+ epilog=EPILOG,
+ add_help=False,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+
+ # Arguments with this clause added to its help are described further in
+ # the epilog's "Additional option details" section.
+ more_details = ' See the section at bottom for more details.'
+
+ group = parser.add_argument_group('General options')
+ # We add help explicitly to control what argument group it renders under.
+ group.add_argument('-h', '--help', action='help',
+ help='show this help message and exit')
+ group.add_argument('--timeout', metavar='TIMEOUT', type=float,
+ help='dump the traceback and exit if a test takes '
+ 'more than TIMEOUT seconds; disabled if TIMEOUT '
+ 'is negative or equals to zero')
+ group.add_argument('--wait', action='store_true',
+ help='wait for user input, e.g., allow a debugger '
+ 'to be attached')
+ group.add_argument('--slaveargs', metavar='ARGS')
+ group.add_argument('-S', '--start', metavar='START',
+ help='the name of the test at which to start.' +
+ more_details)
+
+ group = parser.add_argument_group('Verbosity')
+ group.add_argument('-v', '--verbose', action='count',
+ help='run tests in verbose mode with output to stdout')
+ group.add_argument('-w', '--verbose2', action='store_true',
+ help='re-run failed tests in verbose mode')
+ group.add_argument('-W', '--verbose3', action='store_true',
+ help='display test output on failure')
+ group.add_argument('-q', '--quiet', action='store_true',
+ help='no output unless one or more tests fail')
+ group.add_argument('-o', '--slow', action='store_true', dest='print_slow',
+ help='print the slowest 10 tests')
+ group.add_argument('--header', action='store_true',
+ help='print header with interpreter info')
+
+ group = parser.add_argument_group('Selecting tests')
+ group.add_argument('-r', '--randomize', action='store_true',
+ help='randomize test execution order.' + more_details)
+ group.add_argument('--randseed', metavar='SEED',
+ dest='random_seed', type=int,
+ help='pass a random seed to reproduce a previous '
+ 'random run')
+ group.add_argument('-f', '--fromfile', metavar='FILE',
+ help='read names of tests to run from a file.' +
+ more_details)
+ group.add_argument('-x', '--exclude', action='store_true',
+ help='arguments are tests to *exclude*')
+ group.add_argument('-s', '--single', action='store_true',
+ help='single step through a set of tests.' +
+ more_details)
+ group.add_argument('-m', '--match', metavar='PAT',
+ dest='match_tests',
+ help='match test cases and methods with glob pattern PAT')
+ group.add_argument('-G', '--failfast', action='store_true',
+ help='fail as soon as a test fails (only with -v or -W)')
+ group.add_argument('-u', '--use', metavar='RES1,RES2,...',
+ action='append', type=resources_list,
+ help='specify which special resource intensive tests '
+ 'to run.' + more_details)
+ group.add_argument('-M', '--memlimit', metavar='LIMIT',
+ help='run very large memory-consuming tests.' +
+ more_details)
+ group.add_argument('--testdir', metavar='DIR',
+ type=relative_filename,
+ help='execute test files in the specified directory '
+ '(instead of the Python stdlib test suite)')
+
+ group = parser.add_argument_group('Special runs')
+ group.add_argument('-l', '--findleaks', action='store_true',
+ help='if GC is available detect tests that leak memory')
+ group.add_argument('-L', '--runleaks', action='store_true',
+ help='run the leaks(1) command just before exit.' +
+ more_details)
+ group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
+ type=huntrleaks,
+ help='search for reference leaks (needs debug build, '
+ 'very slow).' + more_details)
+ group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
+ dest='use_mp', type=int,
+ help='run PROCESSES processes at once')
+ group.add_argument('-T', '--coverage', action='store_true',
+ dest='trace',
+ help='turn on code coverage tracing using the trace '
+ 'module')
+ group.add_argument('-D', '--coverdir', metavar='DIR',
+ type=relative_filename,
+ help='directory where coverage files are put')
+ group.add_argument('-N', '--nocoverdir',
+ action='store_const', const=None, dest='coverdir',
+ help='put coverage files alongside modules')
+ group.add_argument('-t', '--threshold', metavar='THRESHOLD',
+ type=int,
+ help='call gc.set_threshold(THRESHOLD)')
+ group.add_argument('-n', '--nowindows', action='store_true',
+ help='suppress error message boxes on Windows')
+ group.add_argument('-F', '--forever', action='store_true',
+ help='run the specified tests in a loop, until an '
+ 'error happens')
+
+ parser.add_argument('args', nargs=argparse.REMAINDER,
+ help=argparse.SUPPRESS)
+
+ return parser
+
+def relative_filename(string):
+ # CWD is replaced with a temporary dir before calling main(), so we
+ # join it with the saved CWD so it ends up where the user expects.
+ return os.path.join(support.SAVEDCWD, string)
+
+def huntrleaks(string):
+ args = string.split(':')
+ if len(args) not in (2, 3):
+ raise argparse.ArgumentTypeError(
+ 'needs 2 or 3 colon-separated arguments')
+ nwarmup = int(args[0]) if args[0] else 5
+ ntracked = int(args[1]) if args[1] else 4
+ fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
+ return nwarmup, ntracked, fname
+
+def resources_list(string):
+ u = [x.lower() for x in string.split(',')]
+ for r in u:
+ if r == 'all' or r == 'none':
+ continue
+ if r[0] == '-':
+ r = r[1:]
+ if r not in RESOURCE_NAMES:
+ raise argparse.ArgumentTypeError('invalid resource: ' + r)
+ return u
-def main(tests=None, testdir=None, verbose=0, quiet=False,
+def _parse_args(args, **kwargs):
+ # Defaults
+ ns = argparse.Namespace(testdir=None, verbose=0, quiet=False,
exclude=False, single=False, randomize=False, fromfile=None,
findleaks=False, use_resources=None, trace=False, coverdir='coverage',
runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
random_seed=None, use_mp=None, verbose3=False, forever=False,
- header=False, failfast=False, match_tests=None):
+ header=False, failfast=False, match_tests=None)
+ for k, v in kwargs.items():
+ if not hasattr(ns, k):
+ raise TypeError('%r is an invalid keyword argument '
+ 'for this function' % k)
+ setattr(ns, k, v)
+ if ns.use_resources is None:
+ ns.use_resources = []
+
+ parser = _create_parser()
+ parser.parse_args(args=args, namespace=ns)
+
+ if ns.single and ns.fromfile:
+ parser.error("-s and -f don't go together!")
+ if ns.use_mp and ns.trace:
+ parser.error("-T and -j don't go together!")
+ if ns.use_mp and ns.findleaks:
+ parser.error("-l and -j don't go together!")
+ if ns.use_mp and ns.memlimit:
+ parser.error("-M and -j don't go together!")
+ if ns.failfast and not (ns.verbose or ns.verbose3):
+ parser.error("-G/--failfast needs either -v or -W")
+
+ if ns.quiet:
+ ns.verbose = 0
+ if ns.timeout is not None:
+ if hasattr(faulthandler, 'dump_traceback_later'):
+ if ns.timeout <= 0:
+ ns.timeout = None
+ else:
+ print("Warning: The timeout option requires "
+ "faulthandler.dump_traceback_later")
+ ns.timeout = None
+ if ns.use_mp is not None:
+ if ns.use_mp <= 0:
+ # Use all cores + extras for tests that like to sleep
+ ns.use_mp = 2 + (os.cpu_count() or 1)
+ if ns.use_mp == 1:
+ ns.use_mp = None
+ if ns.use:
+ for a in ns.use:
+ for r in a:
+ if r == 'all':
+ ns.use_resources[:] = RESOURCE_NAMES
+ continue
+ if r == 'none':
+ del ns.use_resources[:]
+ continue
+ remove = False
+ if r[0] == '-':
+ remove = True
+ r = r[1:]
+ if remove:
+ if r in ns.use_resources:
+ ns.use_resources.remove(r)
+ elif r not in ns.use_resources:
+ ns.use_resources.append(r)
+ if ns.random_seed is not None:
+ ns.randomize = True
+
+ return ns
+
+
+def run_test_in_subprocess(testname, ns):
+ """Run the given test in a subprocess with --slaveargs.
+
+ ns is the option Namespace parsed from command-line arguments. regrtest
+ is invoked in a subprocess with the --slaveargs argument; when the
+ subprocess exits, its return code, stdout and stderr are returned as a
+ 3-tuple.
+ """
+ from subprocess import Popen, PIPE
+ base_cmd = ([sys.executable] + support.args_from_interpreter_flags() +
+ ['-X', 'faulthandler', '-m', 'test.regrtest'])
+
+ slaveargs = (
+ (testname, ns.verbose, ns.quiet),
+ dict(huntrleaks=ns.huntrleaks,
+ use_resources=ns.use_resources,
+ output_on_failure=ns.verbose3,
+ timeout=ns.timeout, failfast=ns.failfast,
+ match_tests=ns.match_tests))
+ # Running the child from the same working directory as regrtest's original
+ # invocation ensures that TEMPDIR for the child is the same when
+ # sysconfig.is_python_build() is true. See issue 15300.
+ popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)],
+ stdout=PIPE, stderr=PIPE,
+ universal_newlines=True,
+ close_fds=(os.name != 'nt'),
+ cwd=support.SAVEDCWD)
+ stdout, stderr = popen.communicate()
+ retcode = popen.wait()
+ return retcode, stdout, stderr
+
+
+def main(tests=None, **kwargs):
"""Execute a test suite.
This also parses command-line options and modifies its behavior
@@ -282,7 +479,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
directly to set the values that would normally be set by flags
on the command line.
"""
-
# Display the Python traceback on fatal errors (e.g. segfault)
faulthandler.enable(all_threads=True)
@@ -298,187 +494,51 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
replace_stdout()
support.record_original_stdout(sys.stdout)
- try:
- opts, args = getopt.getopt(sys.argv[1:], 'hvqxsoS:rf:lu:t:TD:NLR:FdwWM:nj:Gm:',
- ['help', 'verbose', 'verbose2', 'verbose3', 'quiet',
- 'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks',
- 'use=', 'threshold=', 'coverdir=', 'nocoverdir',
- 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
- 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug',
- 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait',
- 'failfast', 'match='])
- except getopt.error as msg:
- usage(msg)
- # Defaults
- if random_seed is None:
- random_seed = random.randrange(10000000)
- if use_resources is None:
- use_resources = []
- debug = False
- start = None
- timeout = None
- for o, a in opts:
- if o in ('-h', '--help'):
- print(__doc__)
- return
- elif o in ('-v', '--verbose'):
- verbose += 1
- elif o in ('-w', '--verbose2'):
- verbose2 = True
- elif o in ('-d', '--debug'):
- debug = True
- elif o in ('-W', '--verbose3'):
- verbose3 = True
- elif o in ('-G', '--failfast'):
- failfast = True
- elif o in ('-q', '--quiet'):
- quiet = True;
- verbose = 0
- elif o in ('-x', '--exclude'):
- exclude = True
- elif o in ('-S', '--start'):
- start = a
- elif o in ('-s', '--single'):
- single = True
- elif o in ('-o', '--slow'):
- print_slow = True
- elif o in ('-r', '--randomize'):
- randomize = True
- elif o == '--randseed':
- randomize = True
- random_seed = int(a)
- elif o in ('-f', '--fromfile'):
- fromfile = a
- elif o in ('-m', '--match'):
- match_tests = a
- elif o in ('-l', '--findleaks'):
- findleaks = True
- elif o in ('-L', '--runleaks'):
- runleaks = True
- elif o in ('-t', '--threshold'):
- import gc
- gc.set_threshold(int(a))
- elif o in ('-T', '--coverage'):
- trace = True
- elif o in ('-D', '--coverdir'):
- # CWD is replaced with a temporary dir before calling main(), so we
- # need join it with the saved CWD so it goes where the user expects.
- coverdir = os.path.join(support.SAVEDCWD, a)
- elif o in ('-N', '--nocoverdir'):
- coverdir = None
- elif o in ('-R', '--huntrleaks'):
- huntrleaks = a.split(':')
- if len(huntrleaks) not in (2, 3):
- print(a, huntrleaks)
- usage('-R takes 2 or 3 colon-separated arguments')
- if not huntrleaks[0]:
- huntrleaks[0] = 5
- else:
- huntrleaks[0] = int(huntrleaks[0])
- if not huntrleaks[1]:
- huntrleaks[1] = 4
- else:
- huntrleaks[1] = int(huntrleaks[1])
- if len(huntrleaks) == 2 or not huntrleaks[2]:
- huntrleaks[2:] = ["reflog.txt"]
- # Avoid false positives due to various caches
- # filling slowly with random data:
- warm_caches()
- elif o in ('-M', '--memlimit'):
- support.set_memlimit(a)
- elif o in ('-u', '--use'):
- u = [x.lower() for x in a.split(',')]
- for r in u:
- if r == 'all':
- use_resources[:] = RESOURCE_NAMES
- continue
- if r == 'none':
- del use_resources[:]
- continue
- remove = False
- if r[0] == '-':
- remove = True
- r = r[1:]
- if r not in RESOURCE_NAMES:
- usage('Invalid -u/--use option: ' + a)
- if remove:
- if r in use_resources:
- use_resources.remove(r)
- elif r not in use_resources:
- use_resources.append(r)
- elif o in ('-n', '--nowindows'):
- import msvcrt
- msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
- msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
- msvcrt.SEM_NOGPFAULTERRORBOX|
- msvcrt.SEM_NOOPENFILEERRORBOX)
- try:
- msvcrt.CrtSetReportMode
- except AttributeError:
- # release build
- pass
- else:
- for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
- msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
- msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
- elif o in ('-F', '--forever'):
- forever = True
- elif o in ('-j', '--multiprocess'):
- use_mp = int(a)
- if use_mp <= 0:
- try:
- import multiprocessing
- # Use all cores + extras for tests that like to sleep
- use_mp = 2 + multiprocessing.cpu_count()
- except (ImportError, NotImplementedError):
- use_mp = 3
- if use_mp == 1:
- use_mp = None
- elif o == '--header':
- header = True
- elif o == '--slaveargs':
- args, kwargs = json.loads(a)
- try:
- result = runtest(*args, **kwargs)
- except KeyboardInterrupt:
- result = INTERRUPTED, ''
- except BaseException as e:
- traceback.print_exc()
- result = CHILD_ERROR, str(e)
- sys.stdout.flush()
- print() # Force a newline (just in case)
- print(json.dumps(result))
- sys.exit(0)
- elif o == '--testdir':
- # CWD is replaced with a temporary dir before calling main(), so we
- # join it with the saved CWD so it ends up where the user expects.
- testdir = os.path.join(support.SAVEDCWD, a)
- elif o == '--timeout':
- if hasattr(faulthandler, 'dump_traceback_later'):
- timeout = float(a)
- if timeout <= 0:
- timeout = None
- else:
- print("Warning: The timeout option requires "
- "faulthandler.dump_traceback_later")
- timeout = None
- elif o == '--wait':
- input("Press any key to continue...")
+ ns = _parse_args(sys.argv[1:], **kwargs)
+
+ if ns.huntrleaks:
+ # Avoid false positives due to various caches
+ # filling slowly with random data:
+ warm_caches()
+ if ns.memlimit is not None:
+ support.set_memlimit(ns.memlimit)
+ if ns.threshold is not None:
+ import gc
+ gc.set_threshold(ns.threshold)
+ if ns.nowindows:
+ import msvcrt
+ msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
+ msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
+ msvcrt.SEM_NOGPFAULTERRORBOX|
+ msvcrt.SEM_NOOPENFILEERRORBOX)
+ try:
+ msvcrt.CrtSetReportMode
+ except AttributeError:
+ # release build
+ pass
else:
- print(("No handler for option {}. Please report this as a bug "
- "at http://bugs.python.org.").format(o), file=sys.stderr)
- sys.exit(1)
- if single and fromfile:
- usage("-s and -f don't go together!")
- if use_mp and trace:
- usage("-T and -j don't go together!")
- if use_mp and findleaks:
- usage("-l and -j don't go together!")
- if use_mp and support.max_memuse:
- usage("-M and -j don't go together!")
- if failfast and not (verbose or verbose3):
- usage("-G/--failfast needs either -v or -W")
+ for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
+ msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
+ msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
+ if ns.wait:
+ input("Press any key to continue...")
+
+ if ns.slaveargs is not None:
+ args, kwargs = json.loads(ns.slaveargs)
+ if kwargs.get('huntrleaks'):
+ unittest.BaseTestSuite._cleanup = False
+ try:
+ result = runtest(*args, **kwargs)
+ except KeyboardInterrupt:
+ result = INTERRUPTED, ''
+ except BaseException as e:
+ traceback.print_exc()
+ result = CHILD_ERROR, str(e)
+ sys.stdout.flush()
+ print() # Force a newline (just in case)
+ print(json.dumps(result))
+ sys.exit(0)
good = []
bad = []
@@ -487,12 +547,12 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
environment_changed = []
interrupted = False
- if findleaks:
+ if ns.findleaks:
try:
import gc
except ImportError:
print('No GC available, disabling findleaks.')
- findleaks = False
+ ns.findleaks = False
else:
# Uncomment the line below to report garbage that is not
# freeable by reference counting alone. By default only
@@ -500,82 +560,87 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
#gc.set_debug(gc.DEBUG_SAVEALL)
found_garbage = []
- if single:
+ if ns.huntrleaks:
+ unittest.BaseTestSuite._cleanup = False
+
+ if ns.single:
filename = os.path.join(TEMPDIR, 'pynexttest')
try:
- fp = open(filename, 'r')
- next_test = fp.read().strip()
- tests = [next_test]
- fp.close()
- except IOError:
+ with open(filename, 'r') as fp:
+ next_test = fp.read().strip()
+ tests = [next_test]
+ except OSError:
pass
- if fromfile:
+ if ns.fromfile:
tests = []
- fp = open(os.path.join(support.SAVEDCWD, fromfile))
- count_pat = re.compile(r'\[\s*\d+/\s*\d+\]')
- for line in fp:
- line = count_pat.sub('', line)
- guts = line.split() # assuming no test has whitespace in its name
- if guts and not guts[0].startswith('#'):
- tests.extend(guts)
- fp.close()
+ with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp:
+ count_pat = re.compile(r'\[\s*\d+/\s*\d+\]')
+ for line in fp:
+ line = count_pat.sub('', line)
+ guts = line.split() # assuming no test has whitespace in its name
+ if guts and not guts[0].startswith('#'):
+ tests.extend(guts)
# Strip .py extensions.
- removepy(args)
+ removepy(ns.args)
removepy(tests)
stdtests = STDTESTS[:]
nottests = NOTTESTS.copy()
- if exclude:
- for arg in args:
+ if ns.exclude:
+ for arg in ns.args:
if arg in stdtests:
stdtests.remove(arg)
nottests.add(arg)
- args = []
+ ns.args = []
# For a partial run, we do not need to clutter the output.
- if verbose or header or not (quiet or single or tests or args):
+ if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args):
# Print basic platform information
print("==", platform.python_implementation(), *sys.version.split())
print("== ", platform.platform(aliased=True),
"%s-endian" % sys.byteorder)
+ print("== ", "hash algorithm:", sys.hash_info.algorithm,
+ "64bit" if sys.maxsize > 2**32 else "32bit")
print("== ", os.getcwd())
print("Testing with flags:", sys.flags)
# if testdir is set, then we are not running the python tests suite, so
# don't add default tests to be executed or skipped (pass empty values)
- if testdir:
- alltests = findtests(testdir, list(), set())
+ if ns.testdir:
+ alltests = findtests(ns.testdir, list(), set())
else:
- alltests = findtests(testdir, stdtests, nottests)
+ alltests = findtests(ns.testdir, stdtests, nottests)
- selected = tests or args or alltests
- if single:
+ selected = tests or ns.args or alltests
+ if ns.single:
selected = selected[:1]
try:
next_single_test = alltests[alltests.index(selected[0])+1]
except IndexError:
next_single_test = None
# Remove all the selected tests that precede start if it's set.
- if start:
+ if ns.start:
try:
- del selected[:selected.index(start)]
+ del selected[:selected.index(ns.start)]
except ValueError:
- print("Couldn't find starting test (%s), using all tests" % start)
- if randomize:
- random.seed(random_seed)
- print("Using random seed", random_seed)
+ print("Couldn't find starting test (%s), using all tests" % ns.start)
+ if ns.randomize:
+ if ns.random_seed is None:
+ ns.random_seed = random.randrange(10000000)
+ random.seed(ns.random_seed)
+ print("Using random seed", ns.random_seed)
random.shuffle(selected)
- if trace:
+ if ns.trace:
import trace, tempfile
tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
tempfile.gettempdir()],
trace=False, count=True)
test_times = []
- support.verbose = verbose # Tell tests to be moderately quiet
- support.use_resources = use_resources
+ support.verbose = ns.verbose # Tell tests to be moderately quiet
+ support.use_resources = ns.use_resources
save_modules = sys.modules.keys()
def accumulate_result(test, result):
@@ -593,7 +658,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
skipped.append(test)
resource_denieds.append(test)
- if forever:
+ if ns.forever:
def test_forever(tests=list(selected)):
while True:
for test in tests:
@@ -608,19 +673,16 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
test_count = '/{}'.format(len(selected))
test_count_width = len(test_count) - 1
- if use_mp:
+ if ns.use_mp:
try:
from threading import Thread
except ImportError:
print("Multiprocess option requires thread support")
sys.exit(2)
from queue import Queue
- from subprocess import Popen, PIPE
- debug_output_pat = re.compile(r"\[\d+ refs\]$")
+ debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$")
output = Queue()
pending = MultiprocessTests(tests)
- opt_args = support.args_from_interpreter_flags()
- base_cmd = [sys.executable] + opt_args + ['-m', 'test.regrtest']
def work():
# A worker thread.
try:
@@ -630,24 +692,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
except StopIteration:
output.put((None, None, None, None))
return
- args_tuple = (
- (test, verbose, quiet),
- dict(huntrleaks=huntrleaks, use_resources=use_resources,
- debug=debug, output_on_failure=verbose3,
- timeout=timeout, failfast=failfast,
- match_tests=match_tests)
- )
- # -E is needed by some tests, e.g. test_import
- # Running the child from the same working directory ensures
- # that TEMPDIR for the child is the same when
- # sysconfig.is_python_build() is true. See issue 15300.
- popen = Popen(base_cmd + ['--slaveargs', json.dumps(args_tuple)],
- stdout=PIPE, stderr=PIPE,
- universal_newlines=True,
- close_fds=(os.name != 'nt'),
- cwd=support.SAVEDCWD)
- stdout, stderr = popen.communicate()
- retcode = popen.wait()
+ retcode, stdout, stderr = run_test_in_subprocess(test, ns)
# Strip last refcount output line if it exists, since it
# comes from the shutdown of the interpreter in the subcommand.
stderr = debug_output_pat.sub("", stderr)
@@ -664,19 +709,19 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
except BaseException:
output.put((None, None, None, None))
raise
- workers = [Thread(target=work) for i in range(use_mp)]
+ workers = [Thread(target=work) for i in range(ns.use_mp)]
for worker in workers:
worker.start()
finished = 0
test_index = 1
try:
- while finished < use_mp:
+ while finished < ns.use_mp:
test, stdout, stderr, result = output.get()
if test is None:
finished += 1
continue
accumulate_result(test, result)
- if not quiet:
+ if not ns.quiet:
fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
print(fmt.format(
test_count_width, test_index, test_count,
@@ -699,29 +744,30 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
worker.join()
else:
for test_index, test in enumerate(tests, 1):
- if not quiet:
+ if not ns.quiet:
fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
print(fmt.format(
test_count_width, test_index, test_count, len(bad), test))
sys.stdout.flush()
- if trace:
+ if ns.trace:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
- tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)',
+ tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)',
globals=globals(), locals=vars())
else:
try:
- result = runtest(test, verbose, quiet, huntrleaks, debug,
- output_on_failure=verbose3,
- timeout=timeout, failfast=failfast,
- match_tests=match_tests)
+ result = runtest(test, ns.verbose, ns.quiet,
+ ns.huntrleaks,
+ output_on_failure=ns.verbose3,
+ timeout=ns.timeout, failfast=ns.failfast,
+ match_tests=ns.match_tests)
accumulate_result(test, result)
except KeyboardInterrupt:
interrupted = True
break
except:
raise
- if findleaks:
+ if ns.findleaks:
gc.collect()
if gc.garbage:
print("Warning: test created", len(gc.garbage), end=' ')
@@ -742,11 +788,11 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
omitted = set(selected) - set(good) - set(bad) - set(skipped)
print(count(len(omitted), "test"), "omitted:")
printlist(omitted)
- if good and not quiet:
+ if good and not ns.quiet:
if not bad and not skipped and not interrupted and len(good) > 1:
print("All", end=' ')
print(count(len(good), "test"), "OK.")
- if print_slow:
+ if ns.print_slow:
test_times.sort(reverse=True)
print("10 slowest tests:")
for time, test in test_times[:10]:
@@ -760,32 +806,19 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
print("{} altered the execution environment:".format(
count(len(environment_changed), "test")))
printlist(environment_changed)
- if skipped and not quiet:
+ if skipped and not ns.quiet:
print(count(len(skipped), "test"), "skipped:")
printlist(skipped)
- e = _ExpectedSkips()
- plat = sys.platform
- if e.isvalid():
- surprise = set(skipped) - e.getexpected() - set(resource_denieds)
- if surprise:
- print(count(len(surprise), "skip"), \
- "unexpected on", plat + ":")
- printlist(surprise)
- else:
- print("Those skips are all expected on", plat + ".")
- else:
- print("Ask someone to teach regrtest.py about which tests are")
- print("expected to get skipped on", plat + ".")
-
- if verbose2 and bad:
+ if ns.verbose2 and bad:
print("Re-running failed tests in verbose mode")
for test in bad:
print("Re-running test %r in verbose mode" % test)
sys.stdout.flush()
try:
- verbose = True
- ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout)
+ ns.verbose = True
+ ok = runtest(test, True, ns.quiet, ns.huntrleaks,
+ timeout=ns.timeout)
except KeyboardInterrupt:
# print a newline separate from the ^C
print()
@@ -793,18 +826,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
except:
raise
- if single:
+ if ns.single:
if next_single_test:
with open(filename, 'w') as fp:
fp.write(next_single_test + '\n')
else:
os.unlink(filename)
- if trace:
+ if ns.trace:
r = tracer.results()
- r.write_results(show_missing=True, summary=True, coverdir=coverdir)
+ r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir)
- if runleaks:
+ if ns.runleaks:
os.system("leaks %d" % os.getpid())
sys.exit(len(bad) > 0 or interrupted)
@@ -877,7 +910,7 @@ def replace_stdout():
atexit.register(restore_stdout)
def runtest(test, verbose, quiet,
- huntrleaks=False, debug=False, use_resources=None,
+ huntrleaks=False, use_resources=None,
output_on_failure=False, failfast=False, match_tests=None,
timeout=None):
"""Run a single test.
@@ -885,14 +918,15 @@ def runtest(test, verbose, quiet,
test -- the name of the test
verbose -- if true, print more messages
quiet -- if true, don't print 'skipped' messages (probably redundant)
- test_times -- a list of (time, test_name) pairs
huntrleaks -- run multiple times to test for leaks; requires a debug
build; a triple corresponding to -R's three arguments
+ use_resources -- list of extra resources to use
output_on_failure -- if true, display test output on failure
timeout -- dump the traceback and exit if a test takes more than
timeout seconds
+ failfast, match_tests -- See regrtest command-line flags for these.
- Returns one of the test result constants:
+ Returns the tuple result, test_time, where result is one of the constants:
INTERRUPTED KeyboardInterrupt when run under -j
RESOURCE_DENIED test skipped because resource denied
SKIPPED test skipped for some other reason
@@ -930,7 +964,7 @@ def runtest(test, verbose, quiet,
sys.stdout = stream
sys.stderr = stream
result = runtest_inner(test, verbose, quiet, huntrleaks,
- debug, display_failure=False)
+ display_failure=False)
if result[0] == FAILED:
output = stream.getvalue()
orig_stderr.write(output)
@@ -940,7 +974,7 @@ def runtest(test, verbose, quiet,
sys.stderr = orig_stderr
else:
support.verbose = verbose # Tell tests to be moderately quiet
- result = runtest_inner(test, verbose, quiet, huntrleaks, debug,
+ result = runtest_inner(test, verbose, quiet, huntrleaks,
display_failure=not verbose)
return result
finally:
@@ -992,10 +1026,12 @@ class saved_test_environment:
'os.environ', 'sys.path', 'sys.path_hooks', '__import__',
'warnings.filters', 'asyncore.socket_map',
'logging._handlers', 'logging._handlerList', 'sys.gettrace',
- 'sys.warnoptions', 'threading._dangling',
- 'multiprocessing.process._dangling',
+ 'sys.warnoptions',
+ # multiprocessing.process._cleanup() may release ref
+ # to a thread, so check processes first.
+ 'multiprocessing.process._dangling', 'threading._dangling',
'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES',
- 'support.TESTFN',
+ 'support.TESTFN', 'locale', 'warnings.showwarning',
)
def get_sys_argv(self):
@@ -1123,6 +1159,8 @@ class saved_test_environment:
def get_multiprocessing_process__dangling(self):
if not multiprocessing:
return None
+ # Unjoined process objects can survive after process exits
+ multiprocessing.process._cleanup()
# This copies the weakrefs without making any strong reference
return multiprocessing.process._dangling.copy()
def restore_multiprocessing_process__dangling(self, saved):
@@ -1164,6 +1202,25 @@ class saved_test_environment:
elif os.path.isdir(support.TESTFN):
shutil.rmtree(support.TESTFN)
+ _lc = [getattr(locale, lc) for lc in dir(locale)
+ if lc.startswith('LC_')]
+ def get_locale(self):
+ pairings = []
+ for lc in self._lc:
+ try:
+ pairings.append((lc, locale.setlocale(lc, None)))
+ except (TypeError, ValueError):
+ continue
+ return pairings
+ def restore_locale(self, saved):
+ for lc, setting in saved:
+ locale.setlocale(lc, setting)
+
+ def get_warnings_showwarning(self):
+ return warnings.showwarning
+ def restore_warnings_showwarning(self, fxn):
+ warnings.showwarning = fxn
+
def resource_info(self):
for name in self.resources:
method_suffix = name.replace('.', '_')
@@ -1198,7 +1255,7 @@ class saved_test_environment:
def runtest_inner(test, verbose, quiet,
- huntrleaks=False, debug=False, display_failure=True):
+ huntrleaks=False, display_failure=True):
support.unload(test)
test_time = 0.0
@@ -1211,8 +1268,7 @@ def runtest_inner(test, verbose, quiet,
abstest = 'test.' + test
with saved_test_environment(test, verbose, quiet) as environment:
start_time = time.time()
- the_package = __import__(abstest, globals(), locals(), [])
- the_module = getattr(the_package, test)
+ the_module = importlib.import_module(abstest)
# If the test has a test_main, that will run the appropriate
# tests. If not, use normal unittest test loading.
test_runner = getattr(the_module, "test_main", None)
@@ -1221,8 +1277,7 @@ def runtest_inner(test, verbose, quiet,
test_runner = lambda: support.run_unittest(tests)
test_runner()
if huntrleaks:
- refleak = dash_R(the_module, test, test_runner,
- huntrleaks)
+ refleak = dash_R(the_module, test, test_runner, huntrleaks)
test_time = time.time() - start_time
except support.ResourceDenied as msg:
if not quiet:
@@ -1328,41 +1383,50 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
for obj in abc.__subclasses__() + [abc]:
abcs[obj] = obj._abc_registry.copy()
- if indirect_test:
- def run_the_test():
- indirect_test()
- else:
- def run_the_test():
- del sys.modules[the_module.__name__]
- exec('import ' + the_module.__name__)
-
- deltas = []
nwarmup, ntracked, fname = huntrleaks
fname = os.path.join(support.SAVEDCWD, fname)
repcount = nwarmup + ntracked
+ rc_deltas = [0] * repcount
+ alloc_deltas = [0] * repcount
+
print("beginning", repcount, "repetitions", file=sys.stderr)
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr)
sys.stderr.flush()
- dash_R_cleanup(fs, ps, pic, zdc, abcs)
for i in range(repcount):
- rc_before = sys.gettotalrefcount()
- run_the_test()
+ indirect_test()
+ alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs)
sys.stderr.write('.')
sys.stderr.flush()
- dash_R_cleanup(fs, ps, pic, zdc, abcs)
- rc_after = sys.gettotalrefcount()
if i >= nwarmup:
- deltas.append(rc_after - rc_before)
+ rc_deltas[i] = rc_after - rc_before
+ alloc_deltas[i] = alloc_after - alloc_before
+ alloc_before, rc_before = alloc_after, rc_after
print(file=sys.stderr)
- if any(deltas):
- msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas))
- print(msg, file=sys.stderr)
- sys.stderr.flush()
- with open(fname, "a") as refrep:
- print(msg, file=refrep)
- refrep.flush()
- return True
- return False
+ # These checkers return False on success, True on failure
+ def check_rc_deltas(deltas):
+ return any(deltas)
+ def check_alloc_deltas(deltas):
+ # At least 1/3rd of 0s
+ if 3 * deltas.count(0) < len(deltas):
+ return True
+ # Nothing else than 1s, 0s and -1s
+ if not set(deltas) <= {1,0,-1}:
+ return True
+ return False
+ failed = False
+ for deltas, item_name, checker in [
+ (rc_deltas, 'references', check_rc_deltas),
+ (alloc_deltas, 'memory blocks', check_alloc_deltas)]:
+ if checker(deltas):
+ msg = '%s leaked %s %s, sum=%s' % (
+ test, deltas[nwarmup:], item_name, sum(deltas))
+ print(msg, file=sys.stderr)
+ sys.stderr.flush()
+ with open(fname, "a") as refrep:
+ print(msg, file=refrep)
+ refrep.flush()
+ failed = True
+ return failed
def dash_R_cleanup(fs, ps, pic, zdc, abcs):
import gc, copyreg
@@ -1428,8 +1492,11 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
else:
ctypes._reset_cache()
- # Collect cyclic trash.
+ # Collect cyclic trash and read memory statistics immediately after.
+ func1 = sys.getallocatedblocks
+ func2 = sys.gettotalrefcount
gc.collect()
+ return func1(), func2()
def warm_caches():
# char cache
@@ -1472,307 +1539,10 @@ def printlist(x, width=70, indent=4):
print(fill(' '.join(str(elt) for elt in sorted(x)), width,
initial_indent=blanks, subsequent_indent=blanks))
-# Map sys.platform to a string containing the basenames of tests
-# expected to be skipped on that platform.
-#
-# Special cases:
-# test_pep277
-# The _ExpectedSkips constructor adds this to the set of expected
-# skips if not os.path.supports_unicode_filenames.
-# test_timeout
-# Controlled by test_timeout.skip_expected. Requires the network
-# resource and a socket module.
-#
-# Tests that are expected to be skipped everywhere except on one platform
-# are also handled separately.
-
-_expectations = (
- ('win32',
- """
- test__locale
- test_crypt
- test_curses
- test_dbm
- test_devpoll
- test_fcntl
- test_fork1
- test_epoll
- test_dbm_gnu
- test_dbm_ndbm
- test_grp
- test_ioctl
- test_largefile
- test_kqueue
- test_openpty
- test_ossaudiodev
- test_pipes
- test_poll
- test_posix
- test_pty
- test_pwd
- test_resource
- test_signal
- test_syslog
- test_threadsignals
- test_wait3
- test_wait4
- """),
- ('linux',
- """
- test_curses
- test_devpoll
- test_largefile
- test_kqueue
- test_ossaudiodev
- """),
- ('unixware',
- """
- test_epoll
- test_largefile
- test_kqueue
- test_minidom
- test_openpty
- test_pyexpat
- test_sax
- test_sundry
- """),
- ('openunix',
- """
- test_epoll
- test_largefile
- test_kqueue
- test_minidom
- test_openpty
- test_pyexpat
- test_sax
- test_sundry
- """),
- ('sco_sv',
- """
- test_asynchat
- test_fork1
- test_epoll
- test_gettext
- test_largefile
- test_locale
- test_kqueue
- test_minidom
- test_openpty
- test_pyexpat
- test_queue
- test_sax
- test_sundry
- test_thread
- test_threaded_import
- test_threadedtempfile
- test_threading
- """),
- ('darwin',
- """
- test__locale
- test_curses
- test_devpoll
- test_epoll
- test_dbm_gnu
- test_gdb
- test_largefile
- test_locale
- test_minidom
- test_ossaudiodev
- test_poll
- """),
- ('sunos',
- """
- test_curses
- test_dbm
- test_epoll
- test_kqueue
- test_dbm_gnu
- test_gzip
- test_openpty
- test_zipfile
- test_zlib
- """),
- ('hp-ux',
- """
- test_curses
- test_epoll
- test_dbm_gnu
- test_gzip
- test_largefile
- test_locale
- test_kqueue
- test_minidom
- test_openpty
- test_pyexpat
- test_sax
- test_zipfile
- test_zlib
- """),
- ('cygwin',
- """
- test_curses
- test_dbm
- test_devpoll
- test_epoll
- test_ioctl
- test_kqueue
- test_largefile
- test_locale
- test_ossaudiodev
- test_socketserver
- """),
- ('os2emx',
- """
- test_audioop
- test_curses
- test_epoll
- test_kqueue
- test_largefile
- test_mmap
- test_openpty
- test_ossaudiodev
- test_pty
- test_resource
- test_signal
- """),
- ('freebsd',
- """
- test_devpoll
- test_epoll
- test_dbm_gnu
- test_locale
- test_ossaudiodev
- test_pep277
- test_pty
- test_socketserver
- test_tcl
- test_tk
- test_ttk_guionly
- test_ttk_textonly
- test_timeout
- test_urllibnet
- test_multiprocessing
- """),
- ('aix',
- """
- test_bz2
- test_epoll
- test_dbm_gnu
- test_gzip
- test_kqueue
- test_ossaudiodev
- test_tcl
- test_tk
- test_ttk_guionly
- test_ttk_textonly
- test_zipimport
- test_zlib
- """),
- ('openbsd',
- """
- test_ctypes
- test_devpoll
- test_epoll
- test_dbm_gnu
- test_locale
- test_normalization
- test_ossaudiodev
- test_pep277
- test_tcl
- test_tk
- test_ttk_guionly
- test_ttk_textonly
- test_multiprocessing
- """),
- ('netbsd',
- """
- test_ctypes
- test_curses
- test_devpoll
- test_epoll
- test_dbm_gnu
- test_locale
- test_ossaudiodev
- test_pep277
- test_tcl
- test_tk
- test_ttk_guionly
- test_ttk_textonly
- test_multiprocessing
- """),
-)
-
-class _ExpectedSkips:
- def __init__(self):
- import os.path
- from test import test_timeout
-
- self.valid = False
- expected = None
- for item in _expectations:
- if sys.platform.startswith(item[0]):
- expected = item[1]
- break
- if expected is not None:
- self.expected = set(expected.split())
-
- # These are broken tests, for now skipped on every platform.
- # XXX Fix these!
- self.expected.add('test_nis')
-
- # expected to be skipped on every platform, even Linux
- if not os.path.supports_unicode_filenames:
- self.expected.add('test_pep277')
-
- # doctest, profile and cProfile tests fail when the codec for the
- # fs encoding isn't built in because PyUnicode_Decode() adds two
- # calls into Python.
- encs = ("utf-8", "latin-1", "ascii", "mbcs", "utf-16", "utf-32")
- if sys.getfilesystemencoding().lower() not in encs:
- self.expected.add('test_profile')
- self.expected.add('test_cProfile')
- self.expected.add('test_doctest')
-
- if test_timeout.skip_expected:
- self.expected.add('test_timeout')
-
- if sys.platform != "win32":
- # test_sqlite is only reliable on Windows where the library
- # is distributed with Python
- WIN_ONLY = {"test_unicode_file", "test_winreg",
- "test_winsound", "test_startfile",
- "test_sqlite", "test_msilib"}
- self.expected |= WIN_ONLY
-
- if sys.platform != 'sunos5':
- self.expected.add('test_nis')
-
- if support.python_is_optimized():
- self.expected.add("test_gdb")
-
- self.valid = True
-
- def isvalid(self):
- "Return true iff _ExpectedSkips knows about the current platform."
- return self.valid
-
- def getexpected(self):
- """Return set of test names we expect to skip on current platform.
-
- self.isvalid() must be true.
- """
-
- assert self.isvalid()
- return self.expected
-
-def _make_temp_dir_for_build(TEMPDIR):
- # When tests are run from the Python build directory, it is best practice
- # to keep the test files in a subfolder. It eases the cleanup of leftover
- # files using command "make distclean".
+
+def main_in_temp_cwd():
+ """Run main() in a temporary working directory."""
if sysconfig.is_python_build():
- TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build')
- TEMPDIR = os.path.abspath(TEMPDIR)
try:
os.mkdir(TEMPDIR)
except FileExistsError:
@@ -1781,10 +1551,16 @@ def _make_temp_dir_for_build(TEMPDIR):
# Define a writable temp dir that will be used as cwd while running
# the tests. The name of the dir includes the pid to allow parallel
# testing (see the -j option).
- TESTCWD = 'test_python_{}'.format(os.getpid())
+ test_cwd = 'test_python_{}'.format(os.getpid())
+ test_cwd = os.path.join(TEMPDIR, test_cwd)
+
+ # Run the tests in a context manager that temporarily changes the CWD to a
+ # temporary and writable directory. If it's not possible to create or
+ # change the CWD, the original CWD will be used. The original CWD is
+ # available from support.SAVEDCWD.
+ with support.temp_cwd(test_cwd, quiet=True):
+ main()
- TESTCWD = os.path.join(TEMPDIR, TESTCWD)
- return TEMPDIR, TESTCWD
if __name__ == '__main__':
# Remove regrtest.py's own directory from the module search path. Despite
@@ -1808,11 +1584,4 @@ if __name__ == '__main__':
# sanity check
assert __file__ == os.path.abspath(sys.argv[0])
- TEMPDIR, TESTCWD = _make_temp_dir_for_build(TEMPDIR)
-
- # Run the tests in a context manager that temporary changes the CWD to a
- # temporary and writable directory. If it's not possible to create or
- # change the CWD, the original CWD will be used. The original CWD is
- # available from support.SAVEDCWD.
- with support.temp_cwd(TESTCWD, quiet=True):
- main()
+ main_in_temp_cwd()
diff --git a/Lib/test/revocation.crl b/Lib/test/revocation.crl
new file mode 100644
index 0000000..6d89b08
--- /dev/null
+++ b/Lib/test/revocation.crl
@@ -0,0 +1,11 @@
+-----BEGIN X509 CRL-----
+MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE
+CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j
+YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud
+FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH
++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m
+unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK
+fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC
+UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc
+HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed
+-----END X509 CRL-----
diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py
index ab20164..af0545b 100644
--- a/Lib/test/script_helper.py
+++ b/Lib/test/script_helper.py
@@ -12,13 +12,22 @@ import contextlib
import shutil
import zipfile
-from imp import source_from_cache
+from importlib.util import source_from_cache
from test.support import make_legacy_pyc, strip_python_stderr, temp_dir
# Executing the interpreter in a subprocess
def _assert_python(expected_success, *args, **env_vars):
- cmd_line = [sys.executable]
- if not env_vars:
+ if '__isolated' in env_vars:
+ isolated = env_vars.pop('__isolated')
+ else:
+ isolated = not env_vars
+ cmd_line = [sys.executable, '-X', 'faulthandler']
+ if isolated:
+ # isolated mode: ignore Python environment variables, ignore user
+ # site-packages, and don't add the current directory to sys.path
+ cmd_line.append('-I')
+ elif not env_vars:
+ # ignore Python environment variables
cmd_line.append('-E')
# Need to preserve the original environment, for in-place testing of
# shared library builds.
@@ -39,7 +48,7 @@ def _assert_python(expected_success, *args, **env_vars):
p.stdout.close()
p.stderr.close()
rc = p.returncode
- err = strip_python_stderr(err)
+ err = strip_python_stderr(err)
if (rc and expected_success) or (not rc and not expected_success):
raise AssertionError(
"Process return code is %d, "
@@ -49,18 +58,32 @@ def _assert_python(expected_success, *args, **env_vars):
def assert_python_ok(*args, **env_vars):
"""
Assert that running the interpreter with `args` and optional environment
- variables `env_vars` is ok and return a (return code, stdout, stderr) tuple.
+ variables `env_vars` succeeds (rc == 0) and return a (return code, stdout,
+ stderr) tuple.
+
+ If the __cleanenv keyword is set, env_vars is used a fresh environment.
+
+ Python is started in isolated mode (command line option -I),
+ except if the __isolated keyword is set to False.
"""
return _assert_python(True, *args, **env_vars)
def assert_python_failure(*args, **env_vars):
"""
Assert that running the interpreter with `args` and optional environment
- variables `env_vars` fails and return a (return code, stdout, stderr) tuple.
+ variables `env_vars` fails (rc != 0) and return a (return code, stdout,
+ stderr) tuple.
+
+ See assert_python_ok() for more options.
"""
return _assert_python(False, *args, **env_vars)
def spawn_python(*args, **kw):
+ """Run a Python subprocess with the given arguments.
+
+ kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
+ object.
+ """
cmd_line = [sys.executable, '-E']
cmd_line.extend(args)
return subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
@@ -68,6 +91,7 @@ def spawn_python(*args, **kw):
**kw)
def kill_python(p):
+ """Run the given Popen process until completion and return stdout."""
p.stdin.close()
data = p.stdout.read()
p.stdout.close()
@@ -77,8 +101,10 @@ def kill_python(p):
subprocess._cleanup()
return data
-def make_script(script_dir, script_basename, source):
- script_filename = script_basename+os.extsep+'py'
+def make_script(script_dir, script_basename, source, omit_suffix=False):
+ script_filename = script_basename
+ if not omit_suffix:
+ script_filename += os.extsep + 'py'
script_name = os.path.join(script_dir, script_filename)
# The script should be encoded to UTF-8, the default string encoding
script_file = open(script_name, 'w', encoding='utf-8')
diff --git a/Lib/test/sortperf.py b/Lib/test/sortperf.py
index af7c0b4..90722f7 100644
--- a/Lib/test/sortperf.py
+++ b/Lib/test/sortperf.py
@@ -22,7 +22,7 @@ def randfloats(n):
fn = os.path.join(td, "rr%06d" % n)
try:
fp = open(fn, "rb")
- except IOError:
+ except OSError:
r = random.random
result = [r() for i in range(n)]
try:
@@ -35,9 +35,9 @@ def randfloats(n):
if fp:
try:
os.unlink(fn)
- except os.error:
+ except OSError:
pass
- except IOError as msg:
+ except OSError as msg:
print("can't write", fn, ":", msg)
else:
result = marshal.load(fp)
diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py
index 8686153..759b3f4 100644
--- a/Lib/test/ssl_servers.py
+++ b/Lib/test/ssl_servers.py
@@ -35,7 +35,7 @@ class HTTPSServer(_HTTPServer):
try:
sock, addr = self.socket.accept()
sslconn = self.context.wrap_socket(sock, server_side=True)
- except socket.error as e:
+ except OSError as e:
# socket errors are silenced by the caller, print them here
if support.verbose:
sys.stderr.write("Got an error:\n%s\n" % e)
@@ -147,9 +147,11 @@ class HTTPSServerThread(threading.Thread):
self.server.shutdown()
-def make_https_server(case, certfile=CERTFILE, host=HOST, handler_class=None):
- # we assume the certfile contains both private key and certificate
- context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+def make_https_server(case, *, context=None, certfile=CERTFILE,
+ host=HOST, handler_class=None):
+ if context is None:
+ context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ # We assume the certfile contains both private key and certificate
context.load_cert_chain(certfile)
server = HTTPSServerThread(context, host, handler_class)
flag = threading.Event()
diff --git a/Lib/test/ssltests.py b/Lib/test/ssltests.py
new file mode 100644
index 0000000..9b0ed22
--- /dev/null
+++ b/Lib/test/ssltests.py
@@ -0,0 +1,22 @@
+# Convenience test module to run all of the SSL-related tests in the
+# standard library.
+
+import sys
+import subprocess
+
+TESTS = ['test_asyncio', 'test_ftplib', 'test_hashlib', 'test_httplib',
+ 'test_imaplib', 'test_nntplib', 'test_poplib', 'test_smtplib',
+ 'test_smtpnet', 'test_urllib2_localnet', 'test_venv']
+
+def run_regrtests(*extra_args):
+ args = [sys.executable, "-m", "test"]
+ if not extra_args:
+ args.append("-unetwork")
+ else:
+ args.extend(extra_args)
+ args.extend(TESTS)
+ result = subprocess.call(args)
+ sys.exit(result)
+
+if __name__ == '__main__':
+ run_regrtests(*sys.argv[1:])
diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py
index 4800d6d..5ed01f2 100644
--- a/Lib/test/string_tests.py
+++ b/Lib/test/string_tests.py
@@ -327,13 +327,26 @@ class BaseTest:
self.checkraises(TypeError, 'hello', 'upper', 42)
def test_expandtabs(self):
- self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs')
- self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 8)
- self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 4)
- self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi', 'expandtabs', 4)
- self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs')
- self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi', 'expandtabs', 8)
- self.checkequal('abc\r\nab\r\ndef\ng\r\nhi', 'abc\r\nab\r\ndef\ng\r\nhi', 'expandtabs', 4)
+ self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi',
+ 'expandtabs')
+ self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi',
+ 'expandtabs', 8)
+ self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi',
+ 'expandtabs', 4)
+ self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi',
+ 'expandtabs')
+ self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi',
+ 'expandtabs', 8)
+ self.checkequal('abc\r\nab def\ng hi', 'abc\r\nab\tdef\ng\thi',
+ 'expandtabs', 4)
+ self.checkequal('abc\r\nab\r\ndef\ng\r\nhi', 'abc\r\nab\r\ndef\ng\r\nhi',
+ 'expandtabs', 4)
+ # check keyword args
+ self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi',
+ 'expandtabs', tabsize=8)
+ self.checkequal('abc\rab def\ng hi', 'abc\rab\tdef\ng\thi',
+ 'expandtabs', tabsize=4)
+
self.checkequal(' a\n b', ' \ta\n\tb', 'expandtabs', 1)
self.checkraises(TypeError, 'hello', 'expandtabs', 42, 42)
@@ -1165,7 +1178,8 @@ class MixinStrUnicodeUserStringTest:
self.checkraises(TypeError, 'abc', '__mod__')
self.checkraises(TypeError, '%(foo)s', '__mod__', 42)
self.checkraises(TypeError, '%s%s', '__mod__', (42,))
- self.checkraises(TypeError, '%c', '__mod__', (None,))
+ with self.assertWarns(DeprecationWarning):
+ self.checkraises(TypeError, '%c', '__mod__', (None,))
self.checkraises(ValueError, '%(foo', '__mod__', {})
self.checkraises(TypeError, '%(foo)s %(bar)s', '__mod__', ('foo', 42))
self.checkraises(TypeError, '%d', '__mod__', "42") # not numeric
diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py
index 1f61e13..d12bd95 100644
--- a/Lib/test/subprocessdata/fd_status.py
+++ b/Lib/test/subprocessdata/fd_status.py
@@ -1,17 +1,27 @@
"""When called as a script, print a comma-separated list of the open
-file descriptors on stdout."""
+file descriptors on stdout.
+
+Usage:
+fd_stats.py: check all file descriptors
+fd_status.py fd1 fd2 ...: check only specified file descriptors
+"""
import errno
import os
-
-try:
- _MAXFD = os.sysconf("SC_OPEN_MAX")
-except:
- _MAXFD = 256
+import stat
+import sys
if __name__ == "__main__":
fds = []
- for fd in range(0, _MAXFD):
+ if len(sys.argv) == 1:
+ try:
+ _MAXFD = os.sysconf("SC_OPEN_MAX")
+ except:
+ _MAXFD = 256
+ test_fds = range(0, _MAXFD)
+ else:
+ test_fds = map(int, sys.argv[1:])
+ for fd in test_fds:
try:
st = os.fstat(fd)
except OSError as e:
@@ -19,6 +29,6 @@ if __name__ == "__main__":
continue
raise
# Ignore Solaris door files
- if st.st_mode & 0xF000 != 0xd000:
+ if not stat.S_ISDOOR(st.st_mode):
fds.append(fd)
print(','.join(map(str, fds)))
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 5c03f54..9857f9d 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -15,10 +15,10 @@ import shutil
import warnings
import unittest
import importlib
+import importlib.util
import collections.abc
import re
import subprocess
-import imp
import time
import sysconfig
import fnmatch
@@ -56,26 +56,49 @@ try:
except ImportError:
lzma = None
+try:
+ import resource
+except ImportError:
+ resource = None
+
__all__ = [
- "Error", "TestFailed", "ResourceDenied", "import_module", "verbose",
- "use_resources", "max_memuse", "record_original_stdout",
- "get_original_stdout", "unload", "unlink", "rmtree", "forget",
+ # globals
+ "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast",
+ # exceptions
+ "Error", "TestFailed", "ResourceDenied",
+ # imports
+ "import_module", "import_fresh_module", "CleanImport",
+ # modules
+ "unload", "forget",
+ # io
+ "record_original_stdout", "get_original_stdout", "captured_stdout",
+ "captured_stdin", "captured_stderr",
+ # filesystem
+ "TESTFN", "SAVEDCWD", "unlink", "rmtree", "temp_cwd", "findfile",
+ "create_empty_file", "can_symlink", "fs_is_case_insensitive",
+ # unittest
"is_resource_enabled", "requires", "requires_freebsd_version",
- "requires_linux_version", "requires_mac_ver", "find_unused_port",
- "bind_port", "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD",
- "temp_cwd", "findfile", "create_empty_file", "sortdict",
- "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport",
- "EnvironmentVarGuard", "TransientResource", "captured_stdout",
- "captured_stdin", "captured_stderr", "time_out", "socket_peer_reset",
- "ioerror_peer_reset", "run_with_locale", 'temp_umask',
- "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest",
- "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup",
- "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail",
- "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754",
- "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink",
- "skip_unless_xattr", "import_fresh_module", "requires_zlib",
- "PIPE_MAX_SIZE", "failfast", "anticipate_failure", "run_with_tz",
- "requires_gzip", "requires_bz2", "requires_lzma", "suppress_crash_popup",
+ "requires_linux_version", "requires_mac_ver", "check_syntax_error",
+ "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_reset",
+ "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest",
+ "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma",
+ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute",
+ "requires_IEEE_754", "skip_unless_xattr", "requires_zlib",
+ "anticipate_failure",
+ # sys
+ "is_jython", "check_impl_detail",
+ # network
+ "HOST", "IPV6_ENABLED", "find_unused_port", "bind_port", "open_urlresource",
+ # processes
+ 'temp_umask', "reap_children",
+ # logging
+ "TestHandler",
+ # threads
+ "threading_setup", "threading_cleanup",
+ # miscellaneous
+ "check_warnings", "EnvironmentVarGuard", "run_with_locale", "swap_item",
+ "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
+ "run_with_tz",
]
class Error(Exception):
@@ -97,7 +120,8 @@ def _ignore_deprecated_imports(ignore=True):
"""Context manager to suppress package and module deprecation
warnings when importing them.
- If ignore is False, this context manager has no effect."""
+ If ignore is False, this context manager has no effect.
+ """
if ignore:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", ".+ (module|package)",
@@ -107,16 +131,21 @@ def _ignore_deprecated_imports(ignore=True):
yield
-def import_module(name, deprecated=False):
+def import_module(name, deprecated=False, *, required_on=()):
"""Import and return the module to be tested, raising SkipTest if
it is not available.
If deprecated is True, any module or package deprecation messages
- will be suppressed."""
+ will be suppressed. If a module is required on a platform but optional for
+ others, set required_on to an iterable of platform prefixes which will be
+ compared against sys.platform.
+ """
with _ignore_deprecated_imports(deprecated):
try:
return importlib.import_module(name)
except ImportError as msg:
+ if sys.platform.startswith(tuple(required_on)):
+ raise
raise unittest.SkipTest(str(msg))
@@ -302,25 +331,20 @@ else:
def unlink(filename):
try:
_unlink(filename)
- except OSError as error:
- # The filename need not exist.
- if error.errno not in (errno.ENOENT, errno.ENOTDIR):
- raise
+ except (FileNotFoundError, NotADirectoryError):
+ pass
def rmdir(dirname):
try:
_rmdir(dirname)
- except OSError as error:
- # The directory need not exist.
- if error.errno != errno.ENOENT:
- raise
+ except FileNotFoundError:
+ pass
def rmtree(path):
try:
_rmtree(path)
- except OSError as error:
- if error.errno != errno.ENOENT:
- raise
+ except FileNotFoundError:
+ pass
def make_legacy_pyc(source):
"""Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location.
@@ -332,7 +356,7 @@ def make_legacy_pyc(source):
does not need to exist, however the PEP 3147 pyc file must exist.
:return: The file system path to the legacy pyc file.
"""
- pyc_file = imp.cache_from_source(source)
+ pyc_file = importlib.util.cache_from_source(source)
up_one = os.path.dirname(os.path.abspath(source))
legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o'))
os.rename(pyc_file, legacy_pyc)
@@ -351,8 +375,8 @@ def forget(modname):
# combinations of PEP 3147 and legacy pyc and pyo files.
unlink(source + 'c')
unlink(source + 'o')
- unlink(imp.cache_from_source(source, debug_override=True))
- unlink(imp.cache_from_source(source, debug_override=False))
+ unlink(importlib.util.cache_from_source(source, debug_override=True))
+ unlink(importlib.util.cache_from_source(source, debug_override=False))
# On some platforms, should not run gui test even if it is allowed
# in `use_resources'.
@@ -513,7 +537,7 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
the SO_REUSEADDR socket option having different semantics on Windows versus
Unix/Linux. On Unix, you can't have two AF_INET SOCK_STREAM sockets bind,
listen and then accept connections on identical host/ports. An EADDRINUSE
- socket.error will be raised at some point (depending on the platform and
+ OSError will be raised at some point (depending on the platform and
the order bind and listen were called on each socket).
However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE
@@ -591,9 +615,9 @@ def _is_ipv6_enabled():
sock = None
try:
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
- sock.bind(('::1', 0))
+ sock.bind((HOSTv6, 0))
return True
- except (socket.error, socket.gaierror):
+ except OSError:
pass
finally:
if sock:
@@ -1180,9 +1204,9 @@ class TransientResource(object):
# Context managers that raise ResourceDenied when various issues
# with the Internet connection manifest themselves as exceptions.
# XXX deprecate these and use transient_internet() instead
-time_out = TransientResource(IOError, errno=errno.ETIMEDOUT)
-socket_peer_reset = TransientResource(socket.error, errno=errno.ECONNRESET)
-ioerror_peer_reset = TransientResource(IOError, errno=errno.ECONNRESET)
+time_out = TransientResource(OSError, errno=errno.ETIMEDOUT)
+socket_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET)
+ioerror_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET)
@contextlib.contextmanager
@@ -1228,17 +1252,17 @@ def transient_internet(resource_name, *, timeout=30.0, errnos=()):
if timeout is not None:
socket.setdefaulttimeout(timeout)
yield
- except IOError as err:
+ except OSError as err:
# urllib can wrap original socket errors multiple times (!), we must
# unwrap to get at the original error.
while True:
a = err.args
- if len(a) >= 1 and isinstance(a[0], IOError):
+ if len(a) >= 1 and isinstance(a[0], OSError):
err = a[0]
# The error can also be wrapped as args[1]:
# except socket.error as msg:
- # raise IOError('socket error', msg).with_traceback(sys.exc_info()[2])
- elif len(a) >= 2 and isinstance(a[1], IOError):
+ # raise OSError('socket error', msg).with_traceback(sys.exc_info()[2])
+ elif len(a) >= 2 and isinstance(a[1], OSError):
err = a[1]
else:
break
@@ -1697,9 +1721,18 @@ def run_unittest(*classes):
#=======================================================================
# Check for the presence of docstrings.
-HAVE_DOCSTRINGS = (check_impl_detail(cpython=False) or
- sys.platform == 'win32' or
- sysconfig.get_config_var('WITH_DOC_STRINGS'))
+# Rather than trying to enumerate all the cases where docstrings may be
+# disabled, we just check for that directly
+
+def _check_docstrings():
+ """Just used to check if docstrings are enabled"""
+
+MISSING_C_DOCSTRINGS = (check_impl_detail() and
+ sys.platform != 'win32' and
+ not sysconfig.get_config_var('WITH_DOC_STRINGS'))
+
+HAVE_DOCSTRINGS = (_check_docstrings.__doc__ is not None and
+ not MISSING_C_DOCSTRINGS)
requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS,
"test requires docstrings")
@@ -1774,12 +1807,12 @@ def threading_setup():
def threading_cleanup(*original_values):
if not _thread:
return
- _MAX_COUNT = 10
+ _MAX_COUNT = 100
for count in range(_MAX_COUNT):
values = _thread._count(), threading._dangling
if values == original_values:
break
- time.sleep(0.1)
+ time.sleep(0.01)
gc_collect()
# XXX print a warning in case of failure?
@@ -1881,7 +1914,7 @@ def strip_python_stderr(stderr):
This will typically be run on the result of the communicate() method
of a subprocess.Popen object.
"""
- stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
+ stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip()
return stderr
def args_from_interpreter_flags():
@@ -2012,27 +2045,81 @@ def skip_unless_xattr(test):
return test if ok else unittest.skip(msg)(test)
-if sys.platform.startswith('win'):
- @contextlib.contextmanager
- def suppress_crash_popup():
- """Disable Windows Error Reporting dialogs using SetErrorMode."""
- # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621%28v=vs.85%29.aspx
- # GetErrorMode is not available on Windows XP and Windows Server 2003,
- # but SetErrorMode returns the previous value, so we can use that
- import ctypes
- k32 = ctypes.windll.kernel32
- SEM_NOGPFAULTERRORBOX = 0x02
- old_error_mode = k32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
- k32.SetErrorMode(old_error_mode | SEM_NOGPFAULTERRORBOX)
- try:
- yield
- finally:
- k32.SetErrorMode(old_error_mode)
-else:
- # this is a no-op for other platforms
- @contextlib.contextmanager
- def suppress_crash_popup():
- yield
+def fs_is_case_insensitive(directory):
+ """Detects if the file system for the specified directory is case-insensitive."""
+ base_fp, base_path = tempfile.mkstemp(dir=directory)
+ case_path = base_path.upper()
+ if case_path == base_path:
+ case_path = base_path.lower()
+ try:
+ return os.path.samefile(base_path, case_path)
+ except FileNotFoundError:
+ return False
+ finally:
+ os.unlink(base_path)
+
+
+class SuppressCrashReport:
+ """Try to prevent a crash report from popping up.
+
+ On Windows, don't display the Windows Error Reporting dialog. On UNIX,
+ disable the creation of coredump file.
+ """
+ old_value = None
+
+ def __enter__(self):
+ """On Windows, disable Windows Error Reporting dialogs using
+ SetErrorMode.
+
+ On UNIX, try to save the previous core file size limit, then set
+ soft limit to 0.
+ """
+ if sys.platform.startswith('win'):
+ # see http://msdn.microsoft.com/en-us/library/windows/desktop/ms680621.aspx
+ # GetErrorMode is not available on Windows XP and Windows Server 2003,
+ # but SetErrorMode returns the previous value, so we can use that
+ import ctypes
+ self._k32 = ctypes.windll.kernel32
+ SEM_NOGPFAULTERRORBOX = 0x02
+ self.old_value = self._k32.SetErrorMode(SEM_NOGPFAULTERRORBOX)
+ self._k32.SetErrorMode(self.old_value | SEM_NOGPFAULTERRORBOX)
+ else:
+ if resource is not None:
+ try:
+ self.old_value = resource.getrlimit(resource.RLIMIT_CORE)
+ resource.setrlimit(resource.RLIMIT_CORE,
+ (0, self.old_value[1]))
+ except (ValueError, OSError):
+ pass
+ if sys.platform == 'darwin':
+ # Check if the 'Crash Reporter' on OSX was configured
+ # in 'Developer' mode and warn that it will get triggered
+ # when it is.
+ #
+ # This assumes that this context manager is used in tests
+ # that might trigger the next manager.
+ value = subprocess.Popen(['/usr/bin/defaults', 'read',
+ 'com.apple.CrashReporter', 'DialogType'],
+ stdout=subprocess.PIPE).communicate()[0]
+ if value.strip() == b'developer':
+ print("this test triggers the Crash Reporter, "
+ "that is intentional", end='', flush=True)
+
+ return self
+
+ def __exit__(self, *ignore_exc):
+ """Restore Windows ErrorMode or core file behavior to initial value."""
+ if self.old_value is None:
+ return
+
+ if sys.platform.startswith('win'):
+ self._k32.SetErrorMode(self.old_value)
+ else:
+ if resource is not None:
+ try:
+ resource.setrlimit(resource.RLIMIT_CORE, self.old_value)
+ except (ValueError, OSError):
+ pass
def patch(test_instance, object_to_patch, attr_name, new_value):
@@ -2067,3 +2154,23 @@ def patch(test_instance, object_to_patch, attr_name, new_value):
# actually override the attribute
setattr(object_to_patch, attr_name, new_value)
+
+
+def run_in_subinterp(code):
+ """
+ Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc
+ module is enabled.
+ """
+ # Issue #10915, #15751: PyGILState_*() functions don't work with
+ # sub-interpreters, the tracemalloc module uses these functions internally
+ try:
+ import tracemalloc
+ except ImportError:
+ pass
+ else:
+ if tracemalloc.is_tracing():
+ raise unittest.SkipTest("run_in_subinterp() cannot be used "
+ "if tracemalloc module is tracing "
+ "memory allocations")
+ import _testcapi
+ return _testcapi.run_in_subinterp(code)
diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py
index 608ec01..8cc285f 100644
--- a/Lib/test/test___all__.py
+++ b/Lib/test/test___all__.py
@@ -29,17 +29,20 @@ class AllTest(unittest.TestCase):
if not hasattr(sys.modules[modname], "__all__"):
raise NoAll(modname)
names = {}
- try:
- exec("from %s import *" % modname, names)
- except Exception as e:
- # Include the module name in the exception string
- self.fail("__all__ failure in {}: {}: {}".format(
- modname, e.__class__.__name__, e))
- if "__builtins__" in names:
- del names["__builtins__"]
- keys = set(names)
- all = set(sys.modules[modname].__all__)
- self.assertEqual(keys, all)
+ with self.subTest(module=modname):
+ try:
+ exec("from %s import *" % modname, names)
+ except Exception as e:
+ # Include the module name in the exception string
+ self.fail("__all__ failure in {}: {}: {}".format(
+ modname, e.__class__.__name__, e))
+ if "__builtins__" in names:
+ del names["__builtins__"]
+ keys = set(names)
+ all_list = sys.modules[modname].__all__
+ all_set = set(all_list)
+ self.assertCountEqual(all_set, all_list, "in module {}".format(modname))
+ self.assertEqual(keys, all_set, "in module {}".format(modname))
def walk_modules(self, basedir, modpath):
for fn in sorted(os.listdir(basedir)):
@@ -110,8 +113,5 @@ class AllTest(unittest.TestCase):
print('Following modules failed to be imported:', failed_imports)
-def test_main():
- support.run_unittest(AllTest)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py
new file mode 100644
index 0000000..0152e9d
--- /dev/null
+++ b/Lib/test/test__opcode.py
@@ -0,0 +1,23 @@
+import dis
+from test.support import run_unittest, import_module
+import unittest
+
+_opcode = import_module("_opcode")
+
+class OpcodeTests(unittest.TestCase):
+
+ def test_stack_effect(self):
+ self.assertEqual(_opcode.stack_effect(dis.opmap['POP_TOP']), -1)
+ self.assertEqual(_opcode.stack_effect(dis.opmap['DUP_TOP_TWO']), 2)
+ self.assertEqual(_opcode.stack_effect(dis.opmap['BUILD_SLICE'], 0), -1)
+ self.assertEqual(_opcode.stack_effect(dis.opmap['BUILD_SLICE'], 1), -1)
+ self.assertEqual(_opcode.stack_effect(dis.opmap['BUILD_SLICE'], 3), -2)
+ self.assertRaises(ValueError, _opcode.stack_effect, 30000)
+ self.assertRaises(ValueError, _opcode.stack_effect, dis.opmap['BUILD_SLICE'])
+ self.assertRaises(ValueError, _opcode.stack_effect, dis.opmap['POP_TOP'], 0)
+
+def test_main():
+ run_unittest(OpcodeTests)
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py
index d4d7556..93f9dae 100644
--- a/Lib/test/test_abc.py
+++ b/Lib/test/test_abc.py
@@ -68,6 +68,19 @@ class TestLegacyAPI(unittest.TestCase):
class TestABC(unittest.TestCase):
+ def test_ABC_helper(self):
+ # create an ABC using the helper class and perform basic checks
+ class C(abc.ABC):
+ @classmethod
+ @abc.abstractmethod
+ def foo(cls): return cls.__name__
+ self.assertEqual(type(C), abc.ABCMeta)
+ self.assertRaises(TypeError, C)
+ class D(C):
+ @classmethod
+ def foo(cls): return super().foo()
+ self.assertEqual(D.foo(), 'D')
+
def test_abstractmethod_basics(self):
@abc.abstractmethod
def foo(self): pass
@@ -288,7 +301,10 @@ class TestABC(unittest.TestCase):
b = B()
self.assertFalse(isinstance(b, A))
self.assertFalse(isinstance(b, (A,)))
+ token_old = abc.get_cache_token()
A.register(B)
+ token_new = abc.get_cache_token()
+ self.assertNotEqual(token_old, token_new)
self.assertTrue(isinstance(b, A))
self.assertTrue(isinstance(b, (A,)))
diff --git a/Lib/test/test_aifc.py b/Lib/test/test_aifc.py
index 10260e3..ab51437 100644
--- a/Lib/test/test_aifc.py
+++ b/Lib/test/test_aifc.py
@@ -1,6 +1,7 @@
from test.support import findfile, TESTFN, unlink
import unittest
from test import audiotests
+from audioop import byteswap
import os
import io
import sys
@@ -119,7 +120,7 @@ class AifcULAWTest(AifcTest, unittest.TestCase):
E5040CBC 617C0A3C 08BC0A3C 2C7C0B3C 517C0E3C 8A8410FC B6840EBC 457C0A3C \
""")
if sys.byteorder != 'big':
- frames = audiotests.byteswap2(frames)
+ frames = byteswap(frames, 2)
class AifcALAWTest(AifcTest, unittest.TestCase):
@@ -140,7 +141,7 @@ class AifcALAWTest(AifcTest, unittest.TestCase):
E4800CC0 62000A40 08C00A40 2B000B40 52000E40 8A001180 B6000EC0 46000A40 \
""")
if sys.byteorder != 'big':
- frames = audiotests.byteswap2(frames)
+ frames = byteswap(frames, 2)
class AifcMiscTest(audiotests.AudioTests, unittest.TestCase):
@@ -149,6 +150,21 @@ class AifcMiscTest(audiotests.AudioTests, unittest.TestCase):
#This file contains chunk types aifc doesn't recognize.
self.f = aifc.open(findfile('Sine-1000Hz-300ms.aif'))
+ def test_params_added(self):
+ f = self.f = aifc.open(TESTFN, 'wb')
+ f.aiff()
+ f.setparams((1, 1, 1, 1, b'NONE', b''))
+ f.close()
+
+ f = self.f = aifc.open(TESTFN, 'rb')
+ params = f.getparams()
+ self.assertEqual(params.nchannels, f.getnchannels())
+ self.assertEqual(params.sampwidth, f.getsampwidth())
+ self.assertEqual(params.framerate, f.getframerate())
+ self.assertEqual(params.nframes, f.getnframes())
+ self.assertEqual(params.comptype, f.getcomptype())
+ self.assertEqual(params.compname, f.getcompname())
+
def test_write_header_comptype_sampwidth(self):
for comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
fout = aifc.open(io.BytesIO(), 'wb')
@@ -352,12 +368,14 @@ class AIFCLowLevelTest(unittest.TestCase):
def test_write_aiff_by_extension(self):
sampwidth = 2
- fout = self.fout = aifc.open(TESTFN + '.aiff', 'wb')
+ filename = TESTFN + '.aiff'
+ fout = self.fout = aifc.open(filename, 'wb')
+ self.addCleanup(unlink, filename)
fout.setparams((1, sampwidth, 1, 1, b'ULAW', b''))
frames = b'\x00' * fout.getnchannels() * sampwidth
fout.writeframes(frames)
fout.close()
- f = self.f = aifc.open(TESTFN + '.aiff', 'rb')
+ f = self.f = aifc.open(filename, 'rb')
self.assertEqual(f.getcomptype(), b'NONE')
f.close()
diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py
index 9cb15c7..25ba9e6 100644
--- a/Lib/test/test_argparse.py
+++ b/Lib/test/test_argparse.py
@@ -14,6 +14,7 @@ import argparse
from io import StringIO
from test import support
+from unittest import mock
class StdIOBuffer(StringIO):
pass
@@ -1421,6 +1422,19 @@ class TestFileTypeRepr(TestCase):
type = argparse.FileType('wb', 1)
self.assertEqual("FileType('wb', 1)", repr(type))
+ def test_r_latin(self):
+ type = argparse.FileType('r', encoding='latin_1')
+ self.assertEqual("FileType('r', encoding='latin_1')", repr(type))
+
+ def test_w_big5_ignore(self):
+ type = argparse.FileType('w', encoding='big5', errors='ignore')
+ self.assertEqual("FileType('w', encoding='big5', errors='ignore')",
+ repr(type))
+
+ def test_r_1_replace(self):
+ type = argparse.FileType('r', 1, errors='replace')
+ self.assertEqual("FileType('r', 1, errors='replace')", repr(type))
+
class RFile(object):
seen = {}
@@ -1557,6 +1571,24 @@ class TestFileTypeWB(TempDirMixin, ParserTestCase):
]
+class TestFileTypeOpenArgs(TestCase):
+ """Test that open (the builtin) is correctly called"""
+
+ def test_open_args(self):
+ FT = argparse.FileType
+ cases = [
+ (FT('rb'), ('rb', -1, None, None)),
+ (FT('w', 1), ('w', 1, None, None)),
+ (FT('w', errors='replace'), ('w', -1, None, 'replace')),
+ (FT('wb', encoding='big5'), ('wb', -1, 'big5', None)),
+ (FT('w', 0, 'l1', 'strict'), ('w', 0, 'l1', 'strict')),
+ ]
+ with mock.patch('builtins.open') as m:
+ for type, args in cases:
+ type('foo')
+ m.assert_called_with('foo', *args)
+
+
class TestTypeCallable(ParserTestCase):
"""Test some callables as option/argument types"""
@@ -4381,7 +4413,7 @@ class TestOptionalsHelpVersionActions(TestCase):
def test_version_format(self):
parser = ErrorRaisingArgumentParser(prog='PPP')
parser.add_argument('-v', '--version', action='version', version='%(prog)s 3.5')
- msg = self._get_error(parser.parse_args, ['-v']).stderr
+ msg = self._get_error(parser.parse_args, ['-v']).stdout
self.assertEqual('PPP 3.5\n', msg)
def test_version_no_help(self):
@@ -4394,7 +4426,7 @@ class TestOptionalsHelpVersionActions(TestCase):
def test_version_action(self):
parser = ErrorRaisingArgumentParser(prog='XXX')
parser.add_argument('-V', action='version', version='%(prog)s 3.7')
- msg = self._get_error(parser.parse_args, ['-V']).stderr
+ msg = self._get_error(parser.parse_args, ['-V']).stdout
self.assertEqual('XXX 3.7\n', msg)
def test_no_help(self):
diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
index d68284f..f8dbf06 100644
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -353,12 +353,12 @@ class BaseTest:
support.unlink(support.TESTFN)
def test_fromfile_ioerror(self):
- # Issue #5395: Check if fromfile raises a proper IOError
+ # Issue #5395: Check if fromfile raises a proper OSError
# instead of EOFError.
a = array.array(self.typecode)
f = open(support.TESTFN, 'wb')
try:
- self.assertRaises(IOError, a.fromfile, f, len(self.example))
+ self.assertRaises(OSError, a.fromfile, f, len(self.example))
finally:
f.close()
support.unlink(support.TESTFN)
@@ -1026,6 +1026,18 @@ class BaseTest:
basesize = support.calcvobjsize('Pn2Pi')
support.check_sizeof(self, a, basesize)
+ def test_initialize_with_unicode(self):
+ if self.typecode != 'u':
+ with self.assertRaises(TypeError) as cm:
+ a = array.array(self.typecode, 'foo')
+ self.assertIn("cannot use a str", str(cm.exception))
+ with self.assertRaises(TypeError) as cm:
+ a = array.array(self.typecode, array.array('u', 'foo'))
+ self.assertIn("cannot use a unicode array", str(cm.exception))
+ else:
+ a = array.array(self.typecode, "foo")
+ a = array.array(self.typecode, array.array('u', 'foo'))
+
class StringTest(BaseTest):
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 6352888..f17c93f 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -37,7 +37,7 @@ exec_tests = [
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args
- "def f(a, b=1, c=None, d=[], e={}, *args, **kwargs): pass",
+ "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): pass",
# ClassDef
"class C:pass",
# ClassDef, new style class
@@ -180,20 +180,36 @@ eval_tests = [
class AST_Tests(unittest.TestCase):
- def _assertTrueorder(self, ast_node, parent_pos):
+ def _assertTrueorder(self, ast_node, parent_pos, reverse_check = False):
+ def should_reverse_check(parent, child):
+ # In some situations, the children of nodes occur before
+ # their parents, for example in a.b.c, a occurs before b
+ # but a is a child of b.
+ if isinstance(parent, ast.Call):
+ if parent.func == child:
+ return True
+ if isinstance(parent, (ast.Attribute, ast.Subscript)):
+ return True
+ return False
+
if not isinstance(ast_node, ast.AST) or ast_node._fields is None:
return
if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)):
node_pos = (ast_node.lineno, ast_node.col_offset)
- self.assertTrue(node_pos >= parent_pos)
+ if reverse_check:
+ self.assertTrue(node_pos <= parent_pos)
+ else:
+ self.assertTrue(node_pos >= parent_pos)
parent_pos = (ast_node.lineno, ast_node.col_offset)
for name in ast_node._fields:
value = getattr(ast_node, name)
if isinstance(value, list):
for child in value:
- self._assertTrueorder(child, parent_pos)
+ self._assertTrueorder(child, parent_pos,
+ should_reverse_check(ast_node, child))
elif value is not None:
- self._assertTrueorder(value, parent_pos)
+ self._assertTrueorder(value, parent_pos,
+ should_reverse_check(ast_node, value))
def test_AST_objects(self):
x = ast.AST()
@@ -262,14 +278,14 @@ class AST_Tests(unittest.TestCase):
def test_arguments(self):
x = ast.arguments()
- self.assertEqual(x._fields, ('args', 'vararg', 'varargannotation',
- 'kwonlyargs', 'kwarg', 'kwargannotation',
- 'defaults', 'kw_defaults'))
+ self.assertEqual(x._fields, ('args', 'vararg',
+ 'kwonlyargs', 'kw_defaults',
+ 'kwarg', 'defaults'))
with self.assertRaises(AttributeError):
x.vararg
- x = ast.arguments(*range(1, 9))
+ x = ast.arguments(*range(1, 7))
self.assertEqual(x.vararg, 2)
def test_field_attr_writable(self):
@@ -439,7 +455,7 @@ class ASTHelpers_Test(unittest.TestCase):
"lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
"lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
"col_offset=11)], keywords=[], starargs=None, kwargs=None, "
- "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
+ "lineno=1, col_offset=4), lineno=1, col_offset=0)])"
)
def test_copy_location(self):
@@ -460,7 +476,7 @@ class ASTHelpers_Test(unittest.TestCase):
"Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
"lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, "
"col_offset=6)], keywords=[], starargs=None, kwargs=None, "
- "lineno=1, col_offset=0), lineno=1, col_offset=0), "
+ "lineno=1, col_offset=5), lineno=1, col_offset=0), "
"Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
"col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
"keywords=[], starargs=None, kwargs=None, lineno=1, "
@@ -560,8 +576,8 @@ class ASTValidatorTests(unittest.TestCase):
self.mod(m, "must have Load context", "eval")
def _check_arguments(self, fac, check):
- def arguments(args=None, vararg=None, varargannotation=None,
- kwonlyargs=None, kwarg=None, kwargannotation=None,
+ def arguments(args=None, vararg=None,
+ kwonlyargs=None, kwarg=None,
defaults=None, kw_defaults=None):
if args is None:
args = []
@@ -571,20 +587,12 @@ class ASTValidatorTests(unittest.TestCase):
defaults = []
if kw_defaults is None:
kw_defaults = []
- args = ast.arguments(args, vararg, varargannotation, kwonlyargs,
- kwarg, kwargannotation, defaults, kw_defaults)
+ args = ast.arguments(args, vararg, kwonlyargs, kw_defaults,
+ kwarg, defaults)
return fac(args)
args = [ast.arg("x", ast.Name("x", ast.Store()))]
check(arguments(args=args), "must have Load context")
- check(arguments(varargannotation=ast.Num(3)),
- "varargannotation but no vararg")
- check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"),
- "must have Load context")
check(arguments(kwonlyargs=args), "must have Load context")
- check(arguments(kwargannotation=ast.Num(42)),
- "kwargannotation but no kwarg")
- check(arguments(kwargannotation=ast.Name("x", ast.Store()),
- kwarg="x"), "must have Load context")
check(arguments(defaults=[ast.Num(3)]),
"more positional defaults than args")
check(arguments(kw_defaults=[ast.Num(4)]),
@@ -599,7 +607,7 @@ class ASTValidatorTests(unittest.TestCase):
"must have Load context")
def test_funcdef(self):
- a = ast.arguments([], None, None, [], None, None, [], [])
+ a = ast.arguments([], None, [], [], None, [])
f = ast.FunctionDef("x", a, [], [], None)
self.stmt(f, "empty body on FunctionDef")
f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())],
@@ -770,7 +778,7 @@ class ASTValidatorTests(unittest.TestCase):
self.expr(u, "must have Load context")
def test_lambda(self):
- a = ast.arguments([], None, None, [], None, None, [], [])
+ a = ast.arguments([], None, [], [], None, [])
self.expr(ast.Lambda(a, ast.Name("x", ast.Store())),
"must have Load context")
def fac(args):
@@ -928,6 +936,9 @@ class ASTValidatorTests(unittest.TestCase):
def test_tuple(self):
self._sequence(ast.Tuple)
+ def test_nameconstant(self):
+ self.expr(ast.NameConstant(4), "singleton must be True, False, or None")
+
def test_stdlib_validates(self):
stdlib = os.path.dirname(ast.__file__)
tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")]
@@ -936,13 +947,10 @@ class ASTValidatorTests(unittest.TestCase):
fn = os.path.join(stdlib, module)
with open(fn, "r", encoding="utf-8") as fp:
source = fp.read()
- mod = ast.parse(source)
+ mod = ast.parse(source, fn)
compile(mod, fn, "exec")
-def test_main():
- support.run_unittest(AST_Tests, ASTHelpers_Test, ASTValidatorTests)
-
def main():
if __name__ != '__main__':
return
@@ -955,20 +963,20 @@ def main():
print("]")
print("main()")
raise SystemExit
- test_main()
+ unittest.main()
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
-('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
+('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
+ ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 58))], [], None)]),
('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
@@ -977,7 +985,7 @@ exec_results = [
('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
-('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
+('Module', [('Raise', (1, 0), ('Call', (1, 15), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
@@ -1002,29 +1010,29 @@ single_results = [
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
]
eval_results = [
-('Expression', ('Name', (1, 0), 'None', ('Load',))),
+('Expression', ('NameConstant', (1, 0), None)),
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
-('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('NameConstant', (1, 7), None))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('Dict', (1, 0), [], [])),
-('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
-('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
+('Expression', ('Call', (1, 1), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
('Expression', ('Num', (1, 0), 10)),
('Expression', ('Str', (1, 0), 'string')),
-('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
-('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
+('Expression', ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
+('Expression', ('Subscript', (1, 2), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
('Expression', ('Name', (1, 0), 'v', ('Load',))),
('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
('Expression', ('List', (1, 0), [], ('Load',))),
('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
('Expression', ('Tuple', (1, 0), [], ('Load',))),
-('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
+('Expression', ('Call', (1, 7), ('Attribute', (1, 6), ('Attribute', (1, 4), ('Attribute', (1, 2), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 12), ('Attribute', (1, 10), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
]
main()
diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py
index c79fe6f..f93a52d 100644
--- a/Lib/test/test_asynchat.py
+++ b/Lib/test/test_asynchat.py
@@ -15,6 +15,7 @@ except ImportError:
HOST = support.HOST
SERVER_QUIT = b'QUIT\n'
+TIMEOUT = 3.0
if threading:
class echo_server(threading.Thread):
@@ -123,7 +124,9 @@ class TestAsynchat(unittest.TestCase):
c.push(b"I'm not dead yet!" + term)
c.push(SERVER_QUIT)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
@@ -154,7 +157,9 @@ class TestAsynchat(unittest.TestCase):
c.push(data)
c.push(SERVER_QUIT)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [data[:termlen]])
@@ -174,7 +179,9 @@ class TestAsynchat(unittest.TestCase):
c.push(data)
c.push(SERVER_QUIT)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [])
self.assertEqual(c.buffer, data)
@@ -186,7 +193,9 @@ class TestAsynchat(unittest.TestCase):
p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8)
c.push_with_producer(p)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
@@ -196,7 +205,9 @@ class TestAsynchat(unittest.TestCase):
data = b"hello world\nI'm not dead yet!\n"
c.push_with_producer(data+SERVER_QUIT)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"])
@@ -207,7 +218,9 @@ class TestAsynchat(unittest.TestCase):
c.push(b"hello world\n\nI'm not dead yet!\n")
c.push(SERVER_QUIT)
asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01)
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents,
[b"hello world", b"", b"I'm not dead yet!"])
@@ -226,7 +239,9 @@ class TestAsynchat(unittest.TestCase):
# where the server echoes all of its data before we can check that it
# got any down below.
s.start_resend_event.set()
- s.join()
+ s.join(timeout=TIMEOUT)
+ if s.is_alive():
+ self.fail("join() timed out")
self.assertEqual(c.contents, [])
# the server might have been able to send a byte or two back, but this
@@ -268,9 +283,5 @@ class TestFifo(unittest.TestCase):
self.assertEqual(f.pop(), (0, None))
-def test_main(verbose=None):
- support.run_unittest(TestAsynchat, TestAsynchat_WithPoll,
- TestHelperFunctions, TestFifo)
-
if __name__ == "__main__":
- test_main(verbose=True)
+ unittest.main()
diff --git a/Lib/test/test_asyncio/__init__.py b/Lib/test/test_asyncio/__init__.py
new file mode 100644
index 0000000..82158af
--- /dev/null
+++ b/Lib/test/test_asyncio/__init__.py
@@ -0,0 +1,29 @@
+import os
+import sys
+import unittest
+from test.support import run_unittest, import_module
+
+# Skip tests if we don't have threading.
+import_module('threading')
+# Skip tests if we don't have concurrent.futures.
+import_module('concurrent.futures')
+
+
+def suite():
+ tests = unittest.TestSuite()
+ loader = unittest.TestLoader()
+ for fn in os.listdir(os.path.dirname(__file__)):
+ if fn.startswith("test") and fn.endswith(".py"):
+ mod_name = 'test.test_asyncio.' + fn[:-3]
+ try:
+ __import__(mod_name)
+ except unittest.SkipTest:
+ pass
+ else:
+ mod = sys.modules[mod_name]
+ tests.addTests(loader.loadTestsFromModule(mod))
+ return tests
+
+
+def test_main():
+ run_unittest(suite())
diff --git a/Lib/test/test_asyncio/__main__.py b/Lib/test/test_asyncio/__main__.py
new file mode 100644
index 0000000..b549492
--- /dev/null
+++ b/Lib/test/test_asyncio/__main__.py
@@ -0,0 +1,5 @@
+from . import test_main
+
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_asyncio/echo.py b/Lib/test/test_asyncio/echo.py
new file mode 100644
index 0000000..006364b
--- /dev/null
+++ b/Lib/test/test_asyncio/echo.py
@@ -0,0 +1,8 @@
+import os
+
+if __name__ == '__main__':
+ while True:
+ buf = os.read(0, 1024)
+ if not buf:
+ break
+ os.write(1, buf)
diff --git a/Lib/test/test_asyncio/echo2.py b/Lib/test/test_asyncio/echo2.py
new file mode 100644
index 0000000..e83ca09
--- /dev/null
+++ b/Lib/test/test_asyncio/echo2.py
@@ -0,0 +1,6 @@
+import os
+
+if __name__ == '__main__':
+ buf = os.read(0, 1024)
+ os.write(1, b'OUT:'+buf)
+ os.write(2, b'ERR:'+buf)
diff --git a/Lib/test/test_asyncio/echo3.py b/Lib/test/test_asyncio/echo3.py
new file mode 100644
index 0000000..0644967
--- /dev/null
+++ b/Lib/test/test_asyncio/echo3.py
@@ -0,0 +1,11 @@
+import os
+
+if __name__ == '__main__':
+ while True:
+ buf = os.read(0, 1024)
+ if not buf:
+ break
+ try:
+ os.write(1, b'OUT:'+buf)
+ except OSError as ex:
+ os.write(2, b'ERR:' + ex.__class__.__name__.encode('ascii'))
diff --git a/Lib/test/test_asyncio/keycert3.pem b/Lib/test/test_asyncio/keycert3.pem
new file mode 100644
index 0000000..5bfa62c
--- /dev/null
+++ b/Lib/test/test_asyncio/keycert3.pem
@@ -0,0 +1,73 @@
+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP
+jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM
+9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ
+aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe
+yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j
+y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+
+AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW
+5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL
+9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9
+1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT
+DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh
+1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m
+JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3
+RnJdHOMXWem7/w==
+-----END PRIVATE KEY-----
+Certificate:
+ Data:
+ Version: 1 (0x0)
+ Serial Number: 12723342612721443281 (0xb09264b1f2da21d1)
+ Signature Algorithm: sha1WithRSAEncryption
+ Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Validity
+ Not Before: Jan 4 19:47:07 2013 GMT
+ Not After : Nov 13 19:47:07 2022 GMT
+ Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost
+ Subject Public Key Info:
+ Public Key Algorithm: rsaEncryption
+ Public-Key: (1024 bit)
+ Modulus:
+ 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d:
+ 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb:
+ c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99:
+ 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c:
+ f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93:
+ 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23:
+ f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5:
+ af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6:
+ 21:82:a5:3c:88:e5:be:1b:b1
+ Exponent: 65537 (0x10001)
+ Signature Algorithm: sha1WithRSAEncryption
+ 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a:
+ e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93:
+ f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13:
+ e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92:
+ d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59:
+ 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8:
+ ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1:
+ 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75:
+ 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96:
+ 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48:
+ 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a:
+ f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6:
+ 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41:
+ a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb:
+ fc:a9:94:71
+-----BEGIN CERTIFICATE-----
+MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY
+WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV
+BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3
+WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV
+BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv
+c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C
+tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola
+N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1
+TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR
+iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG
+xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo
+5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv
+mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF
+YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh
+2EJ36/yplHE=
+-----END CERTIFICATE-----
diff --git a/Lib/test/test_asyncio/pycacert.pem b/Lib/test/test_asyncio/pycacert.pem
new file mode 100644
index 0000000..09b1f3e
--- /dev/null
+++ b/Lib/test/test_asyncio/pycacert.pem
@@ -0,0 +1,78 @@
+Certificate:
+ Data:
+ Version: 3 (0x2)
+ Serial Number: 12723342612721443280 (0xb09264b1f2da21d0)
+ Signature Algorithm: sha1WithRSAEncryption
+ Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Validity
+ Not Before: Jan 4 19:47:07 2013 GMT
+ Not After : Jan 2 19:47:07 2023 GMT
+ Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server
+ Subject Public Key Info:
+ Public Key Algorithm: rsaEncryption
+ Public-Key: (2048 bit)
+ Modulus:
+ 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2:
+ 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4:
+ e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f:
+ e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f:
+ 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf:
+ 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d:
+ a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3:
+ e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4:
+ 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf:
+ 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c:
+ e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6:
+ c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a:
+ cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01:
+ 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87:
+ 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f:
+ 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14:
+ e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4:
+ c5:4d
+ Exponent: 65537 (0x10001)
+ X509v3 extensions:
+ X509v3 Subject Key Identifier:
+ BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B
+ X509v3 Authority Key Identifier:
+ keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B
+
+ X509v3 Basic Constraints:
+ CA:TRUE
+ Signature Algorithm: sha1WithRSAEncryption
+ 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6:
+ 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d:
+ a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95:
+ 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17:
+ 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c:
+ 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4:
+ fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7:
+ 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24:
+ 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33:
+ 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61:
+ ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f:
+ 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64:
+ b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb:
+ 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3:
+ 5e:58:c8:9e
+-----BEGIN CERTIFICATE-----
+MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV
+BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW
+MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx
+OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg
+Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI
+hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV
+q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/
+AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA
+Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni
+0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx
+6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w
+HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2
+2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB
+AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4
+QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1
+Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O
+JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR
+f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf
+9mmvtk57HVjsO6lTo15YyJ4=
+-----END CERTIFICATE-----
diff --git a/Lib/test/test_asyncio/ssl_cert.pem b/Lib/test/test_asyncio/ssl_cert.pem
new file mode 100644
index 0000000..47a7d7e
--- /dev/null
+++ b/Lib/test/test_asyncio/ssl_cert.pem
@@ -0,0 +1,15 @@
+-----BEGIN CERTIFICATE-----
+MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw
+MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH
+Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k
+YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
+gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7
+6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt
+pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw
+FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd
+BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G
+lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1
+CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX
+-----END CERTIFICATE-----
diff --git a/Lib/test/test_asyncio/ssl_key.pem b/Lib/test/test_asyncio/ssl_key.pem
new file mode 100644
index 0000000..3fd3bbd
--- /dev/null
+++ b/Lib/test/test_asyncio/ssl_key.pem
@@ -0,0 +1,16 @@
+-----BEGIN PRIVATE KEY-----
+MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm
+LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0
+ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP
+USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt
+CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq
+SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK
+UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y
+BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ
+ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5
+oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik
+eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F
+0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS
+x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/
+SPIXQuT8RMPDVNQ=
+-----END PRIVATE KEY-----
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
new file mode 100644
index 0000000..544bd3d
--- /dev/null
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -0,0 +1,949 @@
+"""Tests for base_events.py"""
+
+import errno
+import logging
+import socket
+import sys
+import time
+import unittest
+from unittest import mock
+from test.support import IPV6_ENABLED
+
+import asyncio
+from asyncio import base_events
+from asyncio import constants
+from asyncio import test_utils
+
+
+MOCK_ANY = mock.ANY
+PY34 = sys.version_info >= (3, 4)
+
+
+class BaseEventLoopTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = base_events.BaseEventLoop()
+ self.loop._selector = mock.Mock()
+ asyncio.set_event_loop(None)
+
+ def test_not_implemented(self):
+ m = mock.Mock()
+ self.assertRaises(
+ NotImplementedError,
+ self.loop._make_socket_transport, m, m)
+ self.assertRaises(
+ NotImplementedError,
+ self.loop._make_ssl_transport, m, m, m, m)
+ self.assertRaises(
+ NotImplementedError,
+ self.loop._make_datagram_transport, m, m)
+ self.assertRaises(
+ NotImplementedError, self.loop._process_events, [])
+ self.assertRaises(
+ NotImplementedError, self.loop._write_to_self)
+ self.assertRaises(
+ NotImplementedError, self.loop._read_from_self)
+ self.assertRaises(
+ NotImplementedError,
+ self.loop._make_read_pipe_transport, m, m)
+ self.assertRaises(
+ NotImplementedError,
+ self.loop._make_write_pipe_transport, m, m)
+ gen = self.loop._make_subprocess_transport(m, m, m, m, m, m, m)
+ self.assertRaises(NotImplementedError, next, iter(gen))
+
+ def test__add_callback_handle(self):
+ h = asyncio.Handle(lambda: False, (), self.loop)
+
+ self.loop._add_callback(h)
+ self.assertFalse(self.loop._scheduled)
+ self.assertIn(h, self.loop._ready)
+
+ def test__add_callback_timer(self):
+ h = asyncio.TimerHandle(time.monotonic()+10, lambda: False, (),
+ self.loop)
+
+ self.loop._add_callback(h)
+ self.assertIn(h, self.loop._scheduled)
+
+ def test__add_callback_cancelled_handle(self):
+ h = asyncio.Handle(lambda: False, (), self.loop)
+ h.cancel()
+
+ self.loop._add_callback(h)
+ self.assertFalse(self.loop._scheduled)
+ self.assertFalse(self.loop._ready)
+
+ def test_set_default_executor(self):
+ executor = mock.Mock()
+ self.loop.set_default_executor(executor)
+ self.assertIs(executor, self.loop._default_executor)
+
+ def test_getnameinfo(self):
+ sockaddr = mock.Mock()
+ self.loop.run_in_executor = mock.Mock()
+ self.loop.getnameinfo(sockaddr)
+ self.assertEqual(
+ (None, socket.getnameinfo, sockaddr, 0),
+ self.loop.run_in_executor.call_args[0])
+
+ def test_call_soon(self):
+ def cb():
+ pass
+
+ h = self.loop.call_soon(cb)
+ self.assertEqual(h._callback, cb)
+ self.assertIsInstance(h, asyncio.Handle)
+ self.assertIn(h, self.loop._ready)
+
+ def test_call_later(self):
+ def cb():
+ pass
+
+ h = self.loop.call_later(10.0, cb)
+ self.assertIsInstance(h, asyncio.TimerHandle)
+ self.assertIn(h, self.loop._scheduled)
+ self.assertNotIn(h, self.loop._ready)
+
+ def test_call_later_negative_delays(self):
+ calls = []
+
+ def cb(arg):
+ calls.append(arg)
+
+ self.loop._process_events = mock.Mock()
+ self.loop.call_later(-1, cb, 'a')
+ self.loop.call_later(-2, cb, 'b')
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(calls, ['b', 'a'])
+
+ def test_time_and_call_at(self):
+ def cb():
+ self.loop.stop()
+
+ self.loop._process_events = mock.Mock()
+ delay = 0.1
+
+ when = self.loop.time() + delay
+ self.loop.call_at(when, cb)
+ t0 = self.loop.time()
+ self.loop.run_forever()
+ dt = self.loop.time() - t0
+
+ # 50 ms: maximum granularity of the event loop
+ self.assertGreaterEqual(dt, delay - 0.050, dt)
+ # tolerate a difference of +800 ms because some Python buildbots
+ # are really slow
+ self.assertLessEqual(dt, 0.9, dt)
+
+ def test_assert_is_current_event_loop(self):
+ def cb():
+ pass
+
+ other_loop = base_events.BaseEventLoop()
+ other_loop._selector = unittest.mock.Mock()
+ asyncio.set_event_loop(other_loop)
+
+ # raise RuntimeError if the event loop is different in debug mode
+ self.loop.set_debug(True)
+ with self.assertRaises(RuntimeError):
+ self.loop.call_soon(cb)
+ with self.assertRaises(RuntimeError):
+ self.loop.call_later(60, cb)
+ with self.assertRaises(RuntimeError):
+ self.loop.call_at(self.loop.time() + 60, cb)
+
+ # check disabled if debug mode is disabled
+ self.loop.set_debug(False)
+ self.loop.call_soon(cb)
+ self.loop.call_later(60, cb)
+ self.loop.call_at(self.loop.time() + 60, cb)
+
+ def test_run_once_in_executor_handle(self):
+ def cb():
+ pass
+
+ self.assertRaises(
+ AssertionError, self.loop.run_in_executor,
+ None, asyncio.Handle(cb, (), self.loop), ('',))
+ self.assertRaises(
+ AssertionError, self.loop.run_in_executor,
+ None, asyncio.TimerHandle(10, cb, (), self.loop))
+
+ def test_run_once_in_executor_cancelled(self):
+ def cb():
+ pass
+ h = asyncio.Handle(cb, (), self.loop)
+ h.cancel()
+
+ f = self.loop.run_in_executor(None, h)
+ self.assertIsInstance(f, asyncio.Future)
+ self.assertTrue(f.done())
+ self.assertIsNone(f.result())
+
+ def test_run_once_in_executor_plain(self):
+ def cb():
+ pass
+ h = asyncio.Handle(cb, (), self.loop)
+ f = asyncio.Future(loop=self.loop)
+ executor = mock.Mock()
+ executor.submit.return_value = f
+
+ self.loop.set_default_executor(executor)
+
+ res = self.loop.run_in_executor(None, h)
+ self.assertIs(f, res)
+
+ executor = mock.Mock()
+ executor.submit.return_value = f
+ res = self.loop.run_in_executor(executor, h)
+ self.assertIs(f, res)
+ self.assertTrue(executor.submit.called)
+
+ f.cancel() # Don't complain about abandoned Future.
+
+ def test__run_once(self):
+ h1 = asyncio.TimerHandle(time.monotonic() + 5.0, lambda: True, (),
+ self.loop)
+ h2 = asyncio.TimerHandle(time.monotonic() + 10.0, lambda: True, (),
+ self.loop)
+
+ h1.cancel()
+
+ self.loop._process_events = mock.Mock()
+ self.loop._scheduled.append(h1)
+ self.loop._scheduled.append(h2)
+ self.loop._run_once()
+
+ t = self.loop._selector.select.call_args[0][0]
+ self.assertTrue(9.5 < t < 10.5, t)
+ self.assertEqual([h2], self.loop._scheduled)
+ self.assertTrue(self.loop._process_events.called)
+
+ def test_set_debug(self):
+ self.loop.set_debug(True)
+ self.assertTrue(self.loop.get_debug())
+ self.loop.set_debug(False)
+ self.assertFalse(self.loop.get_debug())
+
+ @mock.patch('asyncio.base_events.time')
+ @mock.patch('asyncio.base_events.logger')
+ def test__run_once_logging(self, m_logger, m_time):
+ # Log to INFO level if timeout > 1.0 sec.
+ idx = -1
+ data = [10.0, 10.0, 12.0, 13.0]
+
+ def monotonic():
+ nonlocal data, idx
+ idx += 1
+ return data[idx]
+
+ m_time.monotonic = monotonic
+
+ self.loop._scheduled.append(
+ asyncio.TimerHandle(11.0, lambda: True, (), self.loop))
+ self.loop._process_events = mock.Mock()
+ self.loop._run_once()
+ self.assertEqual(logging.INFO, m_logger.log.call_args[0][0])
+
+ idx = -1
+ data = [10.0, 10.0, 10.3, 13.0]
+ self.loop._scheduled = [asyncio.TimerHandle(11.0, lambda: True, (),
+ self.loop)]
+ self.loop._run_once()
+ self.assertEqual(logging.DEBUG, m_logger.log.call_args[0][0])
+
+ def test__run_once_schedule_handle(self):
+ handle = None
+ processed = False
+
+ def cb(loop):
+ nonlocal processed, handle
+ processed = True
+ handle = loop.call_soon(lambda: True)
+
+ h = asyncio.TimerHandle(time.monotonic() - 1, cb, (self.loop,),
+ self.loop)
+
+ self.loop._process_events = mock.Mock()
+ self.loop._scheduled.append(h)
+ self.loop._run_once()
+
+ self.assertTrue(processed)
+ self.assertEqual([handle], list(self.loop._ready))
+
+ def test_run_until_complete_type_error(self):
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, 'blah')
+
+ def test_subprocess_exec_invalid_args(self):
+ args = [sys.executable, '-c', 'pass']
+
+ # missing program parameter (empty args)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol)
+
+ # exepected multiple arguments, not a list
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol, args)
+
+ # program arguments must be strings, not int
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol, sys.executable, 123)
+
+ # universal_newlines, shell, bufsize must not be set
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol, *args, universal_newlines=True)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol, *args, shell=True)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_exec,
+ asyncio.SubprocessProtocol, *args, bufsize=4096)
+
+ def test_subprocess_shell_invalid_args(self):
+ # expected a string, not an int or a list
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_shell,
+ asyncio.SubprocessProtocol, 123)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_shell,
+ asyncio.SubprocessProtocol, [sys.executable, '-c', 'pass'])
+
+ # universal_newlines, shell, bufsize must not be set
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_shell,
+ asyncio.SubprocessProtocol, 'exit 0', universal_newlines=True)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_shell,
+ asyncio.SubprocessProtocol, 'exit 0', shell=True)
+ self.assertRaises(TypeError,
+ self.loop.run_until_complete, self.loop.subprocess_shell,
+ asyncio.SubprocessProtocol, 'exit 0', bufsize=4096)
+
+ def test_default_exc_handler_callback(self):
+ self.loop._process_events = mock.Mock()
+
+ def zero_error(fut):
+ fut.set_result(True)
+ 1/0
+
+ # Test call_soon (events.Handle)
+ with mock.patch('asyncio.base_events.logger') as log:
+ fut = asyncio.Future(loop=self.loop)
+ self.loop.call_soon(zero_error, fut)
+ fut.add_done_callback(lambda fut: self.loop.stop())
+ self.loop.run_forever()
+ log.error.assert_called_with(
+ test_utils.MockPattern('Exception in callback.*zero'),
+ exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY))
+
+ # Test call_later (events.TimerHandle)
+ with mock.patch('asyncio.base_events.logger') as log:
+ fut = asyncio.Future(loop=self.loop)
+ self.loop.call_later(0.01, zero_error, fut)
+ fut.add_done_callback(lambda fut: self.loop.stop())
+ self.loop.run_forever()
+ log.error.assert_called_with(
+ test_utils.MockPattern('Exception in callback.*zero'),
+ exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY))
+
+ def test_default_exc_handler_coro(self):
+ self.loop._process_events = mock.Mock()
+
+ @asyncio.coroutine
+ def zero_error_coro():
+ yield from asyncio.sleep(0.01, loop=self.loop)
+ 1/0
+
+ # Test Future.__del__
+ with mock.patch('asyncio.base_events.logger') as log:
+ fut = asyncio.async(zero_error_coro(), loop=self.loop)
+ fut.add_done_callback(lambda *args: self.loop.stop())
+ self.loop.run_forever()
+ fut = None # Trigger Future.__del__ or futures._TracebackLogger
+ if PY34:
+ # Future.__del__ in Python 3.4 logs error with
+ # an actual exception context
+ log.error.assert_called_with(
+ test_utils.MockPattern('.*exception was never retrieved'),
+ exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY))
+ else:
+ # futures._TracebackLogger logs only textual traceback
+ log.error.assert_called_with(
+ test_utils.MockPattern(
+ '.*exception was never retrieved.*ZeroDiv'),
+ exc_info=False)
+
+ def test_set_exc_handler_invalid(self):
+ with self.assertRaisesRegex(TypeError, 'A callable object or None'):
+ self.loop.set_exception_handler('spam')
+
+ def test_set_exc_handler_custom(self):
+ def zero_error():
+ 1/0
+
+ def run_loop():
+ self.loop.call_soon(zero_error)
+ self.loop._run_once()
+
+ self.loop._process_events = mock.Mock()
+
+ mock_handler = mock.Mock()
+ self.loop.set_exception_handler(mock_handler)
+ run_loop()
+ mock_handler.assert_called_with(self.loop, {
+ 'exception': MOCK_ANY,
+ 'message': test_utils.MockPattern(
+ 'Exception in callback.*zero_error'),
+ 'handle': MOCK_ANY,
+ })
+ mock_handler.reset_mock()
+
+ self.loop.set_exception_handler(None)
+ with mock.patch('asyncio.base_events.logger') as log:
+ run_loop()
+ log.error.assert_called_with(
+ test_utils.MockPattern(
+ 'Exception in callback.*zero'),
+ exc_info=(ZeroDivisionError, MOCK_ANY, MOCK_ANY))
+
+ assert not mock_handler.called
+
+ def test_set_exc_handler_broken(self):
+ def run_loop():
+ def zero_error():
+ 1/0
+ self.loop.call_soon(zero_error)
+ self.loop._run_once()
+
+ def handler(loop, context):
+ raise AttributeError('spam')
+
+ self.loop._process_events = mock.Mock()
+
+ self.loop.set_exception_handler(handler)
+
+ with mock.patch('asyncio.base_events.logger') as log:
+ run_loop()
+ log.error.assert_called_with(
+ test_utils.MockPattern(
+ 'Unhandled error in exception handler'),
+ exc_info=(AttributeError, MOCK_ANY, MOCK_ANY))
+
+ def test_default_exc_handler_broken(self):
+ _context = None
+
+ class Loop(base_events.BaseEventLoop):
+
+ _selector = mock.Mock()
+ _process_events = mock.Mock()
+
+ def default_exception_handler(self, context):
+ nonlocal _context
+ _context = context
+ # Simulates custom buggy "default_exception_handler"
+ raise ValueError('spam')
+
+ loop = Loop()
+ asyncio.set_event_loop(loop)
+
+ def run_loop():
+ def zero_error():
+ 1/0
+ loop.call_soon(zero_error)
+ loop._run_once()
+
+ with mock.patch('asyncio.base_events.logger') as log:
+ run_loop()
+ log.error.assert_called_with(
+ 'Exception in default exception handler',
+ exc_info=True)
+
+ def custom_handler(loop, context):
+ raise ValueError('ham')
+
+ _context = None
+ loop.set_exception_handler(custom_handler)
+ with mock.patch('asyncio.base_events.logger') as log:
+ run_loop()
+ log.error.assert_called_with(
+ test_utils.MockPattern('Exception in default exception.*'
+ 'while handling.*in custom'),
+ exc_info=True)
+
+ # Check that original context was passed to default
+ # exception handler.
+ self.assertIn('context', _context)
+ self.assertIs(type(_context['context']['exception']),
+ ZeroDivisionError)
+
+
+class MyProto(asyncio.Protocol):
+ done = None
+
+ def __init__(self, create_future=False):
+ self.state = 'INITIAL'
+ self.nbytes = 0
+ if create_future:
+ self.done = asyncio.Future()
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'CONNECTED'
+ transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n')
+
+ def data_received(self, data):
+ assert self.state == 'CONNECTED', self.state
+ self.nbytes += len(data)
+
+ def eof_received(self):
+ assert self.state == 'CONNECTED', self.state
+ self.state = 'EOF'
+
+ def connection_lost(self, exc):
+ assert self.state in ('CONNECTED', 'EOF'), self.state
+ self.state = 'CLOSED'
+ if self.done:
+ self.done.set_result(None)
+
+
+class MyDatagramProto(asyncio.DatagramProtocol):
+ done = None
+
+ def __init__(self, create_future=False):
+ self.state = 'INITIAL'
+ self.nbytes = 0
+ if create_future:
+ self.done = asyncio.Future()
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'INITIALIZED'
+
+ def datagram_received(self, data, addr):
+ assert self.state == 'INITIALIZED', self.state
+ self.nbytes += len(data)
+
+ def error_received(self, exc):
+ assert self.state == 'INITIALIZED', self.state
+
+ def connection_lost(self, exc):
+ assert self.state == 'INITIALIZED', self.state
+ self.state = 'CLOSED'
+ if self.done:
+ self.done.set_result(None)
+
+
+class BaseEventLoopWithSelectorTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_connection_multiple_errors(self, m_socket):
+
+ class MyProto(asyncio.Protocol):
+ pass
+
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ yield from []
+ return [(2, 1, 6, '', ('107.6.106.82', 80)),
+ (2, 1, 6, '', ('107.6.106.82', 80))]
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ idx = -1
+ errors = ['err1', 'err2']
+
+ def _socket(*args, **kw):
+ nonlocal idx, errors
+ idx += 1
+ raise OSError(errors[idx])
+
+ m_socket.socket = _socket
+
+ self.loop.getaddrinfo = getaddrinfo_task
+
+ coro = self.loop.create_connection(MyProto, 'example.com', 80)
+ with self.assertRaises(OSError) as cm:
+ self.loop.run_until_complete(coro)
+
+ self.assertEqual(str(cm.exception), 'Multiple exceptions: err1, err2')
+
+ def test_create_connection_host_port_sock(self):
+ coro = self.loop.create_connection(
+ MyProto, 'example.com', 80, sock=object())
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_no_host_port_sock(self):
+ coro = self.loop.create_connection(MyProto)
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_no_getaddrinfo(self):
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ yield from []
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ self.loop.getaddrinfo = getaddrinfo_task
+ coro = self.loop.create_connection(MyProto, 'example.com', 80)
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_connect_err(self):
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ yield from []
+ return [(2, 1, 6, '', ('107.6.106.82', 80))]
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ self.loop.getaddrinfo = getaddrinfo_task
+ self.loop.sock_connect = mock.Mock()
+ self.loop.sock_connect.side_effect = OSError
+
+ coro = self.loop.create_connection(MyProto, 'example.com', 80)
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_multiple(self):
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ return [(2, 1, 6, '', ('0.0.0.1', 80)),
+ (2, 1, 6, '', ('0.0.0.2', 80))]
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ self.loop.getaddrinfo = getaddrinfo_task
+ self.loop.sock_connect = mock.Mock()
+ self.loop.sock_connect.side_effect = OSError
+
+ coro = self.loop.create_connection(
+ MyProto, 'example.com', 80, family=socket.AF_INET)
+ with self.assertRaises(OSError):
+ self.loop.run_until_complete(coro)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_connection_multiple_errors_local_addr(self, m_socket):
+
+ def bind(addr):
+ if addr[0] == '0.0.0.1':
+ err = OSError('Err')
+ err.strerror = 'Err'
+ raise err
+
+ m_socket.socket.return_value.bind = bind
+
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ return [(2, 1, 6, '', ('0.0.0.1', 80)),
+ (2, 1, 6, '', ('0.0.0.2', 80))]
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ self.loop.getaddrinfo = getaddrinfo_task
+ self.loop.sock_connect = mock.Mock()
+ self.loop.sock_connect.side_effect = OSError('Err2')
+
+ coro = self.loop.create_connection(
+ MyProto, 'example.com', 80, family=socket.AF_INET,
+ local_addr=(None, 8080))
+ with self.assertRaises(OSError) as cm:
+ self.loop.run_until_complete(coro)
+
+ self.assertTrue(str(cm.exception).startswith('Multiple exceptions: '))
+ self.assertTrue(m_socket.socket.return_value.close.called)
+
+ def test_create_connection_no_local_addr(self):
+ @asyncio.coroutine
+ def getaddrinfo(host, *args, **kw):
+ if host == 'example.com':
+ return [(2, 1, 6, '', ('107.6.106.82', 80)),
+ (2, 1, 6, '', ('107.6.106.82', 80))]
+ else:
+ return []
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+ self.loop.getaddrinfo = getaddrinfo_task
+
+ coro = self.loop.create_connection(
+ MyProto, 'example.com', 80, family=socket.AF_INET,
+ local_addr=(None, 8080))
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_ssl_server_hostname_default(self):
+ self.loop.getaddrinfo = mock.Mock()
+
+ def mock_getaddrinfo(*args, **kwds):
+ f = asyncio.Future(loop=self.loop)
+ f.set_result([(socket.AF_INET, socket.SOCK_STREAM,
+ socket.SOL_TCP, '', ('1.2.3.4', 80))])
+ return f
+
+ self.loop.getaddrinfo.side_effect = mock_getaddrinfo
+ self.loop.sock_connect = mock.Mock()
+ self.loop.sock_connect.return_value = ()
+ self.loop._make_ssl_transport = mock.Mock()
+
+ class _SelectorTransportMock:
+ _sock = None
+
+ def close(self):
+ self._sock.close()
+
+ def mock_make_ssl_transport(sock, protocol, sslcontext, waiter,
+ **kwds):
+ waiter.set_result(None)
+ transport = _SelectorTransportMock()
+ transport._sock = sock
+ return transport
+
+ self.loop._make_ssl_transport.side_effect = mock_make_ssl_transport
+ ANY = mock.ANY
+ # First try the default server_hostname.
+ self.loop._make_ssl_transport.reset_mock()
+ coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True)
+ transport, _ = self.loop.run_until_complete(coro)
+ transport.close()
+ self.loop._make_ssl_transport.assert_called_with(
+ ANY, ANY, ANY, ANY,
+ server_side=False,
+ server_hostname='python.org')
+ # Next try an explicit server_hostname.
+ self.loop._make_ssl_transport.reset_mock()
+ coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True,
+ server_hostname='perl.com')
+ transport, _ = self.loop.run_until_complete(coro)
+ transport.close()
+ self.loop._make_ssl_transport.assert_called_with(
+ ANY, ANY, ANY, ANY,
+ server_side=False,
+ server_hostname='perl.com')
+ # Finally try an explicit empty server_hostname.
+ self.loop._make_ssl_transport.reset_mock()
+ coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True,
+ server_hostname='')
+ transport, _ = self.loop.run_until_complete(coro)
+ transport.close()
+ self.loop._make_ssl_transport.assert_called_with(ANY, ANY, ANY, ANY,
+ server_side=False,
+ server_hostname='')
+
+ def test_create_connection_no_ssl_server_hostname_errors(self):
+ # When not using ssl, server_hostname must be None.
+ coro = self.loop.create_connection(MyProto, 'python.org', 80,
+ server_hostname='')
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+ coro = self.loop.create_connection(MyProto, 'python.org', 80,
+ server_hostname='python.org')
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
+ def test_create_connection_ssl_server_hostname_errors(self):
+ # When using ssl, server_hostname may be None if host is non-empty.
+ coro = self.loop.create_connection(MyProto, '', 80, ssl=True)
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+ coro = self.loop.create_connection(MyProto, None, 80, ssl=True)
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+ sock = socket.socket()
+ coro = self.loop.create_connection(MyProto, None, None,
+ ssl=True, sock=sock)
+ self.addCleanup(sock.close)
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
+ def test_create_server_empty_host(self):
+ # if host is empty string use None instead
+ host = object()
+
+ @asyncio.coroutine
+ def getaddrinfo(*args, **kw):
+ nonlocal host
+ host = args[0]
+ yield from []
+
+ def getaddrinfo_task(*args, **kwds):
+ return asyncio.Task(getaddrinfo(*args, **kwds), loop=self.loop)
+
+ self.loop.getaddrinfo = getaddrinfo_task
+ fut = self.loop.create_server(MyProto, '', 0)
+ self.assertRaises(OSError, self.loop.run_until_complete, fut)
+ self.assertIsNone(host)
+
+ def test_create_server_host_port_sock(self):
+ fut = self.loop.create_server(
+ MyProto, '0.0.0.0', 0, sock=object())
+ self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+ def test_create_server_no_host_port_sock(self):
+ fut = self.loop.create_server(MyProto)
+ self.assertRaises(ValueError, self.loop.run_until_complete, fut)
+
+ def test_create_server_no_getaddrinfo(self):
+ getaddrinfo = self.loop.getaddrinfo = mock.Mock()
+ getaddrinfo.return_value = []
+
+ f = self.loop.create_server(MyProto, '0.0.0.0', 0)
+ self.assertRaises(OSError, self.loop.run_until_complete, f)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_server_cant_bind(self, m_socket):
+
+ class Err(OSError):
+ strerror = 'error'
+
+ m_socket.getaddrinfo.return_value = [
+ (2, 1, 6, '', ('127.0.0.1', 10100))]
+ m_socket.getaddrinfo._is_coroutine = False
+ m_sock = m_socket.socket.return_value = mock.Mock()
+ m_sock.bind.side_effect = Err
+
+ fut = self.loop.create_server(MyProto, '0.0.0.0', 0)
+ self.assertRaises(OSError, self.loop.run_until_complete, fut)
+ self.assertTrue(m_sock.close.called)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_datagram_endpoint_no_addrinfo(self, m_socket):
+ m_socket.getaddrinfo.return_value = []
+ m_socket.getaddrinfo._is_coroutine = False
+
+ coro = self.loop.create_datagram_endpoint(
+ MyDatagramProto, local_addr=('localhost', 0))
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ def test_create_datagram_endpoint_addr_error(self):
+ coro = self.loop.create_datagram_endpoint(
+ MyDatagramProto, local_addr='localhost')
+ self.assertRaises(
+ AssertionError, self.loop.run_until_complete, coro)
+ coro = self.loop.create_datagram_endpoint(
+ MyDatagramProto, local_addr=('localhost', 1, 2, 3))
+ self.assertRaises(
+ AssertionError, self.loop.run_until_complete, coro)
+
+ def test_create_datagram_endpoint_connect_err(self):
+ self.loop.sock_connect = mock.Mock()
+ self.loop.sock_connect.side_effect = OSError
+
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol, remote_addr=('127.0.0.1', 0))
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_datagram_endpoint_socket_err(self, m_socket):
+ m_socket.getaddrinfo = socket.getaddrinfo
+ m_socket.socket.side_effect = OSError
+
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol, family=socket.AF_INET)
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol, local_addr=('127.0.0.1', 0))
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+
+ @unittest.skipUnless(IPV6_ENABLED, 'IPv6 not supported or enabled')
+ def test_create_datagram_endpoint_no_matching_family(self):
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol,
+ remote_addr=('127.0.0.1', 0), local_addr=('::1', 0))
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, coro)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_datagram_endpoint_setblk_err(self, m_socket):
+ m_socket.socket.return_value.setblocking.side_effect = OSError
+
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol, family=socket.AF_INET)
+ self.assertRaises(
+ OSError, self.loop.run_until_complete, coro)
+ self.assertTrue(
+ m_socket.socket.return_value.close.called)
+
+ def test_create_datagram_endpoint_noaddr_nofamily(self):
+ coro = self.loop.create_datagram_endpoint(
+ asyncio.DatagramProtocol)
+ self.assertRaises(ValueError, self.loop.run_until_complete, coro)
+
+ @mock.patch('asyncio.base_events.socket')
+ def test_create_datagram_endpoint_cant_bind(self, m_socket):
+ class Err(OSError):
+ pass
+
+ m_socket.AF_INET6 = socket.AF_INET6
+ m_socket.getaddrinfo = socket.getaddrinfo
+ m_sock = m_socket.socket.return_value = mock.Mock()
+ m_sock.bind.side_effect = Err
+
+ fut = self.loop.create_datagram_endpoint(
+ MyDatagramProto,
+ local_addr=('127.0.0.1', 0), family=socket.AF_INET)
+ self.assertRaises(Err, self.loop.run_until_complete, fut)
+ self.assertTrue(m_sock.close.called)
+
+ def test_accept_connection_retry(self):
+ sock = mock.Mock()
+ sock.accept.side_effect = BlockingIOError()
+
+ self.loop._accept_connection(MyProto, sock)
+ self.assertFalse(sock.close.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_accept_connection_exception(self, m_log):
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.accept.side_effect = OSError(errno.EMFILE, 'Too many open files')
+ self.loop.remove_reader = mock.Mock()
+ self.loop.call_later = mock.Mock()
+
+ self.loop._accept_connection(MyProto, sock)
+ self.assertTrue(m_log.error.called)
+ self.assertFalse(sock.close.called)
+ self.loop.remove_reader.assert_called_with(10)
+ self.loop.call_later.assert_called_with(constants.ACCEPT_RETRY_DELAY,
+ # self.loop._start_serving
+ mock.ANY,
+ MyProto, sock, None, None)
+
+ def test_call_coroutine(self):
+ @asyncio.coroutine
+ def coroutine_function():
+ pass
+
+ with self.assertRaises(TypeError):
+ self.loop.call_soon(coroutine_function)
+ with self.assertRaises(TypeError):
+ self.loop.call_soon_threadsafe(coroutine_function)
+ with self.assertRaises(TypeError):
+ self.loop.call_later(60, coroutine_function)
+ with self.assertRaises(TypeError):
+ self.loop.call_at(self.loop.time() + 60, coroutine_function)
+ with self.assertRaises(TypeError):
+ self.loop.run_in_executor(None, coroutine_function)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
new file mode 100644
index 0000000..bafa875
--- /dev/null
+++ b/Lib/test/test_asyncio/test_events.py
@@ -0,0 +1,2044 @@
+"""Tests for events.py."""
+
+import functools
+import gc
+import io
+import os
+import platform
+import signal
+import socket
+try:
+ import ssl
+except ImportError:
+ ssl = None
+ HAS_SNI = False
+else:
+ from ssl import HAS_SNI
+import subprocess
+import sys
+import threading
+import time
+import errno
+import unittest
+from unittest import mock
+from test import support # find_unused_port, IPV6_ENABLED, TEST_HOME_DIR
+
+
+import asyncio
+from asyncio import selector_events
+from asyncio import test_utils
+
+
+def data_file(filename):
+ if hasattr(support, 'TEST_HOME_DIR'):
+ fullname = os.path.join(support.TEST_HOME_DIR, filename)
+ if os.path.isfile(fullname):
+ return fullname
+ fullname = os.path.join(os.path.dirname(__file__), filename)
+ if os.path.isfile(fullname):
+ return fullname
+ raise FileNotFoundError(filename)
+
+
+def osx_tiger():
+ """Return True if the platform is Mac OS 10.4 or older."""
+ if sys.platform != 'darwin':
+ return False
+ version = platform.mac_ver()[0]
+ version = tuple(map(int, version.split('.')))
+ return version < (10, 5)
+
+
+ONLYCERT = data_file('ssl_cert.pem')
+ONLYKEY = data_file('ssl_key.pem')
+SIGNED_CERTFILE = data_file('keycert3.pem')
+SIGNING_CA = data_file('pycacert.pem')
+
+
+class MyBaseProto(asyncio.Protocol):
+ connected = None
+ done = None
+
+ def __init__(self, loop=None):
+ self.transport = None
+ self.state = 'INITIAL'
+ self.nbytes = 0
+ if loop is not None:
+ self.connected = asyncio.Future(loop=loop)
+ self.done = asyncio.Future(loop=loop)
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'CONNECTED'
+ if self.connected:
+ self.connected.set_result(None)
+
+ def data_received(self, data):
+ assert self.state == 'CONNECTED', self.state
+ self.nbytes += len(data)
+
+ def eof_received(self):
+ assert self.state == 'CONNECTED', self.state
+ self.state = 'EOF'
+
+ def connection_lost(self, exc):
+ assert self.state in ('CONNECTED', 'EOF'), self.state
+ self.state = 'CLOSED'
+ if self.done:
+ self.done.set_result(None)
+
+
+class MyProto(MyBaseProto):
+ def connection_made(self, transport):
+ super().connection_made(transport)
+ transport.write(b'GET / HTTP/1.0\r\nHost: example.com\r\n\r\n')
+
+
+class MyDatagramProto(asyncio.DatagramProtocol):
+ done = None
+
+ def __init__(self, loop=None):
+ self.state = 'INITIAL'
+ self.nbytes = 0
+ if loop is not None:
+ self.done = asyncio.Future(loop=loop)
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'INITIALIZED'
+
+ def datagram_received(self, data, addr):
+ assert self.state == 'INITIALIZED', self.state
+ self.nbytes += len(data)
+
+ def error_received(self, exc):
+ assert self.state == 'INITIALIZED', self.state
+
+ def connection_lost(self, exc):
+ assert self.state == 'INITIALIZED', self.state
+ self.state = 'CLOSED'
+ if self.done:
+ self.done.set_result(None)
+
+
+class MyReadPipeProto(asyncio.Protocol):
+ done = None
+
+ def __init__(self, loop=None):
+ self.state = ['INITIAL']
+ self.nbytes = 0
+ self.transport = None
+ if loop is not None:
+ self.done = asyncio.Future(loop=loop)
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == ['INITIAL'], self.state
+ self.state.append('CONNECTED')
+
+ def data_received(self, data):
+ assert self.state == ['INITIAL', 'CONNECTED'], self.state
+ self.nbytes += len(data)
+
+ def eof_received(self):
+ assert self.state == ['INITIAL', 'CONNECTED'], self.state
+ self.state.append('EOF')
+
+ def connection_lost(self, exc):
+ if 'EOF' not in self.state:
+ self.state.append('EOF') # It is okay if EOF is missed.
+ assert self.state == ['INITIAL', 'CONNECTED', 'EOF'], self.state
+ self.state.append('CLOSED')
+ if self.done:
+ self.done.set_result(None)
+
+
+class MyWritePipeProto(asyncio.BaseProtocol):
+ done = None
+
+ def __init__(self, loop=None):
+ self.state = 'INITIAL'
+ self.transport = None
+ if loop is not None:
+ self.done = asyncio.Future(loop=loop)
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'CONNECTED'
+
+ def connection_lost(self, exc):
+ assert self.state == 'CONNECTED', self.state
+ self.state = 'CLOSED'
+ if self.done:
+ self.done.set_result(None)
+
+
+class MySubprocessProtocol(asyncio.SubprocessProtocol):
+
+ def __init__(self, loop):
+ self.state = 'INITIAL'
+ self.transport = None
+ self.connected = asyncio.Future(loop=loop)
+ self.completed = asyncio.Future(loop=loop)
+ self.disconnects = {fd: asyncio.Future(loop=loop) for fd in range(3)}
+ self.data = {1: b'', 2: b''}
+ self.returncode = None
+ self.got_data = {1: asyncio.Event(loop=loop),
+ 2: asyncio.Event(loop=loop)}
+
+ def connection_made(self, transport):
+ self.transport = transport
+ assert self.state == 'INITIAL', self.state
+ self.state = 'CONNECTED'
+ self.connected.set_result(None)
+
+ def connection_lost(self, exc):
+ assert self.state == 'CONNECTED', self.state
+ self.state = 'CLOSED'
+ self.completed.set_result(None)
+
+ def pipe_data_received(self, fd, data):
+ assert self.state == 'CONNECTED', self.state
+ self.data[fd] += data
+ self.got_data[fd].set()
+
+ def pipe_connection_lost(self, fd, exc):
+ assert self.state == 'CONNECTED', self.state
+ if exc:
+ self.disconnects[fd].set_exception(exc)
+ else:
+ self.disconnects[fd].set_result(exc)
+
+ def process_exited(self):
+ assert self.state == 'CONNECTED', self.state
+ self.returncode = self.transport.get_returncode()
+
+
+class EventLoopTestsMixin:
+
+ def setUp(self):
+ super().setUp()
+ self.loop = self.create_event_loop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ # just in case if we have transport close callbacks
+ test_utils.run_briefly(self.loop)
+
+ self.loop.close()
+ gc.collect()
+ super().tearDown()
+
+ def test_run_until_complete_nesting(self):
+ @asyncio.coroutine
+ def coro1():
+ yield
+
+ @asyncio.coroutine
+ def coro2():
+ self.assertTrue(self.loop.is_running())
+ self.loop.run_until_complete(coro1())
+
+ self.assertRaises(
+ RuntimeError, self.loop.run_until_complete, coro2())
+
+ # Note: because of the default Windows timing granularity of
+ # 15.6 msec, we use fairly long sleep times here (~100 msec).
+
+ def test_run_until_complete(self):
+ t0 = self.loop.time()
+ self.loop.run_until_complete(asyncio.sleep(0.1, loop=self.loop))
+ t1 = self.loop.time()
+ self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0)
+
+ def test_run_until_complete_stopped(self):
+ @asyncio.coroutine
+ def cb():
+ self.loop.stop()
+ yield from asyncio.sleep(0.1, loop=self.loop)
+ task = cb()
+ self.assertRaises(RuntimeError,
+ self.loop.run_until_complete, task)
+
+ def test_call_later(self):
+ results = []
+
+ def callback(arg):
+ results.append(arg)
+ self.loop.stop()
+
+ self.loop.call_later(0.1, callback, 'hello world')
+ t0 = time.monotonic()
+ self.loop.run_forever()
+ t1 = time.monotonic()
+ self.assertEqual(results, ['hello world'])
+ self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0)
+
+ def test_call_soon(self):
+ results = []
+
+ def callback(arg1, arg2):
+ results.append((arg1, arg2))
+ self.loop.stop()
+
+ self.loop.call_soon(callback, 'hello', 'world')
+ self.loop.run_forever()
+ self.assertEqual(results, [('hello', 'world')])
+
+ def test_call_soon_threadsafe(self):
+ results = []
+ lock = threading.Lock()
+
+ def callback(arg):
+ results.append(arg)
+ if len(results) >= 2:
+ self.loop.stop()
+
+ def run_in_thread():
+ self.loop.call_soon_threadsafe(callback, 'hello')
+ lock.release()
+
+ lock.acquire()
+ t = threading.Thread(target=run_in_thread)
+ t.start()
+
+ with lock:
+ self.loop.call_soon(callback, 'world')
+ self.loop.run_forever()
+ t.join()
+ self.assertEqual(results, ['hello', 'world'])
+
+ def test_call_soon_threadsafe_same_thread(self):
+ results = []
+
+ def callback(arg):
+ results.append(arg)
+ if len(results) >= 2:
+ self.loop.stop()
+
+ self.loop.call_soon_threadsafe(callback, 'hello')
+ self.loop.call_soon(callback, 'world')
+ self.loop.run_forever()
+ self.assertEqual(results, ['hello', 'world'])
+
+ def test_run_in_executor(self):
+ def run(arg):
+ return (arg, threading.get_ident())
+ f2 = self.loop.run_in_executor(None, run, 'yo')
+ res, thread_id = self.loop.run_until_complete(f2)
+ self.assertEqual(res, 'yo')
+ self.assertNotEqual(thread_id, threading.get_ident())
+
+ def test_reader_callback(self):
+ r, w = test_utils.socketpair()
+ r.setblocking(False)
+ bytes_read = bytearray()
+
+ def reader():
+ try:
+ data = r.recv(1024)
+ except BlockingIOError:
+ # Spurious readiness notifications are possible
+ # at least on Linux -- see man select.
+ return
+ if data:
+ bytes_read.extend(data)
+ else:
+ self.assertTrue(self.loop.remove_reader(r.fileno()))
+ r.close()
+
+ self.loop.add_reader(r.fileno(), reader)
+ self.loop.call_soon(w.send, b'abc')
+ test_utils.run_until(self.loop, lambda: len(bytes_read) >= 3)
+ self.loop.call_soon(w.send, b'def')
+ test_utils.run_until(self.loop, lambda: len(bytes_read) >= 6)
+ self.loop.call_soon(w.close)
+ self.loop.call_soon(self.loop.stop)
+ self.loop.run_forever()
+ self.assertEqual(bytes_read, b'abcdef')
+
+ def test_writer_callback(self):
+ r, w = test_utils.socketpair()
+ w.setblocking(False)
+
+ def writer(data):
+ w.send(data)
+ self.loop.stop()
+
+ data = b'x' * 1024
+ self.loop.add_writer(w.fileno(), writer, data)
+ self.loop.run_forever()
+
+ self.assertTrue(self.loop.remove_writer(w.fileno()))
+ self.assertFalse(self.loop.remove_writer(w.fileno()))
+
+ w.close()
+ read = r.recv(len(data) * 2)
+ r.close()
+ self.assertEqual(read, data)
+
+ def _basetest_sock_client_ops(self, httpd, sock):
+ sock.setblocking(False)
+ self.loop.run_until_complete(
+ self.loop.sock_connect(sock, httpd.address))
+ self.loop.run_until_complete(
+ self.loop.sock_sendall(sock, b'GET / HTTP/1.0\r\n\r\n'))
+ data = self.loop.run_until_complete(
+ self.loop.sock_recv(sock, 1024))
+ # consume data
+ self.loop.run_until_complete(
+ self.loop.sock_recv(sock, 1024))
+ sock.close()
+ self.assertTrue(data.startswith(b'HTTP/1.0 200 OK'))
+
+ def test_sock_client_ops(self):
+ with test_utils.run_test_server() as httpd:
+ sock = socket.socket()
+ self._basetest_sock_client_ops(httpd, sock)
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_unix_sock_client_ops(self):
+ with test_utils.run_test_unix_server() as httpd:
+ sock = socket.socket(socket.AF_UNIX)
+ self._basetest_sock_client_ops(httpd, sock)
+
+ def test_sock_client_fail(self):
+ # Make sure that we will get an unused port
+ address = None
+ try:
+ s = socket.socket()
+ s.bind(('127.0.0.1', 0))
+ address = s.getsockname()
+ finally:
+ s.close()
+
+ sock = socket.socket()
+ sock.setblocking(False)
+ with self.assertRaises(ConnectionRefusedError):
+ self.loop.run_until_complete(
+ self.loop.sock_connect(sock, address))
+ sock.close()
+
+ def test_sock_accept(self):
+ listener = socket.socket()
+ listener.setblocking(False)
+ listener.bind(('127.0.0.1', 0))
+ listener.listen(1)
+ client = socket.socket()
+ client.connect(listener.getsockname())
+
+ f = self.loop.sock_accept(listener)
+ conn, addr = self.loop.run_until_complete(f)
+ self.assertEqual(conn.gettimeout(), 0)
+ self.assertEqual(addr, client.getsockname())
+ self.assertEqual(client.getpeername(), listener.getsockname())
+ client.close()
+ conn.close()
+ listener.close()
+
+ @unittest.skipUnless(hasattr(signal, 'SIGKILL'), 'No SIGKILL')
+ def test_add_signal_handler(self):
+ caught = 0
+
+ def my_handler():
+ nonlocal caught
+ caught += 1
+
+ # Check error behavior first.
+ self.assertRaises(
+ TypeError, self.loop.add_signal_handler, 'boom', my_handler)
+ self.assertRaises(
+ TypeError, self.loop.remove_signal_handler, 'boom')
+ self.assertRaises(
+ ValueError, self.loop.add_signal_handler, signal.NSIG+1,
+ my_handler)
+ self.assertRaises(
+ ValueError, self.loop.remove_signal_handler, signal.NSIG+1)
+ self.assertRaises(
+ ValueError, self.loop.add_signal_handler, 0, my_handler)
+ self.assertRaises(
+ ValueError, self.loop.remove_signal_handler, 0)
+ self.assertRaises(
+ ValueError, self.loop.add_signal_handler, -1, my_handler)
+ self.assertRaises(
+ ValueError, self.loop.remove_signal_handler, -1)
+ self.assertRaises(
+ RuntimeError, self.loop.add_signal_handler, signal.SIGKILL,
+ my_handler)
+ # Removing SIGKILL doesn't raise, since we don't call signal().
+ self.assertFalse(self.loop.remove_signal_handler(signal.SIGKILL))
+ # Now set a handler and handle it.
+ self.loop.add_signal_handler(signal.SIGINT, my_handler)
+
+ os.kill(os.getpid(), signal.SIGINT)
+ test_utils.run_until(self.loop, lambda: caught)
+
+ # Removing it should restore the default handler.
+ self.assertTrue(self.loop.remove_signal_handler(signal.SIGINT))
+ self.assertEqual(signal.getsignal(signal.SIGINT),
+ signal.default_int_handler)
+ # Removing again returns False.
+ self.assertFalse(self.loop.remove_signal_handler(signal.SIGINT))
+
+ @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM')
+ def test_signal_handling_while_selecting(self):
+ # Test with a signal actually arriving during a select() call.
+ caught = 0
+
+ def my_handler():
+ nonlocal caught
+ caught += 1
+ self.loop.stop()
+
+ self.loop.add_signal_handler(signal.SIGALRM, my_handler)
+
+ signal.setitimer(signal.ITIMER_REAL, 0.01, 0) # Send SIGALRM once.
+ self.loop.run_forever()
+ self.assertEqual(caught, 1)
+
+ @unittest.skipUnless(hasattr(signal, 'SIGALRM'), 'No SIGALRM')
+ def test_signal_handling_args(self):
+ some_args = (42,)
+ caught = 0
+
+ def my_handler(*args):
+ nonlocal caught
+ caught += 1
+ self.assertEqual(args, some_args)
+
+ self.loop.add_signal_handler(signal.SIGALRM, my_handler, *some_args)
+
+ signal.setitimer(signal.ITIMER_REAL, 0.1, 0) # Send SIGALRM once.
+ self.loop.call_later(0.5, self.loop.stop)
+ self.loop.run_forever()
+ self.assertEqual(caught, 1)
+
+ def _basetest_create_connection(self, connection_fut, check_sockname=True):
+ tr, pr = self.loop.run_until_complete(connection_fut)
+ self.assertIsInstance(tr, asyncio.Transport)
+ self.assertIsInstance(pr, asyncio.Protocol)
+ if check_sockname:
+ self.assertIsNotNone(tr.get_extra_info('sockname'))
+ self.loop.run_until_complete(pr.done)
+ self.assertGreater(pr.nbytes, 0)
+ tr.close()
+
+ def test_create_connection(self):
+ with test_utils.run_test_server() as httpd:
+ conn_fut = self.loop.create_connection(
+ lambda: MyProto(loop=self.loop), *httpd.address)
+ self._basetest_create_connection(conn_fut)
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_unix_connection(self):
+ # Issue #20682: On Mac OS X Tiger, getsockname() returns a
+ # zero-length address for UNIX socket.
+ check_sockname = not osx_tiger()
+
+ with test_utils.run_test_unix_server() as httpd:
+ conn_fut = self.loop.create_unix_connection(
+ lambda: MyProto(loop=self.loop), httpd.address)
+ self._basetest_create_connection(conn_fut, check_sockname)
+
+ def test_create_connection_sock(self):
+ with test_utils.run_test_server() as httpd:
+ sock = None
+ infos = self.loop.run_until_complete(
+ self.loop.getaddrinfo(
+ *httpd.address, type=socket.SOCK_STREAM))
+ for family, type, proto, cname, address in infos:
+ try:
+ sock = socket.socket(family=family, type=type, proto=proto)
+ sock.setblocking(False)
+ self.loop.run_until_complete(
+ self.loop.sock_connect(sock, address))
+ except:
+ pass
+ else:
+ break
+ else:
+ assert False, 'Can not create socket.'
+
+ f = self.loop.create_connection(
+ lambda: MyProto(loop=self.loop), sock=sock)
+ tr, pr = self.loop.run_until_complete(f)
+ self.assertIsInstance(tr, asyncio.Transport)
+ self.assertIsInstance(pr, asyncio.Protocol)
+ self.loop.run_until_complete(pr.done)
+ self.assertGreater(pr.nbytes, 0)
+ tr.close()
+
+ def _basetest_create_ssl_connection(self, connection_fut,
+ check_sockname=True):
+ tr, pr = self.loop.run_until_complete(connection_fut)
+ self.assertIsInstance(tr, asyncio.Transport)
+ self.assertIsInstance(pr, asyncio.Protocol)
+ self.assertTrue('ssl' in tr.__class__.__name__.lower())
+ if check_sockname:
+ self.assertIsNotNone(tr.get_extra_info('sockname'))
+ self.loop.run_until_complete(pr.done)
+ self.assertGreater(pr.nbytes, 0)
+ tr.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ def test_create_ssl_connection(self):
+ with test_utils.run_test_server(use_ssl=True) as httpd:
+ conn_fut = self.loop.create_connection(
+ lambda: MyProto(loop=self.loop),
+ *httpd.address,
+ ssl=test_utils.dummy_ssl_context())
+
+ self._basetest_create_ssl_connection(conn_fut)
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_ssl_unix_connection(self):
+ # Issue #20682: On Mac OS X Tiger, getsockname() returns a
+ # zero-length address for UNIX socket.
+ check_sockname = not osx_tiger()
+
+ with test_utils.run_test_unix_server(use_ssl=True) as httpd:
+ conn_fut = self.loop.create_unix_connection(
+ lambda: MyProto(loop=self.loop),
+ httpd.address,
+ ssl=test_utils.dummy_ssl_context(),
+ server_hostname='127.0.0.1')
+
+ self._basetest_create_ssl_connection(conn_fut, check_sockname)
+
+ def test_create_connection_local_addr(self):
+ with test_utils.run_test_server() as httpd:
+ port = support.find_unused_port()
+ f = self.loop.create_connection(
+ lambda: MyProto(loop=self.loop),
+ *httpd.address, local_addr=(httpd.address[0], port))
+ tr, pr = self.loop.run_until_complete(f)
+ expected = pr.transport.get_extra_info('sockname')[1]
+ self.assertEqual(port, expected)
+ tr.close()
+
+ def test_create_connection_local_addr_in_use(self):
+ with test_utils.run_test_server() as httpd:
+ f = self.loop.create_connection(
+ lambda: MyProto(loop=self.loop),
+ *httpd.address, local_addr=httpd.address)
+ with self.assertRaises(OSError) as cm:
+ self.loop.run_until_complete(f)
+ self.assertEqual(cm.exception.errno, errno.EADDRINUSE)
+ self.assertIn(str(httpd.address), cm.exception.strerror)
+
+ def test_create_server(self):
+ proto = MyProto(self.loop)
+ f = self.loop.create_server(lambda: proto, '0.0.0.0', 0)
+ server = self.loop.run_until_complete(f)
+ self.assertEqual(len(server.sockets), 1)
+ sock = server.sockets[0]
+ host, port = sock.getsockname()
+ self.assertEqual(host, '0.0.0.0')
+ client = socket.socket()
+ client.connect(('127.0.0.1', port))
+ client.sendall(b'xxx')
+
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+
+ test_utils.run_until(self.loop, lambda: proto.nbytes > 0)
+ self.assertEqual(3, proto.nbytes)
+
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('sockname'))
+ self.assertEqual('127.0.0.1',
+ proto.transport.get_extra_info('peername')[0])
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+
+ self.assertEqual('CLOSED', proto.state)
+
+ # the client socket must be closed after to avoid ECONNRESET upon
+ # recv()/send() on the serving socket
+ client.close()
+
+ # close server
+ server.close()
+
+ def _make_unix_server(self, factory, **kwargs):
+ path = test_utils.gen_unix_socket_path()
+ self.addCleanup(lambda: os.path.exists(path) and os.unlink(path))
+
+ f = self.loop.create_unix_server(factory, path, **kwargs)
+ server = self.loop.run_until_complete(f)
+
+ return server, path
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_unix_server(self):
+ proto = MyProto(loop=self.loop)
+ server, path = self._make_unix_server(lambda: proto)
+ self.assertEqual(len(server.sockets), 1)
+
+ client = socket.socket(socket.AF_UNIX)
+ client.connect(path)
+ client.sendall(b'xxx')
+
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+ test_utils.run_until(self.loop, lambda: proto.nbytes > 0)
+ self.assertEqual(3, proto.nbytes)
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+
+ self.assertEqual('CLOSED', proto.state)
+
+ # the client socket must be closed after to avoid ECONNRESET upon
+ # recv()/send() on the serving socket
+ client.close()
+
+ # close server
+ server.close()
+
+ def _create_ssl_context(self, certfile, keyfile=None):
+ sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext.options |= ssl.OP_NO_SSLv2
+ sslcontext.load_cert_chain(certfile, keyfile)
+ return sslcontext
+
+ def _make_ssl_server(self, factory, certfile, keyfile=None):
+ sslcontext = self._create_ssl_context(certfile, keyfile)
+
+ f = self.loop.create_server(factory, '127.0.0.1', 0, ssl=sslcontext)
+ server = self.loop.run_until_complete(f)
+
+ sock = server.sockets[0]
+ host, port = sock.getsockname()
+ self.assertEqual(host, '127.0.0.1')
+ return server, host, port
+
+ def _make_ssl_unix_server(self, factory, certfile, keyfile=None):
+ sslcontext = self._create_ssl_context(certfile, keyfile)
+ return self._make_unix_server(factory, ssl=sslcontext)
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ def test_create_server_ssl(self):
+ proto = MyProto(loop=self.loop)
+ server, host, port = self._make_ssl_server(
+ lambda: proto, ONLYCERT, ONLYKEY)
+
+ f_c = self.loop.create_connection(MyBaseProto, host, port,
+ ssl=test_utils.dummy_ssl_context())
+ client, pr = self.loop.run_until_complete(f_c)
+
+ client.write(b'xxx')
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+
+ test_utils.run_until(self.loop, lambda: proto.nbytes > 0)
+ self.assertEqual(3, proto.nbytes)
+
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('sockname'))
+ self.assertEqual('127.0.0.1',
+ proto.transport.get_extra_info('peername')[0])
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual('CLOSED', proto.state)
+
+ # the client socket must be closed after to avoid ECONNRESET upon
+ # recv()/send() on the serving socket
+ client.close()
+
+ # stop serving
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_unix_server_ssl(self):
+ proto = MyProto(loop=self.loop)
+ server, path = self._make_ssl_unix_server(
+ lambda: proto, ONLYCERT, ONLYKEY)
+
+ f_c = self.loop.create_unix_connection(
+ MyBaseProto, path, ssl=test_utils.dummy_ssl_context(),
+ server_hostname='')
+
+ client, pr = self.loop.run_until_complete(f_c)
+
+ client.write(b'xxx')
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+ test_utils.run_until(self.loop, lambda: proto.nbytes > 0)
+ self.assertEqual(3, proto.nbytes)
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual('CLOSED', proto.state)
+
+ # the client socket must be closed after to avoid ECONNRESET upon
+ # recv()/send() on the serving socket
+ client.close()
+
+ # stop serving
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_create_server_ssl_verify_failed(self):
+ proto = MyProto(loop=self.loop)
+ server, host, port = self._make_ssl_server(
+ lambda: proto, SIGNED_CERTFILE)
+
+ sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext_client.options |= ssl.OP_NO_SSLv2
+ sslcontext_client.verify_mode = ssl.CERT_REQUIRED
+ if hasattr(sslcontext_client, 'check_hostname'):
+ sslcontext_client.check_hostname = True
+
+ # no CA loaded
+ f_c = self.loop.create_connection(MyProto, host, port,
+ ssl=sslcontext_client)
+ with self.assertRaisesRegex(ssl.SSLError,
+ 'certificate verify failed '):
+ self.loop.run_until_complete(f_c)
+
+ # close connection
+ self.assertIsNone(proto.transport)
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_unix_server_ssl_verify_failed(self):
+ proto = MyProto(loop=self.loop)
+ server, path = self._make_ssl_unix_server(
+ lambda: proto, SIGNED_CERTFILE)
+
+ sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext_client.options |= ssl.OP_NO_SSLv2
+ sslcontext_client.verify_mode = ssl.CERT_REQUIRED
+ if hasattr(sslcontext_client, 'check_hostname'):
+ sslcontext_client.check_hostname = True
+
+ # no CA loaded
+ f_c = self.loop.create_unix_connection(MyProto, path,
+ ssl=sslcontext_client,
+ server_hostname='invalid')
+ with self.assertRaisesRegex(ssl.SSLError,
+ 'certificate verify failed '):
+ self.loop.run_until_complete(f_c)
+
+ # close connection
+ self.assertIsNone(proto.transport)
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_create_server_ssl_match_failed(self):
+ proto = MyProto(loop=self.loop)
+ server, host, port = self._make_ssl_server(
+ lambda: proto, SIGNED_CERTFILE)
+
+ sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext_client.options |= ssl.OP_NO_SSLv2
+ sslcontext_client.verify_mode = ssl.CERT_REQUIRED
+ sslcontext_client.load_verify_locations(
+ cafile=SIGNING_CA)
+ if hasattr(sslcontext_client, 'check_hostname'):
+ sslcontext_client.check_hostname = True
+
+ # incorrect server_hostname
+ f_c = self.loop.create_connection(MyProto, host, port,
+ ssl=sslcontext_client)
+ with self.assertRaisesRegex(
+ ssl.CertificateError,
+ "hostname '127.0.0.1' doesn't match 'localhost'"):
+ self.loop.run_until_complete(f_c)
+
+ # close connection
+ proto.transport.close()
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_create_unix_server_ssl_verified(self):
+ proto = MyProto(loop=self.loop)
+ server, path = self._make_ssl_unix_server(
+ lambda: proto, SIGNED_CERTFILE)
+
+ sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext_client.options |= ssl.OP_NO_SSLv2
+ sslcontext_client.verify_mode = ssl.CERT_REQUIRED
+ sslcontext_client.load_verify_locations(cafile=SIGNING_CA)
+ if hasattr(sslcontext_client, 'check_hostname'):
+ sslcontext_client.check_hostname = True
+
+ # Connection succeeds with correct CA and server hostname.
+ f_c = self.loop.create_unix_connection(MyProto, path,
+ ssl=sslcontext_client,
+ server_hostname='localhost')
+ client, pr = self.loop.run_until_complete(f_c)
+
+ # close connection
+ proto.transport.close()
+ client.close()
+ server.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_create_server_ssl_verified(self):
+ proto = MyProto(loop=self.loop)
+ server, host, port = self._make_ssl_server(
+ lambda: proto, SIGNED_CERTFILE)
+
+ sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ sslcontext_client.options |= ssl.OP_NO_SSLv2
+ sslcontext_client.verify_mode = ssl.CERT_REQUIRED
+ sslcontext_client.load_verify_locations(cafile=SIGNING_CA)
+ if hasattr(sslcontext_client, 'check_hostname'):
+ sslcontext_client.check_hostname = True
+
+ # Connection succeeds with correct CA and server hostname.
+ f_c = self.loop.create_connection(MyProto, host, port,
+ ssl=sslcontext_client,
+ server_hostname='localhost')
+ client, pr = self.loop.run_until_complete(f_c)
+
+ # close connection
+ proto.transport.close()
+ client.close()
+ server.close()
+
+ def test_create_server_sock(self):
+ proto = asyncio.Future(loop=self.loop)
+
+ class TestMyProto(MyProto):
+ def connection_made(self, transport):
+ super().connection_made(transport)
+ proto.set_result(self)
+
+ sock_ob = socket.socket(type=socket.SOCK_STREAM)
+ sock_ob.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock_ob.bind(('0.0.0.0', 0))
+
+ f = self.loop.create_server(TestMyProto, sock=sock_ob)
+ server = self.loop.run_until_complete(f)
+ sock = server.sockets[0]
+ self.assertIs(sock, sock_ob)
+
+ host, port = sock.getsockname()
+ self.assertEqual(host, '0.0.0.0')
+ client = socket.socket()
+ client.connect(('127.0.0.1', port))
+ client.send(b'xxx')
+ client.close()
+ server.close()
+
+ def test_create_server_addr_in_use(self):
+ sock_ob = socket.socket(type=socket.SOCK_STREAM)
+ sock_ob.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock_ob.bind(('0.0.0.0', 0))
+
+ f = self.loop.create_server(MyProto, sock=sock_ob)
+ server = self.loop.run_until_complete(f)
+ sock = server.sockets[0]
+ host, port = sock.getsockname()
+
+ f = self.loop.create_server(MyProto, host=host, port=port)
+ with self.assertRaises(OSError) as cm:
+ self.loop.run_until_complete(f)
+ self.assertEqual(cm.exception.errno, errno.EADDRINUSE)
+
+ server.close()
+
+ @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled')
+ def test_create_server_dual_stack(self):
+ f_proto = asyncio.Future(loop=self.loop)
+
+ class TestMyProto(MyProto):
+ def connection_made(self, transport):
+ super().connection_made(transport)
+ f_proto.set_result(self)
+
+ try_count = 0
+ while True:
+ try:
+ port = support.find_unused_port()
+ f = self.loop.create_server(TestMyProto, host=None, port=port)
+ server = self.loop.run_until_complete(f)
+ except OSError as ex:
+ if ex.errno == errno.EADDRINUSE:
+ try_count += 1
+ self.assertGreaterEqual(5, try_count)
+ continue
+ else:
+ raise
+ else:
+ break
+ client = socket.socket()
+ client.connect(('127.0.0.1', port))
+ client.send(b'xxx')
+ proto = self.loop.run_until_complete(f_proto)
+ proto.transport.close()
+ client.close()
+
+ f_proto = asyncio.Future(loop=self.loop)
+ client = socket.socket(socket.AF_INET6)
+ client.connect(('::1', port))
+ client.send(b'xxx')
+ proto = self.loop.run_until_complete(f_proto)
+ proto.transport.close()
+ client.close()
+
+ server.close()
+
+ def test_server_close(self):
+ f = self.loop.create_server(MyProto, '0.0.0.0', 0)
+ server = self.loop.run_until_complete(f)
+ sock = server.sockets[0]
+ host, port = sock.getsockname()
+
+ client = socket.socket()
+ client.connect(('127.0.0.1', port))
+ client.send(b'xxx')
+ client.close()
+
+ server.close()
+
+ client = socket.socket()
+ self.assertRaises(
+ ConnectionRefusedError, client.connect, ('127.0.0.1', port))
+ client.close()
+
+ def test_create_datagram_endpoint(self):
+ class TestMyDatagramProto(MyDatagramProto):
+ def __init__(inner_self):
+ super().__init__(loop=self.loop)
+
+ def datagram_received(self, data, addr):
+ super().datagram_received(data, addr)
+ self.transport.sendto(b'resp:'+data, addr)
+
+ coro = self.loop.create_datagram_endpoint(
+ TestMyDatagramProto, local_addr=('127.0.0.1', 0))
+ s_transport, server = self.loop.run_until_complete(coro)
+ host, port = s_transport.get_extra_info('sockname')
+
+ coro = self.loop.create_datagram_endpoint(
+ lambda: MyDatagramProto(loop=self.loop),
+ remote_addr=(host, port))
+ transport, client = self.loop.run_until_complete(coro)
+
+ self.assertEqual('INITIALIZED', client.state)
+ transport.sendto(b'xxx')
+ test_utils.run_until(self.loop, lambda: server.nbytes)
+ self.assertEqual(3, server.nbytes)
+ test_utils.run_until(self.loop, lambda: client.nbytes)
+
+ # received
+ self.assertEqual(8, client.nbytes)
+
+ # extra info is available
+ self.assertIsNotNone(transport.get_extra_info('sockname'))
+
+ # close connection
+ transport.close()
+ self.loop.run_until_complete(client.done)
+ self.assertEqual('CLOSED', client.state)
+ server.transport.close()
+
+ def test_internal_fds(self):
+ loop = self.create_event_loop()
+ if not isinstance(loop, selector_events.BaseSelectorEventLoop):
+ self.skipTest('loop is not a BaseSelectorEventLoop')
+
+ self.assertEqual(1, loop._internal_fds)
+ loop.close()
+ self.assertEqual(0, loop._internal_fds)
+ self.assertIsNone(loop._csock)
+ self.assertIsNone(loop._ssock)
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ "Don't support pipes for Windows")
+ def test_read_pipe(self):
+ proto = MyReadPipeProto(loop=self.loop)
+
+ rpipe, wpipe = os.pipe()
+ pipeobj = io.open(rpipe, 'rb', 1024)
+
+ @asyncio.coroutine
+ def connect():
+ t, p = yield from self.loop.connect_read_pipe(
+ lambda: proto, pipeobj)
+ self.assertIs(p, proto)
+ self.assertIs(t, proto.transport)
+ self.assertEqual(['INITIAL', 'CONNECTED'], proto.state)
+ self.assertEqual(0, proto.nbytes)
+
+ self.loop.run_until_complete(connect())
+
+ os.write(wpipe, b'1')
+ test_utils.run_until(self.loop, lambda: proto.nbytes >= 1)
+ self.assertEqual(1, proto.nbytes)
+
+ os.write(wpipe, b'2345')
+ test_utils.run_until(self.loop, lambda: proto.nbytes >= 5)
+ self.assertEqual(['INITIAL', 'CONNECTED'], proto.state)
+ self.assertEqual(5, proto.nbytes)
+
+ os.close(wpipe)
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual(
+ ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state)
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('pipe'))
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ "Don't support pipes for Windows")
+ # select, poll and kqueue don't support character devices (PTY) on Mac OS X
+ # older than 10.6 (Snow Leopard)
+ @support.requires_mac_ver(10, 6)
+ # Issue #20495: The test hangs on FreeBSD 7.2 but pass on FreeBSD 9
+ @support.requires_freebsd_version(8)
+ def test_read_pty_output(self):
+ proto = MyReadPipeProto(loop=self.loop)
+
+ master, slave = os.openpty()
+ master_read_obj = io.open(master, 'rb', 0)
+
+ @asyncio.coroutine
+ def connect():
+ t, p = yield from self.loop.connect_read_pipe(lambda: proto,
+ master_read_obj)
+ self.assertIs(p, proto)
+ self.assertIs(t, proto.transport)
+ self.assertEqual(['INITIAL', 'CONNECTED'], proto.state)
+ self.assertEqual(0, proto.nbytes)
+
+ self.loop.run_until_complete(connect())
+
+ os.write(slave, b'1')
+ test_utils.run_until(self.loop, lambda: proto.nbytes)
+ self.assertEqual(1, proto.nbytes)
+
+ os.write(slave, b'2345')
+ test_utils.run_until(self.loop, lambda: proto.nbytes >= 5)
+ self.assertEqual(['INITIAL', 'CONNECTED'], proto.state)
+ self.assertEqual(5, proto.nbytes)
+
+ os.close(slave)
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual(
+ ['INITIAL', 'CONNECTED', 'EOF', 'CLOSED'], proto.state)
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('pipe'))
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ "Don't support pipes for Windows")
+ def test_write_pipe(self):
+ rpipe, wpipe = os.pipe()
+ pipeobj = io.open(wpipe, 'wb', 1024)
+
+ proto = MyWritePipeProto(loop=self.loop)
+ connect = self.loop.connect_write_pipe(lambda: proto, pipeobj)
+ transport, p = self.loop.run_until_complete(connect)
+ self.assertIs(p, proto)
+ self.assertIs(transport, proto.transport)
+ self.assertEqual('CONNECTED', proto.state)
+
+ transport.write(b'1')
+
+ data = bytearray()
+ def reader(data):
+ chunk = os.read(rpipe, 1024)
+ data += chunk
+ return len(data)
+
+ test_utils.run_until(self.loop, lambda: reader(data) >= 1)
+ self.assertEqual(b'1', data)
+
+ transport.write(b'2345')
+ test_utils.run_until(self.loop, lambda: reader(data) >= 5)
+ self.assertEqual(b'12345', data)
+ self.assertEqual('CONNECTED', proto.state)
+
+ os.close(rpipe)
+
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('pipe'))
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual('CLOSED', proto.state)
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ "Don't support pipes for Windows")
+ def test_write_pipe_disconnect_on_close(self):
+ rsock, wsock = test_utils.socketpair()
+ pipeobj = io.open(wsock.detach(), 'wb', 1024)
+
+ proto = MyWritePipeProto(loop=self.loop)
+ connect = self.loop.connect_write_pipe(lambda: proto, pipeobj)
+ transport, p = self.loop.run_until_complete(connect)
+ self.assertIs(p, proto)
+ self.assertIs(transport, proto.transport)
+ self.assertEqual('CONNECTED', proto.state)
+
+ transport.write(b'1')
+ data = self.loop.run_until_complete(self.loop.sock_recv(rsock, 1024))
+ self.assertEqual(b'1', data)
+
+ rsock.close()
+
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual('CLOSED', proto.state)
+
+ @unittest.skipUnless(sys.platform != 'win32',
+ "Don't support pipes for Windows")
+ # select, poll and kqueue don't support character devices (PTY) on Mac OS X
+ # older than 10.6 (Snow Leopard)
+ @support.requires_mac_ver(10, 6)
+ def test_write_pty(self):
+ master, slave = os.openpty()
+ slave_write_obj = io.open(slave, 'wb', 0)
+
+ proto = MyWritePipeProto(loop=self.loop)
+ connect = self.loop.connect_write_pipe(lambda: proto, slave_write_obj)
+ transport, p = self.loop.run_until_complete(connect)
+ self.assertIs(p, proto)
+ self.assertIs(transport, proto.transport)
+ self.assertEqual('CONNECTED', proto.state)
+
+ transport.write(b'1')
+
+ data = bytearray()
+ def reader(data):
+ chunk = os.read(master, 1024)
+ data += chunk
+ return len(data)
+
+ test_utils.run_until(self.loop, lambda: reader(data) >= 1,
+ timeout=10)
+ self.assertEqual(b'1', data)
+
+ transport.write(b'2345')
+ test_utils.run_until(self.loop, lambda: reader(data) >= 5,
+ timeout=10)
+ self.assertEqual(b'12345', data)
+ self.assertEqual('CONNECTED', proto.state)
+
+ os.close(master)
+
+ # extra info is available
+ self.assertIsNotNone(proto.transport.get_extra_info('pipe'))
+
+ # close connection
+ proto.transport.close()
+ self.loop.run_until_complete(proto.done)
+ self.assertEqual('CLOSED', proto.state)
+
+ def test_prompt_cancellation(self):
+ r, w = test_utils.socketpair()
+ r.setblocking(False)
+ f = self.loop.sock_recv(r, 1)
+ ov = getattr(f, 'ov', None)
+ if ov is not None:
+ self.assertTrue(ov.pending)
+
+ @asyncio.coroutine
+ def main():
+ try:
+ self.loop.call_soon(f.cancel)
+ yield from f
+ except asyncio.CancelledError:
+ res = 'cancelled'
+ else:
+ res = None
+ finally:
+ self.loop.stop()
+ return res
+
+ start = time.monotonic()
+ t = asyncio.Task(main(), loop=self.loop)
+ self.loop.run_forever()
+ elapsed = time.monotonic() - start
+
+ self.assertLess(elapsed, 0.1)
+ self.assertEqual(t.result(), 'cancelled')
+ self.assertRaises(asyncio.CancelledError, f.result)
+ if ov is not None:
+ self.assertFalse(ov.pending)
+ self.loop._stop_serving(r)
+
+ r.close()
+ w.close()
+
+ def test_timeout_rounding(self):
+ def _run_once():
+ self.loop._run_once_counter += 1
+ orig_run_once()
+
+ orig_run_once = self.loop._run_once
+ self.loop._run_once_counter = 0
+ self.loop._run_once = _run_once
+
+ @asyncio.coroutine
+ def wait():
+ loop = self.loop
+ yield from asyncio.sleep(1e-2, loop=loop)
+ yield from asyncio.sleep(1e-4, loop=loop)
+ yield from asyncio.sleep(1e-6, loop=loop)
+ yield from asyncio.sleep(1e-8, loop=loop)
+ yield from asyncio.sleep(1e-10, loop=loop)
+
+ self.loop.run_until_complete(wait())
+ # The ideal number of call is 12, but on some platforms, the selector
+ # may sleep at little bit less than timeout depending on the resolution
+ # of the clock used by the kernel. Tolerate a few useless calls on
+ # these platforms.
+ self.assertLessEqual(self.loop._run_once_counter, 20,
+ {'clock_resolution': self.loop._clock_resolution,
+ 'selector': self.loop._selector.__class__.__name__})
+
+ def test_sock_connect_address(self):
+ addresses = [(socket.AF_INET, ('www.python.org', 80))]
+ if support.IPV6_ENABLED:
+ addresses.extend((
+ (socket.AF_INET6, ('www.python.org', 80)),
+ (socket.AF_INET6, ('www.python.org', 80, 0, 0)),
+ ))
+
+ for family, address in addresses:
+ for sock_type in (socket.SOCK_STREAM, socket.SOCK_DGRAM):
+ sock = socket.socket(family, sock_type)
+ with sock:
+ connect = self.loop.sock_connect(sock, address)
+ with self.assertRaises(ValueError) as cm:
+ self.loop.run_until_complete(connect)
+ self.assertIn('address must be resolved',
+ str(cm.exception))
+
+ def test_remove_fds_after_closing(self):
+ loop = self.create_event_loop()
+ callback = lambda: None
+ r, w = test_utils.socketpair()
+ self.addCleanup(r.close)
+ self.addCleanup(w.close)
+ loop.add_reader(r, callback)
+ loop.add_writer(w, callback)
+ loop.close()
+ self.assertFalse(loop.remove_reader(r))
+ self.assertFalse(loop.remove_writer(w))
+
+ def test_add_fds_after_closing(self):
+ loop = self.create_event_loop()
+ callback = lambda: None
+ r, w = test_utils.socketpair()
+ self.addCleanup(r.close)
+ self.addCleanup(w.close)
+ loop.close()
+ with self.assertRaises(RuntimeError):
+ loop.add_reader(r, callback)
+ with self.assertRaises(RuntimeError):
+ loop.add_writer(w, callback)
+
+
+class SubprocessTestsMixin:
+
+ def check_terminated(self, returncode):
+ if sys.platform == 'win32':
+ self.assertIsInstance(returncode, int)
+ # expect 1 but sometimes get 0
+ else:
+ self.assertEqual(-signal.SIGTERM, returncode)
+
+ def check_killed(self, returncode):
+ if sys.platform == 'win32':
+ self.assertIsInstance(returncode, int)
+ # expect 1 but sometimes get 0
+ else:
+ self.assertEqual(-signal.SIGKILL, returncode)
+
+ def test_subprocess_exec(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+
+ stdin = transp.get_pipe_transport(0)
+ stdin.write(b'Python The Winner')
+ self.loop.run_until_complete(proto.got_data[1].wait())
+ transp.close()
+ self.loop.run_until_complete(proto.completed)
+ self.check_terminated(proto.returncode)
+ self.assertEqual(b'Python The Winner', proto.data[1])
+
+ def test_subprocess_interactive(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+ self.assertEqual('CONNECTED', proto.state)
+
+ try:
+ stdin = transp.get_pipe_transport(0)
+ stdin.write(b'Python ')
+ self.loop.run_until_complete(proto.got_data[1].wait())
+ proto.got_data[1].clear()
+ self.assertEqual(b'Python ', proto.data[1])
+
+ stdin.write(b'The Winner')
+ self.loop.run_until_complete(proto.got_data[1].wait())
+ self.assertEqual(b'Python The Winner', proto.data[1])
+ finally:
+ transp.close()
+
+ self.loop.run_until_complete(proto.completed)
+ self.check_terminated(proto.returncode)
+
+ def test_subprocess_shell(self):
+ connect = self.loop.subprocess_shell(
+ functools.partial(MySubprocessProtocol, self.loop),
+ 'echo Python')
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ transp.get_pipe_transport(0).close()
+ self.loop.run_until_complete(proto.completed)
+ self.assertEqual(0, proto.returncode)
+ self.assertTrue(all(f.done() for f in proto.disconnects.values()))
+ self.assertEqual(proto.data[1].rstrip(b'\r\n'), b'Python')
+ self.assertEqual(proto.data[2], b'')
+
+ def test_subprocess_exitcode(self):
+ connect = self.loop.subprocess_shell(
+ functools.partial(MySubprocessProtocol, self.loop),
+ 'exit 7', stdin=None, stdout=None, stderr=None)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.completed)
+ self.assertEqual(7, proto.returncode)
+
+ def test_subprocess_close_after_finish(self):
+ connect = self.loop.subprocess_shell(
+ functools.partial(MySubprocessProtocol, self.loop),
+ 'exit 7', stdin=None, stdout=None, stderr=None)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.assertIsNone(transp.get_pipe_transport(0))
+ self.assertIsNone(transp.get_pipe_transport(1))
+ self.assertIsNone(transp.get_pipe_transport(2))
+ self.loop.run_until_complete(proto.completed)
+ self.assertEqual(7, proto.returncode)
+ self.assertIsNone(transp.close())
+
+ def test_subprocess_kill(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ transp.kill()
+ self.loop.run_until_complete(proto.completed)
+ self.check_killed(proto.returncode)
+
+ def test_subprocess_terminate(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ transp.terminate()
+ self.loop.run_until_complete(proto.completed)
+ self.check_terminated(proto.returncode)
+
+ @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP")
+ def test_subprocess_send_signal(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ transp.send_signal(signal.SIGHUP)
+ self.loop.run_until_complete(proto.completed)
+ self.assertEqual(-signal.SIGHUP, proto.returncode)
+
+ def test_subprocess_stderr(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo2.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ stdin = transp.get_pipe_transport(0)
+ stdin.write(b'test')
+
+ self.loop.run_until_complete(proto.completed)
+
+ transp.close()
+ self.assertEqual(b'OUT:test', proto.data[1])
+ self.assertTrue(proto.data[2].startswith(b'ERR:test'), proto.data[2])
+ self.assertEqual(0, proto.returncode)
+
+ def test_subprocess_stderr_redirect_to_stdout(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo2.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog, stderr=subprocess.STDOUT)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ stdin = transp.get_pipe_transport(0)
+ self.assertIsNotNone(transp.get_pipe_transport(1))
+ self.assertIsNone(transp.get_pipe_transport(2))
+
+ stdin.write(b'test')
+ self.loop.run_until_complete(proto.completed)
+ self.assertTrue(proto.data[1].startswith(b'OUT:testERR:test'),
+ proto.data[1])
+ self.assertEqual(b'', proto.data[2])
+
+ transp.close()
+ self.assertEqual(0, proto.returncode)
+
+ def test_subprocess_close_client_stream(self):
+ prog = os.path.join(os.path.dirname(__file__), 'echo3.py')
+
+ connect = self.loop.subprocess_exec(
+ functools.partial(MySubprocessProtocol, self.loop),
+ sys.executable, prog)
+ transp, proto = self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.connected)
+
+ stdin = transp.get_pipe_transport(0)
+ stdout = transp.get_pipe_transport(1)
+ stdin.write(b'test')
+ self.loop.run_until_complete(proto.got_data[1].wait())
+ self.assertEqual(b'OUT:test', proto.data[1])
+
+ stdout.close()
+ self.loop.run_until_complete(proto.disconnects[1])
+ stdin.write(b'xxx')
+ self.loop.run_until_complete(proto.got_data[2].wait())
+ if sys.platform != 'win32':
+ self.assertEqual(b'ERR:BrokenPipeError', proto.data[2])
+ else:
+ # After closing the read-end of a pipe, writing to the
+ # write-end using os.write() fails with errno==EINVAL and
+ # GetLastError()==ERROR_INVALID_NAME on Windows!?! (Using
+ # WriteFile() we get ERROR_BROKEN_PIPE as expected.)
+ self.assertEqual(b'ERR:OSError', proto.data[2])
+ transp.close()
+ self.loop.run_until_complete(proto.completed)
+ self.check_terminated(proto.returncode)
+
+ def test_subprocess_wait_no_same_group(self):
+ # start the new process in a new session
+ connect = self.loop.subprocess_shell(
+ functools.partial(MySubprocessProtocol, self.loop),
+ 'exit 7', stdin=None, stdout=None, stderr=None,
+ start_new_session=True)
+ _, proto = yield self.loop.run_until_complete(connect)
+ self.assertIsInstance(proto, MySubprocessProtocol)
+ self.loop.run_until_complete(proto.completed)
+ self.assertEqual(7, proto.returncode)
+
+ def test_subprocess_exec_invalid_args(self):
+ @asyncio.coroutine
+ def connect(**kwds):
+ yield from self.loop.subprocess_exec(
+ asyncio.SubprocessProtocol,
+ 'pwd', **kwds)
+
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(universal_newlines=True))
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(bufsize=4096))
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(shell=True))
+
+ def test_subprocess_shell_invalid_args(self):
+ @asyncio.coroutine
+ def connect(cmd=None, **kwds):
+ if not cmd:
+ cmd = 'pwd'
+ yield from self.loop.subprocess_shell(
+ asyncio.SubprocessProtocol,
+ cmd, **kwds)
+
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(['ls', '-l']))
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(universal_newlines=True))
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(bufsize=4096))
+ with self.assertRaises(ValueError):
+ self.loop.run_until_complete(connect(shell=False))
+
+
+if sys.platform == 'win32':
+
+ class SelectEventLoopTests(EventLoopTestsMixin, unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.SelectorEventLoop()
+
+ class ProactorEventLoopTests(EventLoopTestsMixin,
+ SubprocessTestsMixin,
+ unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.ProactorEventLoop()
+
+ def test_create_ssl_connection(self):
+ raise unittest.SkipTest("IocpEventLoop incompatible with SSL")
+
+ def test_create_server_ssl(self):
+ raise unittest.SkipTest("IocpEventLoop incompatible with SSL")
+
+ def test_create_server_ssl_verify_failed(self):
+ raise unittest.SkipTest("IocpEventLoop incompatible with SSL")
+
+ def test_create_server_ssl_match_failed(self):
+ raise unittest.SkipTest("IocpEventLoop incompatible with SSL")
+
+ def test_create_server_ssl_verified(self):
+ raise unittest.SkipTest("IocpEventLoop incompatible with SSL")
+
+ def test_reader_callback(self):
+ raise unittest.SkipTest("IocpEventLoop does not have add_reader()")
+
+ def test_reader_callback_cancel(self):
+ raise unittest.SkipTest("IocpEventLoop does not have add_reader()")
+
+ def test_writer_callback(self):
+ raise unittest.SkipTest("IocpEventLoop does not have add_writer()")
+
+ def test_writer_callback_cancel(self):
+ raise unittest.SkipTest("IocpEventLoop does not have add_writer()")
+
+ def test_create_datagram_endpoint(self):
+ raise unittest.SkipTest(
+ "IocpEventLoop does not have create_datagram_endpoint()")
+
+ def test_remove_fds_after_closing(self):
+ raise unittest.SkipTest("IocpEventLoop does not have add_reader()")
+else:
+ from asyncio import selectors
+
+ class UnixEventLoopTestsMixin(EventLoopTestsMixin):
+ def setUp(self):
+ super().setUp()
+ watcher = asyncio.SafeChildWatcher()
+ watcher.attach_loop(self.loop)
+ asyncio.set_child_watcher(watcher)
+
+ def tearDown(self):
+ asyncio.set_child_watcher(None)
+ super().tearDown()
+
+ if hasattr(selectors, 'KqueueSelector'):
+ class KqueueEventLoopTests(UnixEventLoopTestsMixin,
+ SubprocessTestsMixin,
+ unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.SelectorEventLoop(
+ selectors.KqueueSelector())
+
+ # kqueue doesn't support character devices (PTY) on Mac OS X older
+ # than 10.9 (Maverick)
+ @support.requires_mac_ver(10, 9)
+ # Issue #20667: KqueueEventLoopTests.test_read_pty_output()
+ # hangs on OpenBSD 5.5
+ @unittest.skipIf(sys.platform.startswith('openbsd'),
+ 'test hangs on OpenBSD')
+ def test_read_pty_output(self):
+ super().test_read_pty_output()
+
+ # kqueue doesn't support character devices (PTY) on Mac OS X older
+ # than 10.9 (Maverick)
+ @support.requires_mac_ver(10, 9)
+ def test_write_pty(self):
+ super().test_write_pty()
+
+ if hasattr(selectors, 'EpollSelector'):
+ class EPollEventLoopTests(UnixEventLoopTestsMixin,
+ SubprocessTestsMixin,
+ unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.SelectorEventLoop(selectors.EpollSelector())
+
+ if hasattr(selectors, 'PollSelector'):
+ class PollEventLoopTests(UnixEventLoopTestsMixin,
+ SubprocessTestsMixin,
+ unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.SelectorEventLoop(selectors.PollSelector())
+
+ # Should always exist.
+ class SelectEventLoopTests(UnixEventLoopTestsMixin,
+ SubprocessTestsMixin,
+ unittest.TestCase):
+
+ def create_event_loop(self):
+ return asyncio.SelectorEventLoop(selectors.SelectSelector())
+
+
+class HandleTests(unittest.TestCase):
+
+ def test_handle(self):
+ def callback(*args):
+ return args
+
+ args = ()
+ h = asyncio.Handle(callback, args, mock.Mock())
+ self.assertIs(h._callback, callback)
+ self.assertIs(h._args, args)
+ self.assertFalse(h._cancelled)
+
+ r = repr(h)
+ self.assertTrue(r.startswith(
+ 'Handle('
+ '<function HandleTests.test_handle.<locals>.callback'))
+ self.assertTrue(r.endswith('())'))
+
+ h.cancel()
+ self.assertTrue(h._cancelled)
+
+ r = repr(h)
+ self.assertTrue(r.startswith(
+ 'Handle('
+ '<function HandleTests.test_handle.<locals>.callback'))
+ self.assertTrue(r.endswith('())<cancelled>'), r)
+
+ def test_handle_from_handle(self):
+ def callback(*args):
+ return args
+ m_loop = object()
+ h1 = asyncio.Handle(callback, (), loop=m_loop)
+ self.assertRaises(
+ AssertionError, asyncio.Handle, h1, (), m_loop)
+
+ def test_callback_with_exception(self):
+ def callback():
+ raise ValueError()
+
+ m_loop = mock.Mock()
+ m_loop.call_exception_handler = mock.Mock()
+
+ h = asyncio.Handle(callback, (), m_loop)
+ h._run()
+
+ m_loop.call_exception_handler.assert_called_with({
+ 'message': test_utils.MockPattern('Exception in callback.*'),
+ 'exception': mock.ANY,
+ 'handle': h
+ })
+
+
+class TimerTests(unittest.TestCase):
+
+ def test_hash(self):
+ when = time.monotonic()
+ h = asyncio.TimerHandle(when, lambda: False, (),
+ mock.Mock())
+ self.assertEqual(hash(h), hash(when))
+
+ def test_timer(self):
+ def callback(*args):
+ return args
+
+ args = ()
+ when = time.monotonic()
+ h = asyncio.TimerHandle(when, callback, args, mock.Mock())
+ self.assertIs(h._callback, callback)
+ self.assertIs(h._args, args)
+ self.assertFalse(h._cancelled)
+
+ r = repr(h)
+ self.assertTrue(r.endswith('())'))
+
+ h.cancel()
+ self.assertTrue(h._cancelled)
+
+ r = repr(h)
+ self.assertTrue(r.endswith('())<cancelled>'), r)
+
+ self.assertRaises(AssertionError,
+ asyncio.TimerHandle, None, callback, args,
+ mock.Mock())
+
+ def test_timer_comparison(self):
+ loop = mock.Mock()
+
+ def callback(*args):
+ return args
+
+ when = time.monotonic()
+
+ h1 = asyncio.TimerHandle(when, callback, (), loop)
+ h2 = asyncio.TimerHandle(when, callback, (), loop)
+ # TODO: Use assertLess etc.
+ self.assertFalse(h1 < h2)
+ self.assertFalse(h2 < h1)
+ self.assertTrue(h1 <= h2)
+ self.assertTrue(h2 <= h1)
+ self.assertFalse(h1 > h2)
+ self.assertFalse(h2 > h1)
+ self.assertTrue(h1 >= h2)
+ self.assertTrue(h2 >= h1)
+ self.assertTrue(h1 == h2)
+ self.assertFalse(h1 != h2)
+
+ h2.cancel()
+ self.assertFalse(h1 == h2)
+
+ h1 = asyncio.TimerHandle(when, callback, (), loop)
+ h2 = asyncio.TimerHandle(when + 10.0, callback, (), loop)
+ self.assertTrue(h1 < h2)
+ self.assertFalse(h2 < h1)
+ self.assertTrue(h1 <= h2)
+ self.assertFalse(h2 <= h1)
+ self.assertFalse(h1 > h2)
+ self.assertTrue(h2 > h1)
+ self.assertFalse(h1 >= h2)
+ self.assertTrue(h2 >= h1)
+ self.assertFalse(h1 == h2)
+ self.assertTrue(h1 != h2)
+
+ h3 = asyncio.Handle(callback, (), loop)
+ self.assertIs(NotImplemented, h1.__eq__(h3))
+ self.assertIs(NotImplemented, h1.__ne__(h3))
+
+
+class AbstractEventLoopTests(unittest.TestCase):
+
+ def test_not_implemented(self):
+ f = mock.Mock()
+ loop = asyncio.AbstractEventLoop()
+ self.assertRaises(
+ NotImplementedError, loop.run_forever)
+ self.assertRaises(
+ NotImplementedError, loop.run_until_complete, None)
+ self.assertRaises(
+ NotImplementedError, loop.stop)
+ self.assertRaises(
+ NotImplementedError, loop.is_running)
+ self.assertRaises(
+ NotImplementedError, loop.close)
+ self.assertRaises(
+ NotImplementedError, loop.call_later, None, None)
+ self.assertRaises(
+ NotImplementedError, loop.call_at, f, f)
+ self.assertRaises(
+ NotImplementedError, loop.call_soon, None)
+ self.assertRaises(
+ NotImplementedError, loop.time)
+ self.assertRaises(
+ NotImplementedError, loop.call_soon_threadsafe, None)
+ self.assertRaises(
+ NotImplementedError, loop.run_in_executor, f, f)
+ self.assertRaises(
+ NotImplementedError, loop.set_default_executor, f)
+ self.assertRaises(
+ NotImplementedError, loop.getaddrinfo, 'localhost', 8080)
+ self.assertRaises(
+ NotImplementedError, loop.getnameinfo, ('localhost', 8080))
+ self.assertRaises(
+ NotImplementedError, loop.create_connection, f)
+ self.assertRaises(
+ NotImplementedError, loop.create_server, f)
+ self.assertRaises(
+ NotImplementedError, loop.create_datagram_endpoint, f)
+ self.assertRaises(
+ NotImplementedError, loop.add_reader, 1, f)
+ self.assertRaises(
+ NotImplementedError, loop.remove_reader, 1)
+ self.assertRaises(
+ NotImplementedError, loop.add_writer, 1, f)
+ self.assertRaises(
+ NotImplementedError, loop.remove_writer, 1)
+ self.assertRaises(
+ NotImplementedError, loop.sock_recv, f, 10)
+ self.assertRaises(
+ NotImplementedError, loop.sock_sendall, f, 10)
+ self.assertRaises(
+ NotImplementedError, loop.sock_connect, f, f)
+ self.assertRaises(
+ NotImplementedError, loop.sock_accept, f)
+ self.assertRaises(
+ NotImplementedError, loop.add_signal_handler, 1, f)
+ self.assertRaises(
+ NotImplementedError, loop.remove_signal_handler, 1)
+ self.assertRaises(
+ NotImplementedError, loop.remove_signal_handler, 1)
+ self.assertRaises(
+ NotImplementedError, loop.connect_read_pipe, f,
+ mock.sentinel.pipe)
+ self.assertRaises(
+ NotImplementedError, loop.connect_write_pipe, f,
+ mock.sentinel.pipe)
+ self.assertRaises(
+ NotImplementedError, loop.subprocess_shell, f,
+ mock.sentinel)
+ self.assertRaises(
+ NotImplementedError, loop.subprocess_exec, f)
+
+
+class ProtocolsAbsTests(unittest.TestCase):
+
+ def test_empty(self):
+ f = mock.Mock()
+ p = asyncio.Protocol()
+ self.assertIsNone(p.connection_made(f))
+ self.assertIsNone(p.connection_lost(f))
+ self.assertIsNone(p.data_received(f))
+ self.assertIsNone(p.eof_received())
+
+ dp = asyncio.DatagramProtocol()
+ self.assertIsNone(dp.connection_made(f))
+ self.assertIsNone(dp.connection_lost(f))
+ self.assertIsNone(dp.error_received(f))
+ self.assertIsNone(dp.datagram_received(f, f))
+
+ sp = asyncio.SubprocessProtocol()
+ self.assertIsNone(sp.connection_made(f))
+ self.assertIsNone(sp.connection_lost(f))
+ self.assertIsNone(sp.pipe_data_received(1, f))
+ self.assertIsNone(sp.pipe_connection_lost(1, f))
+ self.assertIsNone(sp.process_exited())
+
+
+class PolicyTests(unittest.TestCase):
+
+ def test_event_loop_policy(self):
+ policy = asyncio.AbstractEventLoopPolicy()
+ self.assertRaises(NotImplementedError, policy.get_event_loop)
+ self.assertRaises(NotImplementedError, policy.set_event_loop, object())
+ self.assertRaises(NotImplementedError, policy.new_event_loop)
+ self.assertRaises(NotImplementedError, policy.get_child_watcher)
+ self.assertRaises(NotImplementedError, policy.set_child_watcher,
+ object())
+
+ def test_get_event_loop(self):
+ policy = asyncio.DefaultEventLoopPolicy()
+ self.assertIsNone(policy._local._loop)
+
+ loop = policy.get_event_loop()
+ self.assertIsInstance(loop, asyncio.AbstractEventLoop)
+
+ self.assertIs(policy._local._loop, loop)
+ self.assertIs(loop, policy.get_event_loop())
+ loop.close()
+
+ def test_get_event_loop_calls_set_event_loop(self):
+ policy = asyncio.DefaultEventLoopPolicy()
+
+ with mock.patch.object(
+ policy, "set_event_loop",
+ wraps=policy.set_event_loop) as m_set_event_loop:
+
+ loop = policy.get_event_loop()
+
+ # policy._local._loop must be set through .set_event_loop()
+ # (the unix DefaultEventLoopPolicy needs this call to attach
+ # the child watcher correctly)
+ m_set_event_loop.assert_called_with(loop)
+
+ loop.close()
+
+ def test_get_event_loop_after_set_none(self):
+ policy = asyncio.DefaultEventLoopPolicy()
+ policy.set_event_loop(None)
+ self.assertRaises(AssertionError, policy.get_event_loop)
+
+ @mock.patch('asyncio.events.threading.current_thread')
+ def test_get_event_loop_thread(self, m_current_thread):
+
+ def f():
+ policy = asyncio.DefaultEventLoopPolicy()
+ self.assertRaises(AssertionError, policy.get_event_loop)
+
+ th = threading.Thread(target=f)
+ th.start()
+ th.join()
+
+ def test_new_event_loop(self):
+ policy = asyncio.DefaultEventLoopPolicy()
+
+ loop = policy.new_event_loop()
+ self.assertIsInstance(loop, asyncio.AbstractEventLoop)
+ loop.close()
+
+ def test_set_event_loop(self):
+ policy = asyncio.DefaultEventLoopPolicy()
+ old_loop = policy.get_event_loop()
+
+ self.assertRaises(AssertionError, policy.set_event_loop, object())
+
+ loop = policy.new_event_loop()
+ policy.set_event_loop(loop)
+ self.assertIs(loop, policy.get_event_loop())
+ self.assertIsNot(old_loop, policy.get_event_loop())
+ loop.close()
+ old_loop.close()
+
+ def test_get_event_loop_policy(self):
+ policy = asyncio.get_event_loop_policy()
+ self.assertIsInstance(policy, asyncio.AbstractEventLoopPolicy)
+ self.assertIs(policy, asyncio.get_event_loop_policy())
+
+ def test_set_event_loop_policy(self):
+ self.assertRaises(
+ AssertionError, asyncio.set_event_loop_policy, object())
+
+ old_policy = asyncio.get_event_loop_policy()
+
+ policy = asyncio.DefaultEventLoopPolicy()
+ asyncio.set_event_loop_policy(policy)
+ self.assertIs(policy, asyncio.get_event_loop_policy())
+ self.assertIsNot(policy, old_policy)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py
new file mode 100644
index 0000000..399e8f4
--- /dev/null
+++ b/Lib/test/test_asyncio/test_futures.py
@@ -0,0 +1,351 @@
+"""Tests for futures.py."""
+
+import concurrent.futures
+import threading
+import unittest
+from unittest import mock
+
+import asyncio
+from asyncio import test_utils
+
+
+def _fakefunc(f):
+ return f
+
+
+class FutureTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_initial_state(self):
+ f = asyncio.Future(loop=self.loop)
+ self.assertFalse(f.cancelled())
+ self.assertFalse(f.done())
+ f.cancel()
+ self.assertTrue(f.cancelled())
+
+ def test_init_constructor_default_loop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ f = asyncio.Future()
+ self.assertIs(f._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_constructor_positional(self):
+ # Make sure Future doesn't accept a positional argument
+ self.assertRaises(TypeError, asyncio.Future, 42)
+
+ def test_cancel(self):
+ f = asyncio.Future(loop=self.loop)
+ self.assertTrue(f.cancel())
+ self.assertTrue(f.cancelled())
+ self.assertTrue(f.done())
+ self.assertRaises(asyncio.CancelledError, f.result)
+ self.assertRaises(asyncio.CancelledError, f.exception)
+ self.assertRaises(asyncio.InvalidStateError, f.set_result, None)
+ self.assertRaises(asyncio.InvalidStateError, f.set_exception, None)
+ self.assertFalse(f.cancel())
+
+ def test_result(self):
+ f = asyncio.Future(loop=self.loop)
+ self.assertRaises(asyncio.InvalidStateError, f.result)
+
+ f.set_result(42)
+ self.assertFalse(f.cancelled())
+ self.assertTrue(f.done())
+ self.assertEqual(f.result(), 42)
+ self.assertEqual(f.exception(), None)
+ self.assertRaises(asyncio.InvalidStateError, f.set_result, None)
+ self.assertRaises(asyncio.InvalidStateError, f.set_exception, None)
+ self.assertFalse(f.cancel())
+
+ def test_exception(self):
+ exc = RuntimeError()
+ f = asyncio.Future(loop=self.loop)
+ self.assertRaises(asyncio.InvalidStateError, f.exception)
+
+ f.set_exception(exc)
+ self.assertFalse(f.cancelled())
+ self.assertTrue(f.done())
+ self.assertRaises(RuntimeError, f.result)
+ self.assertEqual(f.exception(), exc)
+ self.assertRaises(asyncio.InvalidStateError, f.set_result, None)
+ self.assertRaises(asyncio.InvalidStateError, f.set_exception, None)
+ self.assertFalse(f.cancel())
+
+ def test_exception_class(self):
+ f = asyncio.Future(loop=self.loop)
+ f.set_exception(RuntimeError)
+ self.assertIsInstance(f.exception(), RuntimeError)
+
+ def test_yield_from_twice(self):
+ f = asyncio.Future(loop=self.loop)
+
+ def fixture():
+ yield 'A'
+ x = yield from f
+ yield 'B', x
+ y = yield from f
+ yield 'C', y
+
+ g = fixture()
+ self.assertEqual(next(g), 'A') # yield 'A'.
+ self.assertEqual(next(g), f) # First yield from f.
+ f.set_result(42)
+ self.assertEqual(next(g), ('B', 42)) # yield 'B', x.
+ # The second "yield from f" does not yield f.
+ self.assertEqual(next(g), ('C', 42)) # yield 'C', y.
+
+ def test_repr(self):
+ f_pending = asyncio.Future(loop=self.loop)
+ self.assertEqual(repr(f_pending), 'Future<PENDING>')
+ f_pending.cancel()
+
+ f_cancelled = asyncio.Future(loop=self.loop)
+ f_cancelled.cancel()
+ self.assertEqual(repr(f_cancelled), 'Future<CANCELLED>')
+
+ f_result = asyncio.Future(loop=self.loop)
+ f_result.set_result(4)
+ self.assertEqual(repr(f_result), 'Future<result=4>')
+ self.assertEqual(f_result.result(), 4)
+
+ exc = RuntimeError()
+ f_exception = asyncio.Future(loop=self.loop)
+ f_exception.set_exception(exc)
+ self.assertEqual(repr(f_exception), 'Future<exception=RuntimeError()>')
+ self.assertIs(f_exception.exception(), exc)
+
+ f_few_callbacks = asyncio.Future(loop=self.loop)
+ f_few_callbacks.add_done_callback(_fakefunc)
+ self.assertIn('Future<PENDING, [<function _fakefunc',
+ repr(f_few_callbacks))
+ f_few_callbacks.cancel()
+
+ f_many_callbacks = asyncio.Future(loop=self.loop)
+ for i in range(20):
+ f_many_callbacks.add_done_callback(_fakefunc)
+ r = repr(f_many_callbacks)
+ self.assertIn('Future<PENDING, [<function _fakefunc', r)
+ self.assertIn('<18 more>', r)
+ f_many_callbacks.cancel()
+
+ def test_copy_state(self):
+ # Test the internal _copy_state method since it's being directly
+ # invoked in other modules.
+ f = asyncio.Future(loop=self.loop)
+ f.set_result(10)
+
+ newf = asyncio.Future(loop=self.loop)
+ newf._copy_state(f)
+ self.assertTrue(newf.done())
+ self.assertEqual(newf.result(), 10)
+
+ f_exception = asyncio.Future(loop=self.loop)
+ f_exception.set_exception(RuntimeError())
+
+ newf_exception = asyncio.Future(loop=self.loop)
+ newf_exception._copy_state(f_exception)
+ self.assertTrue(newf_exception.done())
+ self.assertRaises(RuntimeError, newf_exception.result)
+
+ f_cancelled = asyncio.Future(loop=self.loop)
+ f_cancelled.cancel()
+
+ newf_cancelled = asyncio.Future(loop=self.loop)
+ newf_cancelled._copy_state(f_cancelled)
+ self.assertTrue(newf_cancelled.cancelled())
+
+ def test_iter(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ def coro():
+ yield from fut
+
+ def test():
+ arg1, arg2 = coro()
+
+ self.assertRaises(AssertionError, test)
+ fut.cancel()
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_abandoned(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ del fut
+ self.assertFalse(m_log.error.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_result_unretrieved(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_result(42)
+ del fut
+ self.assertFalse(m_log.error.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_result_retrieved(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_result(42)
+ fut.result()
+ del fut
+ self.assertFalse(m_log.error.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_exception_unretrieved(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_exception(RuntimeError('boom'))
+ del fut
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(m_log.error.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_exception_retrieved(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_exception(RuntimeError('boom'))
+ fut.exception()
+ del fut
+ self.assertFalse(m_log.error.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_tb_logger_exception_result_retrieved(self, m_log):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_exception(RuntimeError('boom'))
+ self.assertRaises(RuntimeError, fut.result)
+ del fut
+ self.assertFalse(m_log.error.called)
+
+ def test_wrap_future(self):
+
+ def run(arg):
+ return (arg, threading.get_ident())
+ ex = concurrent.futures.ThreadPoolExecutor(1)
+ f1 = ex.submit(run, 'oi')
+ f2 = asyncio.wrap_future(f1, loop=self.loop)
+ res, ident = self.loop.run_until_complete(f2)
+ self.assertIsInstance(f2, asyncio.Future)
+ self.assertEqual(res, 'oi')
+ self.assertNotEqual(ident, threading.get_ident())
+
+ def test_wrap_future_future(self):
+ f1 = asyncio.Future(loop=self.loop)
+ f2 = asyncio.wrap_future(f1)
+ self.assertIs(f1, f2)
+
+ @mock.patch('asyncio.futures.events')
+ def test_wrap_future_use_global_loop(self, m_events):
+ def run(arg):
+ return (arg, threading.get_ident())
+ ex = concurrent.futures.ThreadPoolExecutor(1)
+ f1 = ex.submit(run, 'oi')
+ f2 = asyncio.wrap_future(f1)
+ self.assertIs(m_events.get_event_loop.return_value, f2._loop)
+
+ def test_wrap_future_cancel(self):
+ f1 = concurrent.futures.Future()
+ f2 = asyncio.wrap_future(f1, loop=self.loop)
+ f2.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(f1.cancelled())
+ self.assertTrue(f2.cancelled())
+
+ def test_wrap_future_cancel2(self):
+ f1 = concurrent.futures.Future()
+ f2 = asyncio.wrap_future(f1, loop=self.loop)
+ f1.set_result(42)
+ f2.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(f1.cancelled())
+ self.assertEqual(f1.result(), 42)
+ self.assertTrue(f2.cancelled())
+
+
+class FutureDoneCallbackTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def run_briefly(self):
+ test_utils.run_briefly(self.loop)
+
+ def _make_callback(self, bag, thing):
+ # Create a callback function that appends thing to bag.
+ def bag_appender(future):
+ bag.append(thing)
+ return bag_appender
+
+ def _new_future(self):
+ return asyncio.Future(loop=self.loop)
+
+ def test_callbacks_invoked_on_set_result(self):
+ bag = []
+ f = self._new_future()
+ f.add_done_callback(self._make_callback(bag, 42))
+ f.add_done_callback(self._make_callback(bag, 17))
+
+ self.assertEqual(bag, [])
+ f.set_result('foo')
+
+ self.run_briefly()
+
+ self.assertEqual(bag, [42, 17])
+ self.assertEqual(f.result(), 'foo')
+
+ def test_callbacks_invoked_on_set_exception(self):
+ bag = []
+ f = self._new_future()
+ f.add_done_callback(self._make_callback(bag, 100))
+
+ self.assertEqual(bag, [])
+ exc = RuntimeError()
+ f.set_exception(exc)
+
+ self.run_briefly()
+
+ self.assertEqual(bag, [100])
+ self.assertEqual(f.exception(), exc)
+
+ def test_remove_done_callback(self):
+ bag = []
+ f = self._new_future()
+ cb1 = self._make_callback(bag, 1)
+ cb2 = self._make_callback(bag, 2)
+ cb3 = self._make_callback(bag, 3)
+
+ # Add one cb1 and one cb2.
+ f.add_done_callback(cb1)
+ f.add_done_callback(cb2)
+
+ # One instance of cb2 removed. Now there's only one cb1.
+ self.assertEqual(f.remove_done_callback(cb2), 1)
+
+ # Never had any cb3 in there.
+ self.assertEqual(f.remove_done_callback(cb3), 0)
+
+ # After this there will be 6 instances of cb1 and one of cb2.
+ f.add_done_callback(cb2)
+ for i in range(5):
+ f.add_done_callback(cb1)
+
+ # Remove all instances of cb1. One cb2 remains.
+ self.assertEqual(f.remove_done_callback(cb1), 6)
+
+ self.assertEqual(bag, [])
+ f.set_result('foo')
+
+ self.run_briefly()
+
+ self.assertEqual(bag, [2])
+ self.assertEqual(f.result(), 'foo')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py
new file mode 100644
index 0000000..f542463
--- /dev/null
+++ b/Lib/test/test_asyncio/test_locks.py
@@ -0,0 +1,870 @@
+"""Tests for lock.py"""
+
+import unittest
+from unittest import mock
+import re
+
+import asyncio
+from asyncio import test_utils
+
+
+STR_RGX_REPR = (
+ r'^<(?P<class>.*?) object at (?P<address>.*?)'
+ r'\[(?P<extras>'
+ r'(set|unset|locked|unlocked)(,value:\d)?(,waiters:\d+)?'
+ r')\]>\Z'
+)
+RGX_REPR = re.compile(STR_RGX_REPR)
+
+
+class LockTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_ctor_loop(self):
+ loop = mock.Mock()
+ lock = asyncio.Lock(loop=loop)
+ self.assertIs(lock._loop, loop)
+
+ lock = asyncio.Lock(loop=self.loop)
+ self.assertIs(lock._loop, self.loop)
+
+ def test_ctor_noloop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ lock = asyncio.Lock()
+ self.assertIs(lock._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_repr(self):
+ lock = asyncio.Lock(loop=self.loop)
+ self.assertTrue(repr(lock).endswith('[unlocked]>'))
+ self.assertTrue(RGX_REPR.match(repr(lock)))
+
+ @asyncio.coroutine
+ def acquire_lock():
+ yield from lock
+
+ self.loop.run_until_complete(acquire_lock())
+ self.assertTrue(repr(lock).endswith('[locked]>'))
+ self.assertTrue(RGX_REPR.match(repr(lock)))
+
+ def test_lock(self):
+ lock = asyncio.Lock(loop=self.loop)
+
+ @asyncio.coroutine
+ def acquire_lock():
+ return (yield from lock)
+
+ res = self.loop.run_until_complete(acquire_lock())
+
+ self.assertTrue(res)
+ self.assertTrue(lock.locked())
+
+ lock.release()
+ self.assertFalse(lock.locked())
+
+ def test_acquire(self):
+ lock = asyncio.Lock(loop=self.loop)
+ result = []
+
+ self.assertTrue(self.loop.run_until_complete(lock.acquire()))
+
+ @asyncio.coroutine
+ def c1(result):
+ if (yield from lock.acquire()):
+ result.append(1)
+ return True
+
+ @asyncio.coroutine
+ def c2(result):
+ if (yield from lock.acquire()):
+ result.append(2)
+ return True
+
+ @asyncio.coroutine
+ def c3(result):
+ if (yield from lock.acquire()):
+ result.append(3)
+ return True
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ lock.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+
+ t3 = asyncio.Task(c3(result), loop=self.loop)
+
+ lock.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2], result)
+
+ lock.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2, 3], result)
+
+ self.assertTrue(t1.done())
+ self.assertTrue(t1.result())
+ self.assertTrue(t2.done())
+ self.assertTrue(t2.result())
+ self.assertTrue(t3.done())
+ self.assertTrue(t3.result())
+
+ def test_acquire_cancel(self):
+ lock = asyncio.Lock(loop=self.loop)
+ self.assertTrue(self.loop.run_until_complete(lock.acquire()))
+
+ task = asyncio.Task(lock.acquire(), loop=self.loop)
+ self.loop.call_soon(task.cancel)
+ self.assertRaises(
+ asyncio.CancelledError,
+ self.loop.run_until_complete, task)
+ self.assertFalse(lock._waiters)
+
+ def test_cancel_race(self):
+ # Several tasks:
+ # - A acquires the lock
+ # - B is blocked in aqcuire()
+ # - C is blocked in aqcuire()
+ #
+ # Now, concurrently:
+ # - B is cancelled
+ # - A releases the lock
+ #
+ # If B's waiter is marked cancelled but not yet removed from
+ # _waiters, A's release() call will crash when trying to set
+ # B's waiter; instead, it should move on to C's waiter.
+
+ # Setup: A has the lock, b and c are waiting.
+ lock = asyncio.Lock(loop=self.loop)
+
+ @asyncio.coroutine
+ def lockit(name, blocker):
+ yield from lock.acquire()
+ try:
+ if blocker is not None:
+ yield from blocker
+ finally:
+ lock.release()
+
+ fa = asyncio.Future(loop=self.loop)
+ ta = asyncio.Task(lockit('A', fa), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(lock.locked())
+ tb = asyncio.Task(lockit('B', None), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(len(lock._waiters), 1)
+ tc = asyncio.Task(lockit('C', None), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(len(lock._waiters), 2)
+
+ # Create the race and check.
+ # Without the fix this failed at the last assert.
+ fa.set_result(None)
+ tb.cancel()
+ self.assertTrue(lock._waiters[0].cancelled())
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(lock.locked())
+ self.assertTrue(ta.done())
+ self.assertTrue(tb.cancelled())
+ self.assertTrue(tc.done())
+
+ def test_release_not_acquired(self):
+ lock = asyncio.Lock(loop=self.loop)
+
+ self.assertRaises(RuntimeError, lock.release)
+
+ def test_release_no_waiters(self):
+ lock = asyncio.Lock(loop=self.loop)
+ self.loop.run_until_complete(lock.acquire())
+ self.assertTrue(lock.locked())
+
+ lock.release()
+ self.assertFalse(lock.locked())
+
+ def test_context_manager(self):
+ lock = asyncio.Lock(loop=self.loop)
+
+ @asyncio.coroutine
+ def acquire_lock():
+ return (yield from lock)
+
+ with self.loop.run_until_complete(acquire_lock()):
+ self.assertTrue(lock.locked())
+
+ self.assertFalse(lock.locked())
+
+ def test_context_manager_cant_reuse(self):
+ lock = asyncio.Lock(loop=self.loop)
+
+ @asyncio.coroutine
+ def acquire_lock():
+ return (yield from lock)
+
+ # This spells "yield from lock" outside a generator.
+ cm = self.loop.run_until_complete(acquire_lock())
+ with cm:
+ self.assertTrue(lock.locked())
+
+ self.assertFalse(lock.locked())
+
+ with self.assertRaises(AttributeError):
+ with cm:
+ pass
+
+ def test_context_manager_no_yield(self):
+ lock = asyncio.Lock(loop=self.loop)
+
+ try:
+ with lock:
+ self.fail('RuntimeError is not raised in with expression')
+ except RuntimeError as err:
+ self.assertEqual(
+ str(err),
+ '"yield from" should be used as context manager expression')
+
+ self.assertFalse(lock.locked())
+
+
+class EventTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_ctor_loop(self):
+ loop = mock.Mock()
+ ev = asyncio.Event(loop=loop)
+ self.assertIs(ev._loop, loop)
+
+ ev = asyncio.Event(loop=self.loop)
+ self.assertIs(ev._loop, self.loop)
+
+ def test_ctor_noloop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ ev = asyncio.Event()
+ self.assertIs(ev._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_repr(self):
+ ev = asyncio.Event(loop=self.loop)
+ self.assertTrue(repr(ev).endswith('[unset]>'))
+ match = RGX_REPR.match(repr(ev))
+ self.assertEqual(match.group('extras'), 'unset')
+
+ ev.set()
+ self.assertTrue(repr(ev).endswith('[set]>'))
+ self.assertTrue(RGX_REPR.match(repr(ev)))
+
+ ev._waiters.append(mock.Mock())
+ self.assertTrue('waiters:1' in repr(ev))
+ self.assertTrue(RGX_REPR.match(repr(ev)))
+
+ def test_wait(self):
+ ev = asyncio.Event(loop=self.loop)
+ self.assertFalse(ev.is_set())
+
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ if (yield from ev.wait()):
+ result.append(1)
+
+ @asyncio.coroutine
+ def c2(result):
+ if (yield from ev.wait()):
+ result.append(2)
+
+ @asyncio.coroutine
+ def c3(result):
+ if (yield from ev.wait()):
+ result.append(3)
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ t3 = asyncio.Task(c3(result), loop=self.loop)
+
+ ev.set()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([3, 1, 2], result)
+
+ self.assertTrue(t1.done())
+ self.assertIsNone(t1.result())
+ self.assertTrue(t2.done())
+ self.assertIsNone(t2.result())
+ self.assertTrue(t3.done())
+ self.assertIsNone(t3.result())
+
+ def test_wait_on_set(self):
+ ev = asyncio.Event(loop=self.loop)
+ ev.set()
+
+ res = self.loop.run_until_complete(ev.wait())
+ self.assertTrue(res)
+
+ def test_wait_cancel(self):
+ ev = asyncio.Event(loop=self.loop)
+
+ wait = asyncio.Task(ev.wait(), loop=self.loop)
+ self.loop.call_soon(wait.cancel)
+ self.assertRaises(
+ asyncio.CancelledError,
+ self.loop.run_until_complete, wait)
+ self.assertFalse(ev._waiters)
+
+ def test_clear(self):
+ ev = asyncio.Event(loop=self.loop)
+ self.assertFalse(ev.is_set())
+
+ ev.set()
+ self.assertTrue(ev.is_set())
+
+ ev.clear()
+ self.assertFalse(ev.is_set())
+
+ def test_clear_with_waiters(self):
+ ev = asyncio.Event(loop=self.loop)
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ if (yield from ev.wait()):
+ result.append(1)
+ return True
+
+ t = asyncio.Task(c1(result), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ ev.set()
+ ev.clear()
+ self.assertFalse(ev.is_set())
+
+ ev.set()
+ ev.set()
+ self.assertEqual(1, len(ev._waiters))
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+ self.assertEqual(0, len(ev._waiters))
+
+ self.assertTrue(t.done())
+ self.assertTrue(t.result())
+
+
+class ConditionTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_ctor_loop(self):
+ loop = mock.Mock()
+ cond = asyncio.Condition(loop=loop)
+ self.assertIs(cond._loop, loop)
+
+ cond = asyncio.Condition(loop=self.loop)
+ self.assertIs(cond._loop, self.loop)
+
+ def test_ctor_noloop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ cond = asyncio.Condition()
+ self.assertIs(cond._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_wait(self):
+ cond = asyncio.Condition(loop=self.loop)
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(1)
+ return True
+
+ @asyncio.coroutine
+ def c2(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(2)
+ return True
+
+ @asyncio.coroutine
+ def c3(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(3)
+ return True
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+ t3 = asyncio.Task(c3(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+ self.assertFalse(cond.locked())
+
+ self.assertTrue(self.loop.run_until_complete(cond.acquire()))
+ cond.notify()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+ self.assertTrue(cond.locked())
+
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+ self.assertTrue(cond.locked())
+
+ cond.notify(2)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+ self.assertTrue(cond.locked())
+
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2], result)
+ self.assertTrue(cond.locked())
+
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2, 3], result)
+ self.assertTrue(cond.locked())
+
+ self.assertTrue(t1.done())
+ self.assertTrue(t1.result())
+ self.assertTrue(t2.done())
+ self.assertTrue(t2.result())
+ self.assertTrue(t3.done())
+ self.assertTrue(t3.result())
+
+ def test_wait_cancel(self):
+ cond = asyncio.Condition(loop=self.loop)
+ self.loop.run_until_complete(cond.acquire())
+
+ wait = asyncio.Task(cond.wait(), loop=self.loop)
+ self.loop.call_soon(wait.cancel)
+ self.assertRaises(
+ asyncio.CancelledError,
+ self.loop.run_until_complete, wait)
+ self.assertFalse(cond._waiters)
+ self.assertTrue(cond.locked())
+
+ def test_wait_unacquired(self):
+ cond = asyncio.Condition(loop=self.loop)
+ self.assertRaises(
+ RuntimeError,
+ self.loop.run_until_complete, cond.wait())
+
+ def test_wait_for(self):
+ cond = asyncio.Condition(loop=self.loop)
+ presult = False
+
+ def predicate():
+ return presult
+
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ yield from cond.acquire()
+ if (yield from cond.wait_for(predicate)):
+ result.append(1)
+ cond.release()
+ return True
+
+ t = asyncio.Task(c1(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ self.loop.run_until_complete(cond.acquire())
+ cond.notify()
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ presult = True
+ self.loop.run_until_complete(cond.acquire())
+ cond.notify()
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+
+ self.assertTrue(t.done())
+ self.assertTrue(t.result())
+
+ def test_wait_for_unacquired(self):
+ cond = asyncio.Condition(loop=self.loop)
+
+ # predicate can return true immediately
+ res = self.loop.run_until_complete(cond.wait_for(lambda: [1, 2, 3]))
+ self.assertEqual([1, 2, 3], res)
+
+ self.assertRaises(
+ RuntimeError,
+ self.loop.run_until_complete,
+ cond.wait_for(lambda: False))
+
+ def test_notify(self):
+ cond = asyncio.Condition(loop=self.loop)
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(1)
+ cond.release()
+ return True
+
+ @asyncio.coroutine
+ def c2(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(2)
+ cond.release()
+ return True
+
+ @asyncio.coroutine
+ def c3(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(3)
+ cond.release()
+ return True
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+ t3 = asyncio.Task(c3(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ self.loop.run_until_complete(cond.acquire())
+ cond.notify(1)
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+
+ self.loop.run_until_complete(cond.acquire())
+ cond.notify(1)
+ cond.notify(2048)
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2, 3], result)
+
+ self.assertTrue(t1.done())
+ self.assertTrue(t1.result())
+ self.assertTrue(t2.done())
+ self.assertTrue(t2.result())
+ self.assertTrue(t3.done())
+ self.assertTrue(t3.result())
+
+ def test_notify_all(self):
+ cond = asyncio.Condition(loop=self.loop)
+
+ result = []
+
+ @asyncio.coroutine
+ def c1(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(1)
+ cond.release()
+ return True
+
+ @asyncio.coroutine
+ def c2(result):
+ yield from cond.acquire()
+ if (yield from cond.wait()):
+ result.append(2)
+ cond.release()
+ return True
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([], result)
+
+ self.loop.run_until_complete(cond.acquire())
+ cond.notify_all()
+ cond.release()
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1, 2], result)
+
+ self.assertTrue(t1.done())
+ self.assertTrue(t1.result())
+ self.assertTrue(t2.done())
+ self.assertTrue(t2.result())
+
+ def test_notify_unacquired(self):
+ cond = asyncio.Condition(loop=self.loop)
+ self.assertRaises(RuntimeError, cond.notify)
+
+ def test_notify_all_unacquired(self):
+ cond = asyncio.Condition(loop=self.loop)
+ self.assertRaises(RuntimeError, cond.notify_all)
+
+ def test_repr(self):
+ cond = asyncio.Condition(loop=self.loop)
+ self.assertTrue('unlocked' in repr(cond))
+ self.assertTrue(RGX_REPR.match(repr(cond)))
+
+ self.loop.run_until_complete(cond.acquire())
+ self.assertTrue('locked' in repr(cond))
+
+ cond._waiters.append(mock.Mock())
+ self.assertTrue('waiters:1' in repr(cond))
+ self.assertTrue(RGX_REPR.match(repr(cond)))
+
+ cond._waiters.append(mock.Mock())
+ self.assertTrue('waiters:2' in repr(cond))
+ self.assertTrue(RGX_REPR.match(repr(cond)))
+
+ def test_context_manager(self):
+ cond = asyncio.Condition(loop=self.loop)
+
+ @asyncio.coroutine
+ def acquire_cond():
+ return (yield from cond)
+
+ with self.loop.run_until_complete(acquire_cond()):
+ self.assertTrue(cond.locked())
+
+ self.assertFalse(cond.locked())
+
+ def test_context_manager_no_yield(self):
+ cond = asyncio.Condition(loop=self.loop)
+
+ try:
+ with cond:
+ self.fail('RuntimeError is not raised in with expression')
+ except RuntimeError as err:
+ self.assertEqual(
+ str(err),
+ '"yield from" should be used as context manager expression')
+
+ self.assertFalse(cond.locked())
+
+
+class SemaphoreTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_ctor_loop(self):
+ loop = mock.Mock()
+ sem = asyncio.Semaphore(loop=loop)
+ self.assertIs(sem._loop, loop)
+
+ sem = asyncio.Semaphore(loop=self.loop)
+ self.assertIs(sem._loop, self.loop)
+
+ def test_ctor_noloop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ sem = asyncio.Semaphore()
+ self.assertIs(sem._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_initial_value_zero(self):
+ sem = asyncio.Semaphore(0, loop=self.loop)
+ self.assertTrue(sem.locked())
+
+ def test_repr(self):
+ sem = asyncio.Semaphore(loop=self.loop)
+ self.assertTrue(repr(sem).endswith('[unlocked,value:1]>'))
+ self.assertTrue(RGX_REPR.match(repr(sem)))
+
+ self.loop.run_until_complete(sem.acquire())
+ self.assertTrue(repr(sem).endswith('[locked]>'))
+ self.assertTrue('waiters' not in repr(sem))
+ self.assertTrue(RGX_REPR.match(repr(sem)))
+
+ sem._waiters.append(mock.Mock())
+ self.assertTrue('waiters:1' in repr(sem))
+ self.assertTrue(RGX_REPR.match(repr(sem)))
+
+ sem._waiters.append(mock.Mock())
+ self.assertTrue('waiters:2' in repr(sem))
+ self.assertTrue(RGX_REPR.match(repr(sem)))
+
+ def test_semaphore(self):
+ sem = asyncio.Semaphore(loop=self.loop)
+ self.assertEqual(1, sem._value)
+
+ @asyncio.coroutine
+ def acquire_lock():
+ return (yield from sem)
+
+ res = self.loop.run_until_complete(acquire_lock())
+
+ self.assertTrue(res)
+ self.assertTrue(sem.locked())
+ self.assertEqual(0, sem._value)
+
+ sem.release()
+ self.assertFalse(sem.locked())
+ self.assertEqual(1, sem._value)
+
+ def test_semaphore_value(self):
+ self.assertRaises(ValueError, asyncio.Semaphore, -1)
+
+ def test_acquire(self):
+ sem = asyncio.Semaphore(3, loop=self.loop)
+ result = []
+
+ self.assertTrue(self.loop.run_until_complete(sem.acquire()))
+ self.assertTrue(self.loop.run_until_complete(sem.acquire()))
+ self.assertFalse(sem.locked())
+
+ @asyncio.coroutine
+ def c1(result):
+ yield from sem.acquire()
+ result.append(1)
+ return True
+
+ @asyncio.coroutine
+ def c2(result):
+ yield from sem.acquire()
+ result.append(2)
+ return True
+
+ @asyncio.coroutine
+ def c3(result):
+ yield from sem.acquire()
+ result.append(3)
+ return True
+
+ @asyncio.coroutine
+ def c4(result):
+ yield from sem.acquire()
+ result.append(4)
+ return True
+
+ t1 = asyncio.Task(c1(result), loop=self.loop)
+ t2 = asyncio.Task(c2(result), loop=self.loop)
+ t3 = asyncio.Task(c3(result), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual([1], result)
+ self.assertTrue(sem.locked())
+ self.assertEqual(2, len(sem._waiters))
+ self.assertEqual(0, sem._value)
+
+ t4 = asyncio.Task(c4(result), loop=self.loop)
+
+ sem.release()
+ sem.release()
+ self.assertEqual(2, sem._value)
+
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(0, sem._value)
+ self.assertEqual([1, 2, 3], result)
+ self.assertTrue(sem.locked())
+ self.assertEqual(1, len(sem._waiters))
+ self.assertEqual(0, sem._value)
+
+ self.assertTrue(t1.done())
+ self.assertTrue(t1.result())
+ self.assertTrue(t2.done())
+ self.assertTrue(t2.result())
+ self.assertTrue(t3.done())
+ self.assertTrue(t3.result())
+ self.assertFalse(t4.done())
+
+ # cleanup locked semaphore
+ sem.release()
+
+ def test_acquire_cancel(self):
+ sem = asyncio.Semaphore(loop=self.loop)
+ self.loop.run_until_complete(sem.acquire())
+
+ acquire = asyncio.Task(sem.acquire(), loop=self.loop)
+ self.loop.call_soon(acquire.cancel)
+ self.assertRaises(
+ asyncio.CancelledError,
+ self.loop.run_until_complete, acquire)
+ self.assertFalse(sem._waiters)
+
+ def test_release_not_acquired(self):
+ sem = asyncio.BoundedSemaphore(loop=self.loop)
+
+ self.assertRaises(ValueError, sem.release)
+
+ def test_release_no_waiters(self):
+ sem = asyncio.Semaphore(loop=self.loop)
+ self.loop.run_until_complete(sem.acquire())
+ self.assertTrue(sem.locked())
+
+ sem.release()
+ self.assertFalse(sem.locked())
+
+ def test_context_manager(self):
+ sem = asyncio.Semaphore(2, loop=self.loop)
+
+ @asyncio.coroutine
+ def acquire_lock():
+ return (yield from sem)
+
+ with self.loop.run_until_complete(acquire_lock()):
+ self.assertFalse(sem.locked())
+ self.assertEqual(1, sem._value)
+
+ with self.loop.run_until_complete(acquire_lock()):
+ self.assertTrue(sem.locked())
+
+ self.assertEqual(2, sem._value)
+
+ def test_context_manager_no_yield(self):
+ sem = asyncio.Semaphore(2, loop=self.loop)
+
+ try:
+ with sem:
+ self.fail('RuntimeError is not raised in with expression')
+ except RuntimeError as err:
+ self.assertEqual(
+ str(err),
+ '"yield from" should be used as context manager expression')
+
+ self.assertEqual(2, sem._value)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py
new file mode 100644
index 0000000..5bf24a4
--- /dev/null
+++ b/Lib/test/test_asyncio/test_proactor_events.py
@@ -0,0 +1,490 @@
+"""Tests for proactor_events.py"""
+
+import socket
+import unittest
+from unittest import mock
+
+import asyncio
+from asyncio.proactor_events import BaseProactorEventLoop
+from asyncio.proactor_events import _ProactorSocketTransport
+from asyncio.proactor_events import _ProactorWritePipeTransport
+from asyncio.proactor_events import _ProactorDuplexPipeTransport
+from asyncio import test_utils
+
+
+class ProactorSocketTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.proactor = mock.Mock()
+ self.loop._proactor = self.proactor
+ self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
+ self.sock = mock.Mock(socket.socket)
+
+ def test_ctor(self):
+ fut = asyncio.Future(loop=self.loop)
+ tr = _ProactorSocketTransport(
+ self.loop, self.sock, self.protocol, fut)
+ test_utils.run_briefly(self.loop)
+ self.assertIsNone(fut.result())
+ self.protocol.connection_made(tr)
+ self.proactor.recv.assert_called_with(self.sock, 4096)
+
+ def test_loop_reading(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._loop_reading()
+ self.loop._proactor.recv.assert_called_with(self.sock, 4096)
+ self.assertFalse(self.protocol.data_received.called)
+ self.assertFalse(self.protocol.eof_received.called)
+
+ def test_loop_reading_data(self):
+ res = asyncio.Future(loop=self.loop)
+ res.set_result(b'data')
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+
+ tr._read_fut = res
+ tr._loop_reading(res)
+ self.loop._proactor.recv.assert_called_with(self.sock, 4096)
+ self.protocol.data_received.assert_called_with(b'data')
+
+ def test_loop_reading_no_data(self):
+ res = asyncio.Future(loop=self.loop)
+ res.set_result(b'')
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+
+ self.assertRaises(AssertionError, tr._loop_reading, res)
+
+ tr.close = mock.Mock()
+ tr._read_fut = res
+ tr._loop_reading(res)
+ self.assertFalse(self.loop._proactor.recv.called)
+ self.assertTrue(self.protocol.eof_received.called)
+ self.assertTrue(tr.close.called)
+
+ def test_loop_reading_aborted(self):
+ err = self.loop._proactor.recv.side_effect = ConnectionAbortedError()
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._fatal_error = mock.Mock()
+ tr._loop_reading()
+ tr._fatal_error.assert_called_with(
+ err,
+ 'Fatal read error on pipe transport')
+
+ def test_loop_reading_aborted_closing(self):
+ self.loop._proactor.recv.side_effect = ConnectionAbortedError()
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._closing = True
+ tr._fatal_error = mock.Mock()
+ tr._loop_reading()
+ self.assertFalse(tr._fatal_error.called)
+
+ def test_loop_reading_aborted_is_fatal(self):
+ self.loop._proactor.recv.side_effect = ConnectionAbortedError()
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._closing = False
+ tr._fatal_error = mock.Mock()
+ tr._loop_reading()
+ self.assertTrue(tr._fatal_error.called)
+
+ def test_loop_reading_conn_reset_lost(self):
+ err = self.loop._proactor.recv.side_effect = ConnectionResetError()
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._closing = False
+ tr._fatal_error = mock.Mock()
+ tr._force_close = mock.Mock()
+ tr._loop_reading()
+ self.assertFalse(tr._fatal_error.called)
+ tr._force_close.assert_called_with(err)
+
+ def test_loop_reading_exception(self):
+ err = self.loop._proactor.recv.side_effect = (OSError())
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._fatal_error = mock.Mock()
+ tr._loop_reading()
+ tr._fatal_error.assert_called_with(
+ err,
+ 'Fatal read error on pipe transport')
+
+ def test_write(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._loop_writing = mock.Mock()
+ tr.write(b'data')
+ self.assertEqual(tr._buffer, None)
+ tr._loop_writing.assert_called_with(data=b'data')
+
+ def test_write_no_data(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr.write(b'')
+ self.assertFalse(tr._buffer)
+
+ def test_write_more(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._write_fut = mock.Mock()
+ tr._loop_writing = mock.Mock()
+ tr.write(b'data')
+ self.assertEqual(tr._buffer, b'data')
+ self.assertFalse(tr._loop_writing.called)
+
+ def test_loop_writing(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._buffer = bytearray(b'data')
+ tr._loop_writing()
+ self.loop._proactor.send.assert_called_with(self.sock, b'data')
+ self.loop._proactor.send.return_value.add_done_callback.\
+ assert_called_with(tr._loop_writing)
+
+ @mock.patch('asyncio.proactor_events.logger')
+ def test_loop_writing_err(self, m_log):
+ err = self.loop._proactor.send.side_effect = OSError()
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._fatal_error = mock.Mock()
+ tr._buffer = [b'da', b'ta']
+ tr._loop_writing()
+ tr._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on pipe transport')
+ tr._conn_lost = 1
+
+ tr.write(b'data')
+ tr.write(b'data')
+ tr.write(b'data')
+ tr.write(b'data')
+ tr.write(b'data')
+ self.assertEqual(tr._buffer, None)
+ m_log.warning.assert_called_with('socket.send() raised exception.')
+
+ def test_loop_writing_stop(self):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_result(b'data')
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._write_fut = fut
+ tr._loop_writing(fut)
+ self.assertIsNone(tr._write_fut)
+
+ def test_loop_writing_closing(self):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_result(1)
+
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._write_fut = fut
+ tr.close()
+ tr._loop_writing(fut)
+ self.assertIsNone(tr._write_fut)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_abort(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._force_close = mock.Mock()
+ tr.abort()
+ tr._force_close.assert_called_with(None)
+
+ def test_close(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr.close()
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+ self.assertTrue(tr._closing)
+ self.assertEqual(tr._conn_lost, 1)
+
+ self.protocol.connection_lost.reset_mock()
+ tr.close()
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+ def test_close_write_fut(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._write_fut = mock.Mock()
+ tr.close()
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+ def test_close_buffer(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._buffer = [b'data']
+ tr.close()
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_fatal_error(self, m_logging):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._force_close = mock.Mock()
+ tr._fatal_error(None)
+ self.assertTrue(tr._force_close.called)
+ self.assertTrue(m_logging.error.called)
+
+ def test_force_close(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._buffer = [b'data']
+ read_fut = tr._read_fut = mock.Mock()
+ write_fut = tr._write_fut = mock.Mock()
+ tr._force_close(None)
+
+ read_fut.cancel.assert_called_with()
+ write_fut.cancel.assert_called_with()
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+ self.assertEqual(None, tr._buffer)
+ self.assertEqual(tr._conn_lost, 1)
+
+ def test_force_close_idempotent(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._closing = True
+ tr._force_close(None)
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+ def test_fatal_error_2(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._buffer = [b'data']
+ tr._force_close(None)
+
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+ self.assertEqual(None, tr._buffer)
+
+ def test_call_connection_lost(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ tr._call_connection_lost(None)
+ self.assertTrue(self.protocol.connection_lost.called)
+ self.assertTrue(self.sock.close.called)
+
+ def test_write_eof(self):
+ tr = _ProactorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertTrue(tr.can_write_eof())
+ tr.write_eof()
+ self.sock.shutdown.assert_called_with(socket.SHUT_WR)
+ tr.write_eof()
+ self.assertEqual(self.sock.shutdown.call_count, 1)
+ tr.close()
+
+ def test_write_eof_buffer(self):
+ tr = _ProactorSocketTransport(self.loop, self.sock, self.protocol)
+ f = asyncio.Future(loop=self.loop)
+ tr._loop._proactor.send.return_value = f
+ tr.write(b'data')
+ tr.write_eof()
+ self.assertTrue(tr._eof_written)
+ self.assertFalse(self.sock.shutdown.called)
+ tr._loop._proactor.send.assert_called_with(self.sock, b'data')
+ f.set_result(4)
+ self.loop._run_once()
+ self.sock.shutdown.assert_called_with(socket.SHUT_WR)
+ tr.close()
+
+ def test_write_eof_write_pipe(self):
+ tr = _ProactorWritePipeTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertTrue(tr.can_write_eof())
+ tr.write_eof()
+ self.assertTrue(tr._closing)
+ self.loop._run_once()
+ self.assertTrue(self.sock.close.called)
+ tr.close()
+
+ def test_write_eof_buffer_write_pipe(self):
+ tr = _ProactorWritePipeTransport(self.loop, self.sock, self.protocol)
+ f = asyncio.Future(loop=self.loop)
+ tr._loop._proactor.send.return_value = f
+ tr.write(b'data')
+ tr.write_eof()
+ self.assertTrue(tr._closing)
+ self.assertFalse(self.sock.shutdown.called)
+ tr._loop._proactor.send.assert_called_with(self.sock, b'data')
+ f.set_result(4)
+ self.loop._run_once()
+ self.loop._run_once()
+ self.assertTrue(self.sock.close.called)
+ tr.close()
+
+ def test_write_eof_duplex_pipe(self):
+ tr = _ProactorDuplexPipeTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertFalse(tr.can_write_eof())
+ with self.assertRaises(NotImplementedError):
+ tr.write_eof()
+ tr.close()
+
+ def test_pause_resume_reading(self):
+ tr = _ProactorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ futures = []
+ for msg in [b'data1', b'data2', b'data3', b'data4', b'']:
+ f = asyncio.Future(loop=self.loop)
+ f.set_result(msg)
+ futures.append(f)
+ self.loop._proactor.recv.side_effect = futures
+ self.loop._run_once()
+ self.assertFalse(tr._paused)
+ self.loop._run_once()
+ self.protocol.data_received.assert_called_with(b'data1')
+ self.loop._run_once()
+ self.protocol.data_received.assert_called_with(b'data2')
+ tr.pause_reading()
+ self.assertTrue(tr._paused)
+ for i in range(10):
+ self.loop._run_once()
+ self.protocol.data_received.assert_called_with(b'data2')
+ tr.resume_reading()
+ self.assertFalse(tr._paused)
+ self.loop._run_once()
+ self.protocol.data_received.assert_called_with(b'data3')
+ self.loop._run_once()
+ self.protocol.data_received.assert_called_with(b'data4')
+ tr.close()
+
+
+class BaseProactorEventLoopTests(unittest.TestCase):
+
+ def setUp(self):
+ self.sock = mock.Mock(socket.socket)
+ self.proactor = mock.Mock()
+
+ self.ssock, self.csock = mock.Mock(), mock.Mock()
+
+ class EventLoop(BaseProactorEventLoop):
+ def _socketpair(s):
+ return (self.ssock, self.csock)
+
+ self.loop = EventLoop(self.proactor)
+
+ @mock.patch.object(BaseProactorEventLoop, 'call_soon')
+ @mock.patch.object(BaseProactorEventLoop, '_socketpair')
+ def test_ctor(self, socketpair, call_soon):
+ ssock, csock = socketpair.return_value = (
+ mock.Mock(), mock.Mock())
+ loop = BaseProactorEventLoop(self.proactor)
+ self.assertIs(loop._ssock, ssock)
+ self.assertIs(loop._csock, csock)
+ self.assertEqual(loop._internal_fds, 1)
+ call_soon.assert_called_with(loop._loop_self_reading)
+
+ def test_close_self_pipe(self):
+ self.loop._close_self_pipe()
+ self.assertEqual(self.loop._internal_fds, 0)
+ self.assertTrue(self.ssock.close.called)
+ self.assertTrue(self.csock.close.called)
+ self.assertIsNone(self.loop._ssock)
+ self.assertIsNone(self.loop._csock)
+
+ def test_close(self):
+ self.loop._close_self_pipe = mock.Mock()
+ self.loop.close()
+ self.assertTrue(self.loop._close_self_pipe.called)
+ self.assertTrue(self.proactor.close.called)
+ self.assertIsNone(self.loop._proactor)
+
+ self.loop._close_self_pipe.reset_mock()
+ self.loop.close()
+ self.assertFalse(self.loop._close_self_pipe.called)
+
+ def test_sock_recv(self):
+ self.loop.sock_recv(self.sock, 1024)
+ self.proactor.recv.assert_called_with(self.sock, 1024)
+
+ def test_sock_sendall(self):
+ self.loop.sock_sendall(self.sock, b'data')
+ self.proactor.send.assert_called_with(self.sock, b'data')
+
+ def test_sock_connect(self):
+ self.loop.sock_connect(self.sock, 123)
+ self.proactor.connect.assert_called_with(self.sock, 123)
+
+ def test_sock_accept(self):
+ self.loop.sock_accept(self.sock)
+ self.proactor.accept.assert_called_with(self.sock)
+
+ def test_socketpair(self):
+ self.assertRaises(
+ NotImplementedError, BaseProactorEventLoop, self.proactor)
+
+ def test_make_socket_transport(self):
+ tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol())
+ self.assertIsInstance(tr, _ProactorSocketTransport)
+
+ def test_loop_self_reading(self):
+ self.loop._loop_self_reading()
+ self.proactor.recv.assert_called_with(self.ssock, 4096)
+ self.proactor.recv.return_value.add_done_callback.assert_called_with(
+ self.loop._loop_self_reading)
+
+ def test_loop_self_reading_fut(self):
+ fut = mock.Mock()
+ self.loop._loop_self_reading(fut)
+ self.assertTrue(fut.result.called)
+ self.proactor.recv.assert_called_with(self.ssock, 4096)
+ self.proactor.recv.return_value.add_done_callback.assert_called_with(
+ self.loop._loop_self_reading)
+
+ def test_loop_self_reading_exception(self):
+ self.loop.close = mock.Mock()
+ self.proactor.recv.side_effect = OSError()
+ self.assertRaises(OSError, self.loop._loop_self_reading)
+ self.assertTrue(self.loop.close.called)
+
+ def test_write_to_self(self):
+ self.loop._write_to_self()
+ self.csock.send.assert_called_with(b'x')
+
+ def test_process_events(self):
+ self.loop._process_events([])
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_create_server(self, m_log):
+ pf = mock.Mock()
+ call_soon = self.loop.call_soon = mock.Mock()
+
+ self.loop._start_serving(pf, self.sock)
+ self.assertTrue(call_soon.called)
+
+ # callback
+ loop = call_soon.call_args[0][0]
+ loop()
+ self.proactor.accept.assert_called_with(self.sock)
+
+ # conn
+ fut = mock.Mock()
+ fut.result.return_value = (mock.Mock(), mock.Mock())
+
+ make_tr = self.loop._make_socket_transport = mock.Mock()
+ loop(fut)
+ self.assertTrue(fut.result.called)
+ self.assertTrue(make_tr.called)
+
+ # exception
+ fut.result.side_effect = OSError()
+ loop(fut)
+ self.assertTrue(self.sock.close.called)
+ self.assertTrue(m_log.error.called)
+
+ def test_create_server_cancel(self):
+ pf = mock.Mock()
+ call_soon = self.loop.call_soon = mock.Mock()
+
+ self.loop._start_serving(pf, self.sock)
+ loop = call_soon.call_args[0][0]
+
+ # cancelled
+ fut = asyncio.Future(loop=self.loop)
+ fut.cancel()
+ loop(fut)
+ self.assertTrue(self.sock.close.called)
+
+ def test_stop_serving(self):
+ sock = mock.Mock()
+ self.loop._stop_serving(sock)
+ self.assertTrue(sock.close.called)
+ self.proactor._stop_serving.assert_called_with(sock)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
new file mode 100644
index 0000000..f79fee2
--- /dev/null
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -0,0 +1,466 @@
+"""Tests for queues.py"""
+
+import unittest
+from unittest import mock
+
+import asyncio
+from asyncio import test_utils
+
+
+class _QueueTestBase(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+
+class QueueBasicTests(_QueueTestBase):
+
+ def _test_repr_or_str(self, fn, expect_id):
+ """Test Queue's repr or str.
+
+ fn is repr or str. expect_id is True if we expect the Queue's id to
+ appear in fn(Queue()).
+ """
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0.1
+ self.assertAlmostEqual(0.2, when)
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ q = asyncio.Queue(loop=loop)
+ self.assertTrue(fn(q).startswith('<Queue'), fn(q))
+ id_is_present = hex(id(q)) in fn(q)
+ self.assertEqual(expect_id, id_is_present)
+
+ @asyncio.coroutine
+ def add_getter():
+ q = asyncio.Queue(loop=loop)
+ # Start a task that waits to get.
+ asyncio.Task(q.get(), loop=loop)
+ # Let it start waiting.
+ yield from asyncio.sleep(0.1, loop=loop)
+ self.assertTrue('_getters[1]' in fn(q))
+ # resume q.get coroutine to finish generator
+ q.put_nowait(0)
+
+ loop.run_until_complete(add_getter())
+
+ @asyncio.coroutine
+ def add_putter():
+ q = asyncio.Queue(maxsize=1, loop=loop)
+ q.put_nowait(1)
+ # Start a task that waits to put.
+ asyncio.Task(q.put(2), loop=loop)
+ # Let it start waiting.
+ yield from asyncio.sleep(0.1, loop=loop)
+ self.assertTrue('_putters[1]' in fn(q))
+ # resume q.put coroutine to finish generator
+ q.get_nowait()
+
+ loop.run_until_complete(add_putter())
+
+ q = asyncio.Queue(loop=loop)
+ q.put_nowait(1)
+ self.assertTrue('_queue=[1]' in fn(q))
+
+ def test_ctor_loop(self):
+ loop = mock.Mock()
+ q = asyncio.Queue(loop=loop)
+ self.assertIs(q._loop, loop)
+
+ q = asyncio.Queue(loop=self.loop)
+ self.assertIs(q._loop, self.loop)
+
+ def test_ctor_noloop(self):
+ try:
+ asyncio.set_event_loop(self.loop)
+ q = asyncio.Queue()
+ self.assertIs(q._loop, self.loop)
+ finally:
+ asyncio.set_event_loop(None)
+
+ def test_repr(self):
+ self._test_repr_or_str(repr, True)
+
+ def test_str(self):
+ self._test_repr_or_str(str, False)
+
+ def test_empty(self):
+ q = asyncio.Queue(loop=self.loop)
+ self.assertTrue(q.empty())
+ q.put_nowait(1)
+ self.assertFalse(q.empty())
+ self.assertEqual(1, q.get_nowait())
+ self.assertTrue(q.empty())
+
+ def test_full(self):
+ q = asyncio.Queue(loop=self.loop)
+ self.assertFalse(q.full())
+
+ q = asyncio.Queue(maxsize=1, loop=self.loop)
+ q.put_nowait(1)
+ self.assertTrue(q.full())
+
+ def test_order(self):
+ q = asyncio.Queue(loop=self.loop)
+ for i in [1, 3, 2]:
+ q.put_nowait(i)
+
+ items = [q.get_nowait() for _ in range(3)]
+ self.assertEqual([1, 3, 2], items)
+
+ def test_maxsize(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.01, when)
+ when = yield 0.01
+ self.assertAlmostEqual(0.02, when)
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ q = asyncio.Queue(maxsize=2, loop=loop)
+ self.assertEqual(2, q.maxsize)
+ have_been_put = []
+
+ @asyncio.coroutine
+ def putter():
+ for i in range(3):
+ yield from q.put(i)
+ have_been_put.append(i)
+ return True
+
+ @asyncio.coroutine
+ def test():
+ t = asyncio.Task(putter(), loop=loop)
+ yield from asyncio.sleep(0.01, loop=loop)
+
+ # The putter is blocked after putting two items.
+ self.assertEqual([0, 1], have_been_put)
+ self.assertEqual(0, q.get_nowait())
+
+ # Let the putter resume and put last item.
+ yield from asyncio.sleep(0.01, loop=loop)
+ self.assertEqual([0, 1, 2], have_been_put)
+ self.assertEqual(1, q.get_nowait())
+ self.assertEqual(2, q.get_nowait())
+
+ self.assertTrue(t.done())
+ self.assertTrue(t.result())
+
+ loop.run_until_complete(test())
+ self.assertAlmostEqual(0.02, loop.time())
+
+
+class QueueGetTests(_QueueTestBase):
+
+ def test_blocking_get(self):
+ q = asyncio.Queue(loop=self.loop)
+ q.put_nowait(1)
+
+ @asyncio.coroutine
+ def queue_get():
+ return (yield from q.get())
+
+ res = self.loop.run_until_complete(queue_get())
+ self.assertEqual(1, res)
+
+ def test_get_with_putters(self):
+ q = asyncio.Queue(1, loop=self.loop)
+ q.put_nowait(1)
+
+ waiter = asyncio.Future(loop=self.loop)
+ q._putters.append((2, waiter))
+
+ res = self.loop.run_until_complete(q.get())
+ self.assertEqual(1, res)
+ self.assertTrue(waiter.done())
+ self.assertIsNone(waiter.result())
+
+ def test_blocking_get_wait(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.01, when)
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ q = asyncio.Queue(loop=loop)
+ started = asyncio.Event(loop=loop)
+ finished = False
+
+ @asyncio.coroutine
+ def queue_get():
+ nonlocal finished
+ started.set()
+ res = yield from q.get()
+ finished = True
+ return res
+
+ @asyncio.coroutine
+ def queue_put():
+ loop.call_later(0.01, q.put_nowait, 1)
+ queue_get_task = asyncio.Task(queue_get(), loop=loop)
+ yield from started.wait()
+ self.assertFalse(finished)
+ res = yield from queue_get_task
+ self.assertTrue(finished)
+ return res
+
+ res = loop.run_until_complete(queue_put())
+ self.assertEqual(1, res)
+ self.assertAlmostEqual(0.01, loop.time())
+
+ def test_nonblocking_get(self):
+ q = asyncio.Queue(loop=self.loop)
+ q.put_nowait(1)
+ self.assertEqual(1, q.get_nowait())
+
+ def test_nonblocking_get_exception(self):
+ q = asyncio.Queue(loop=self.loop)
+ self.assertRaises(asyncio.QueueEmpty, q.get_nowait)
+
+ def test_get_cancelled(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.01, when)
+ when = yield 0.01
+ self.assertAlmostEqual(0.061, when)
+ yield 0.05
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ q = asyncio.Queue(loop=loop)
+
+ @asyncio.coroutine
+ def queue_get():
+ return (yield from asyncio.wait_for(q.get(), 0.051, loop=loop))
+
+ @asyncio.coroutine
+ def test():
+ get_task = asyncio.Task(queue_get(), loop=loop)
+ yield from asyncio.sleep(0.01, loop=loop) # let the task start
+ q.put_nowait(1)
+ return (yield from get_task)
+
+ self.assertEqual(1, loop.run_until_complete(test()))
+ self.assertAlmostEqual(0.06, loop.time())
+
+ def test_get_cancelled_race(self):
+ q = asyncio.Queue(loop=self.loop)
+
+ t1 = asyncio.Task(q.get(), loop=self.loop)
+ t2 = asyncio.Task(q.get(), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ t1.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(t1.done())
+ q.put_nowait('a')
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(t2.result(), 'a')
+
+ def test_get_with_waiting_putters(self):
+ q = asyncio.Queue(loop=self.loop, maxsize=1)
+ asyncio.Task(q.put('a'), loop=self.loop)
+ asyncio.Task(q.put('b'), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(self.loop.run_until_complete(q.get()), 'a')
+ self.assertEqual(self.loop.run_until_complete(q.get()), 'b')
+
+
+class QueuePutTests(_QueueTestBase):
+
+ def test_blocking_put(self):
+ q = asyncio.Queue(loop=self.loop)
+
+ @asyncio.coroutine
+ def queue_put():
+ # No maxsize, won't block.
+ yield from q.put(1)
+
+ self.loop.run_until_complete(queue_put())
+
+ def test_blocking_put_wait(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.01, when)
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ q = asyncio.Queue(maxsize=1, loop=loop)
+ started = asyncio.Event(loop=loop)
+ finished = False
+
+ @asyncio.coroutine
+ def queue_put():
+ nonlocal finished
+ started.set()
+ yield from q.put(1)
+ yield from q.put(2)
+ finished = True
+
+ @asyncio.coroutine
+ def queue_get():
+ loop.call_later(0.01, q.get_nowait)
+ queue_put_task = asyncio.Task(queue_put(), loop=loop)
+ yield from started.wait()
+ self.assertFalse(finished)
+ yield from queue_put_task
+ self.assertTrue(finished)
+
+ loop.run_until_complete(queue_get())
+ self.assertAlmostEqual(0.01, loop.time())
+
+ def test_nonblocking_put(self):
+ q = asyncio.Queue(loop=self.loop)
+ q.put_nowait(1)
+ self.assertEqual(1, q.get_nowait())
+
+ def test_nonblocking_put_exception(self):
+ q = asyncio.Queue(maxsize=1, loop=self.loop)
+ q.put_nowait(1)
+ self.assertRaises(asyncio.QueueFull, q.put_nowait, 2)
+
+ def test_put_cancelled(self):
+ q = asyncio.Queue(loop=self.loop)
+
+ @asyncio.coroutine
+ def queue_put():
+ yield from q.put(1)
+ return True
+
+ @asyncio.coroutine
+ def test():
+ return (yield from q.get())
+
+ t = asyncio.Task(queue_put(), loop=self.loop)
+ self.assertEqual(1, self.loop.run_until_complete(test()))
+ self.assertTrue(t.done())
+ self.assertTrue(t.result())
+
+ def test_put_cancelled_race(self):
+ q = asyncio.Queue(loop=self.loop, maxsize=1)
+
+ asyncio.Task(q.put('a'), loop=self.loop)
+ asyncio.Task(q.put('c'), loop=self.loop)
+ t = asyncio.Task(q.put('b'), loop=self.loop)
+
+ test_utils.run_briefly(self.loop)
+ t.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(t.done())
+ self.assertEqual(q.get_nowait(), 'a')
+ self.assertEqual(q.get_nowait(), 'c')
+
+ def test_put_with_waiting_getters(self):
+ q = asyncio.Queue(loop=self.loop)
+ t = asyncio.Task(q.get(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.loop.run_until_complete(q.put('a'))
+ self.assertEqual(self.loop.run_until_complete(t), 'a')
+
+
+class LifoQueueTests(_QueueTestBase):
+
+ def test_order(self):
+ q = asyncio.LifoQueue(loop=self.loop)
+ for i in [1, 3, 2]:
+ q.put_nowait(i)
+
+ items = [q.get_nowait() for _ in range(3)]
+ self.assertEqual([2, 3, 1], items)
+
+
+class PriorityQueueTests(_QueueTestBase):
+
+ def test_order(self):
+ q = asyncio.PriorityQueue(loop=self.loop)
+ for i in [1, 3, 2]:
+ q.put_nowait(i)
+
+ items = [q.get_nowait() for _ in range(3)]
+ self.assertEqual([1, 2, 3], items)
+
+
+class JoinableQueueTests(_QueueTestBase):
+
+ def test_task_done_underflow(self):
+ q = asyncio.JoinableQueue(loop=self.loop)
+ self.assertRaises(ValueError, q.task_done)
+
+ def test_task_done(self):
+ q = asyncio.JoinableQueue(loop=self.loop)
+ for i in range(100):
+ q.put_nowait(i)
+
+ accumulator = 0
+
+ # Two workers get items from the queue and call task_done after each.
+ # Join the queue and assert all items have been processed.
+ running = True
+
+ @asyncio.coroutine
+ def worker():
+ nonlocal accumulator
+
+ while running:
+ item = yield from q.get()
+ accumulator += item
+ q.task_done()
+
+ @asyncio.coroutine
+ def test():
+ for _ in range(2):
+ asyncio.Task(worker(), loop=self.loop)
+
+ yield from q.join()
+
+ self.loop.run_until_complete(test())
+ self.assertEqual(sum(range(100)), accumulator)
+
+ # close running generators
+ running = False
+ for i in range(2):
+ q.put_nowait(0)
+
+ def test_join_empty_queue(self):
+ q = asyncio.JoinableQueue(loop=self.loop)
+
+ # Test that a queue join()s successfully, and before anything else
+ # (done twice for insurance).
+
+ @asyncio.coroutine
+ def join():
+ yield from q.join()
+ yield from q.join()
+
+ self.loop.run_until_complete(join())
+
+ def test_format(self):
+ q = asyncio.JoinableQueue(loop=self.loop)
+ self.assertEqual(q._format(), 'maxsize=0')
+
+ q._unfinished_tasks = 2
+ self.assertEqual(q._format(), 'maxsize=0 tasks=2')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py
new file mode 100644
index 0000000..964b2e8
--- /dev/null
+++ b/Lib/test/test_asyncio/test_selector_events.py
@@ -0,0 +1,1690 @@
+"""Tests for selector_events.py"""
+
+import errno
+import gc
+import pprint
+import socket
+import sys
+import unittest
+from unittest import mock
+try:
+ import ssl
+except ImportError:
+ ssl = None
+
+import asyncio
+from asyncio import selectors
+from asyncio import test_utils
+from asyncio.selector_events import BaseSelectorEventLoop
+from asyncio.selector_events import _SelectorTransport
+from asyncio.selector_events import _SelectorSslTransport
+from asyncio.selector_events import _SelectorSocketTransport
+from asyncio.selector_events import _SelectorDatagramTransport
+
+
+MOCK_ANY = mock.ANY
+
+
+class TestBaseSelectorEventLoop(BaseSelectorEventLoop):
+
+ def _make_self_pipe(self):
+ self._ssock = mock.Mock()
+ self._csock = mock.Mock()
+ self._internal_fds += 1
+
+
+def list_to_buffer(l=()):
+ return bytearray().join(l)
+
+
+class BaseSelectorEventLoopTests(unittest.TestCase):
+
+ def setUp(self):
+ selector = mock.Mock()
+ self.loop = TestBaseSelectorEventLoop(selector)
+
+ def test_make_socket_transport(self):
+ m = mock.Mock()
+ self.loop.add_reader = mock.Mock()
+ transport = self.loop._make_socket_transport(m, asyncio.Protocol())
+ self.assertIsInstance(transport, _SelectorSocketTransport)
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ def test_make_ssl_transport(self):
+ m = mock.Mock()
+ self.loop.add_reader = mock.Mock()
+ self.loop.add_writer = mock.Mock()
+ self.loop.remove_reader = mock.Mock()
+ self.loop.remove_writer = mock.Mock()
+ waiter = asyncio.Future(loop=self.loop)
+ transport = self.loop._make_ssl_transport(
+ m, asyncio.Protocol(), m, waiter)
+ self.assertIsInstance(transport, _SelectorSslTransport)
+
+ @mock.patch('asyncio.selector_events.ssl', None)
+ def test_make_ssl_transport_without_ssl_error(self):
+ m = mock.Mock()
+ self.loop.add_reader = mock.Mock()
+ self.loop.add_writer = mock.Mock()
+ self.loop.remove_reader = mock.Mock()
+ self.loop.remove_writer = mock.Mock()
+ with self.assertRaises(RuntimeError):
+ self.loop._make_ssl_transport(m, m, m, m)
+
+ def test_close(self):
+ ssock = self.loop._ssock
+ ssock.fileno.return_value = 7
+ csock = self.loop._csock
+ csock.fileno.return_value = 1
+ remove_reader = self.loop.remove_reader = mock.Mock()
+
+ self.loop._selector.close()
+ self.loop._selector = selector = mock.Mock()
+ self.loop.close()
+ self.assertIsNone(self.loop._selector)
+ self.assertIsNone(self.loop._csock)
+ self.assertIsNone(self.loop._ssock)
+ selector.close.assert_called_with()
+ ssock.close.assert_called_with()
+ csock.close.assert_called_with()
+ remove_reader.assert_called_with(7)
+
+ self.loop.close()
+ self.loop.close()
+
+ def test_close_no_selector(self):
+ ssock = self.loop._ssock
+ csock = self.loop._csock
+ remove_reader = self.loop.remove_reader = mock.Mock()
+
+ self.loop._selector.close()
+ self.loop._selector = None
+ self.loop.close()
+ self.assertIsNone(self.loop._selector)
+ self.assertFalse(ssock.close.called)
+ self.assertFalse(csock.close.called)
+ self.assertFalse(remove_reader.called)
+
+ def test_socketpair(self):
+ self.assertRaises(NotImplementedError, self.loop._socketpair)
+
+ def test_read_from_self_tryagain(self):
+ self.loop._ssock.recv.side_effect = BlockingIOError
+ self.assertIsNone(self.loop._read_from_self())
+
+ def test_read_from_self_exception(self):
+ self.loop._ssock.recv.side_effect = OSError
+ self.assertRaises(OSError, self.loop._read_from_self)
+
+ def test_write_to_self_tryagain(self):
+ self.loop._csock.send.side_effect = BlockingIOError
+ self.assertIsNone(self.loop._write_to_self())
+
+ def test_write_to_self_exception(self):
+ self.loop._csock.send.side_effect = OSError()
+ self.assertRaises(OSError, self.loop._write_to_self)
+
+ def test_sock_recv(self):
+ sock = mock.Mock()
+ self.loop._sock_recv = mock.Mock()
+
+ f = self.loop.sock_recv(sock, 1024)
+ self.assertIsInstance(f, asyncio.Future)
+ self.loop._sock_recv.assert_called_with(f, False, sock, 1024)
+
+ def test__sock_recv_canceled_fut(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop._sock_recv(f, False, sock, 1024)
+ self.assertFalse(sock.recv.called)
+
+ def test__sock_recv_unregister(self):
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop.remove_reader = mock.Mock()
+ self.loop._sock_recv(f, True, sock, 1024)
+ self.assertEqual((10,), self.loop.remove_reader.call_args[0])
+
+ def test__sock_recv_tryagain(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.recv.side_effect = BlockingIOError
+
+ self.loop.add_reader = mock.Mock()
+ self.loop._sock_recv(f, False, sock, 1024)
+ self.assertEqual((10, self.loop._sock_recv, f, True, sock, 1024),
+ self.loop.add_reader.call_args[0])
+
+ def test__sock_recv_exception(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ err = sock.recv.side_effect = OSError()
+
+ self.loop._sock_recv(f, False, sock, 1024)
+ self.assertIs(err, f.exception())
+
+ def test_sock_sendall(self):
+ sock = mock.Mock()
+ self.loop._sock_sendall = mock.Mock()
+
+ f = self.loop.sock_sendall(sock, b'data')
+ self.assertIsInstance(f, asyncio.Future)
+ self.assertEqual(
+ (f, False, sock, b'data'),
+ self.loop._sock_sendall.call_args[0])
+
+ def test_sock_sendall_nodata(self):
+ sock = mock.Mock()
+ self.loop._sock_sendall = mock.Mock()
+
+ f = self.loop.sock_sendall(sock, b'')
+ self.assertIsInstance(f, asyncio.Future)
+ self.assertTrue(f.done())
+ self.assertIsNone(f.result())
+ self.assertFalse(self.loop._sock_sendall.called)
+
+ def test__sock_sendall_canceled_fut(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertFalse(sock.send.called)
+
+ def test__sock_sendall_unregister(self):
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop.remove_writer = mock.Mock()
+ self.loop._sock_sendall(f, True, sock, b'data')
+ self.assertEqual((10,), self.loop.remove_writer.call_args[0])
+
+ def test__sock_sendall_tryagain(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.send.side_effect = BlockingIOError
+
+ self.loop.add_writer = mock.Mock()
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertEqual(
+ (10, self.loop._sock_sendall, f, True, sock, b'data'),
+ self.loop.add_writer.call_args[0])
+
+ def test__sock_sendall_interrupted(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.send.side_effect = InterruptedError
+
+ self.loop.add_writer = mock.Mock()
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertEqual(
+ (10, self.loop._sock_sendall, f, True, sock, b'data'),
+ self.loop.add_writer.call_args[0])
+
+ def test__sock_sendall_exception(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ err = sock.send.side_effect = OSError()
+
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertIs(f.exception(), err)
+
+ def test__sock_sendall(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ sock.fileno.return_value = 10
+ sock.send.return_value = 4
+
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertTrue(f.done())
+ self.assertIsNone(f.result())
+
+ def test__sock_sendall_partial(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ sock.fileno.return_value = 10
+ sock.send.return_value = 2
+
+ self.loop.add_writer = mock.Mock()
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertFalse(f.done())
+ self.assertEqual(
+ (10, self.loop._sock_sendall, f, True, sock, b'ta'),
+ self.loop.add_writer.call_args[0])
+
+ def test__sock_sendall_none(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ sock.fileno.return_value = 10
+ sock.send.return_value = 0
+
+ self.loop.add_writer = mock.Mock()
+ self.loop._sock_sendall(f, False, sock, b'data')
+ self.assertFalse(f.done())
+ self.assertEqual(
+ (10, self.loop._sock_sendall, f, True, sock, b'data'),
+ self.loop.add_writer.call_args[0])
+
+ def test_sock_connect(self):
+ sock = mock.Mock()
+ self.loop._sock_connect = mock.Mock()
+
+ f = self.loop.sock_connect(sock, ('127.0.0.1', 8080))
+ self.assertIsInstance(f, asyncio.Future)
+ self.assertEqual(
+ (f, False, sock, ('127.0.0.1', 8080)),
+ self.loop._sock_connect.call_args[0])
+
+ def test__sock_connect(self):
+ f = asyncio.Future(loop=self.loop)
+
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+
+ self.loop._sock_connect(f, False, sock, ('127.0.0.1', 8080))
+ self.assertTrue(f.done())
+ self.assertIsNone(f.result())
+ self.assertTrue(sock.connect.called)
+
+ def test__sock_connect_canceled_fut(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop._sock_connect(f, False, sock, ('127.0.0.1', 8080))
+ self.assertFalse(sock.connect.called)
+
+ def test__sock_connect_unregister(self):
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop.remove_writer = mock.Mock()
+ self.loop._sock_connect(f, True, sock, ('127.0.0.1', 8080))
+ self.assertEqual((10,), self.loop.remove_writer.call_args[0])
+
+ def test__sock_connect_tryagain(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.getsockopt.return_value = errno.EAGAIN
+
+ self.loop.add_writer = mock.Mock()
+ self.loop.remove_writer = mock.Mock()
+
+ self.loop._sock_connect(f, True, sock, ('127.0.0.1', 8080))
+ self.assertEqual(
+ (10, self.loop._sock_connect, f,
+ True, sock, ('127.0.0.1', 8080)),
+ self.loop.add_writer.call_args[0])
+
+ def test__sock_connect_exception(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.getsockopt.return_value = errno.ENOTCONN
+
+ self.loop.remove_writer = mock.Mock()
+ self.loop._sock_connect(f, True, sock, ('127.0.0.1', 8080))
+ self.assertIsInstance(f.exception(), OSError)
+
+ def test_sock_accept(self):
+ sock = mock.Mock()
+ self.loop._sock_accept = mock.Mock()
+
+ f = self.loop.sock_accept(sock)
+ self.assertIsInstance(f, asyncio.Future)
+ self.assertEqual(
+ (f, False, sock), self.loop._sock_accept.call_args[0])
+
+ def test__sock_accept(self):
+ f = asyncio.Future(loop=self.loop)
+
+ conn = mock.Mock()
+
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.accept.return_value = conn, ('127.0.0.1', 1000)
+
+ self.loop._sock_accept(f, False, sock)
+ self.assertTrue(f.done())
+ self.assertEqual((conn, ('127.0.0.1', 1000)), f.result())
+ self.assertEqual((False,), conn.setblocking.call_args[0])
+
+ def test__sock_accept_canceled_fut(self):
+ sock = mock.Mock()
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop._sock_accept(f, False, sock)
+ self.assertFalse(sock.accept.called)
+
+ def test__sock_accept_unregister(self):
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+
+ f = asyncio.Future(loop=self.loop)
+ f.cancel()
+
+ self.loop.remove_reader = mock.Mock()
+ self.loop._sock_accept(f, True, sock)
+ self.assertEqual((10,), self.loop.remove_reader.call_args[0])
+
+ def test__sock_accept_tryagain(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ sock.accept.side_effect = BlockingIOError
+
+ self.loop.add_reader = mock.Mock()
+ self.loop._sock_accept(f, False, sock)
+ self.assertEqual(
+ (10, self.loop._sock_accept, f, True, sock),
+ self.loop.add_reader.call_args[0])
+
+ def test__sock_accept_exception(self):
+ f = asyncio.Future(loop=self.loop)
+ sock = mock.Mock()
+ sock.fileno.return_value = 10
+ err = sock.accept.side_effect = OSError()
+
+ self.loop._sock_accept(f, False, sock)
+ self.assertIs(err, f.exception())
+
+ def test_add_reader(self):
+ self.loop._selector.get_key.side_effect = KeyError
+ cb = lambda: True
+ self.loop.add_reader(1, cb)
+
+ self.assertTrue(self.loop._selector.register.called)
+ fd, mask, (r, w) = self.loop._selector.register.call_args[0]
+ self.assertEqual(1, fd)
+ self.assertEqual(selectors.EVENT_READ, mask)
+ self.assertEqual(cb, r._callback)
+ self.assertIsNone(w)
+
+ def test_add_reader_existing(self):
+ reader = mock.Mock()
+ writer = mock.Mock()
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_WRITE, (reader, writer))
+ cb = lambda: True
+ self.loop.add_reader(1, cb)
+
+ self.assertTrue(reader.cancel.called)
+ self.assertFalse(self.loop._selector.register.called)
+ self.assertTrue(self.loop._selector.modify.called)
+ fd, mask, (r, w) = self.loop._selector.modify.call_args[0]
+ self.assertEqual(1, fd)
+ self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask)
+ self.assertEqual(cb, r._callback)
+ self.assertEqual(writer, w)
+
+ def test_add_reader_existing_writer(self):
+ writer = mock.Mock()
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_WRITE, (None, writer))
+ cb = lambda: True
+ self.loop.add_reader(1, cb)
+
+ self.assertFalse(self.loop._selector.register.called)
+ self.assertTrue(self.loop._selector.modify.called)
+ fd, mask, (r, w) = self.loop._selector.modify.call_args[0]
+ self.assertEqual(1, fd)
+ self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask)
+ self.assertEqual(cb, r._callback)
+ self.assertEqual(writer, w)
+
+ def test_remove_reader(self):
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ, (None, None))
+ self.assertFalse(self.loop.remove_reader(1))
+
+ self.assertTrue(self.loop._selector.unregister.called)
+
+ def test_remove_reader_read_write(self):
+ reader = mock.Mock()
+ writer = mock.Mock()
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE,
+ (reader, writer))
+ self.assertTrue(
+ self.loop.remove_reader(1))
+
+ self.assertFalse(self.loop._selector.unregister.called)
+ self.assertEqual(
+ (1, selectors.EVENT_WRITE, (None, writer)),
+ self.loop._selector.modify.call_args[0])
+
+ def test_remove_reader_unknown(self):
+ self.loop._selector.get_key.side_effect = KeyError
+ self.assertFalse(
+ self.loop.remove_reader(1))
+
+ def test_add_writer(self):
+ self.loop._selector.get_key.side_effect = KeyError
+ cb = lambda: True
+ self.loop.add_writer(1, cb)
+
+ self.assertTrue(self.loop._selector.register.called)
+ fd, mask, (r, w) = self.loop._selector.register.call_args[0]
+ self.assertEqual(1, fd)
+ self.assertEqual(selectors.EVENT_WRITE, mask)
+ self.assertIsNone(r)
+ self.assertEqual(cb, w._callback)
+
+ def test_add_writer_existing(self):
+ reader = mock.Mock()
+ writer = mock.Mock()
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ, (reader, writer))
+ cb = lambda: True
+ self.loop.add_writer(1, cb)
+
+ self.assertTrue(writer.cancel.called)
+ self.assertFalse(self.loop._selector.register.called)
+ self.assertTrue(self.loop._selector.modify.called)
+ fd, mask, (r, w) = self.loop._selector.modify.call_args[0]
+ self.assertEqual(1, fd)
+ self.assertEqual(selectors.EVENT_WRITE | selectors.EVENT_READ, mask)
+ self.assertEqual(reader, r)
+ self.assertEqual(cb, w._callback)
+
+ def test_remove_writer(self):
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_WRITE, (None, None))
+ self.assertFalse(self.loop.remove_writer(1))
+
+ self.assertTrue(self.loop._selector.unregister.called)
+
+ def test_remove_writer_read_write(self):
+ reader = mock.Mock()
+ writer = mock.Mock()
+ self.loop._selector.get_key.return_value = selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE,
+ (reader, writer))
+ self.assertTrue(
+ self.loop.remove_writer(1))
+
+ self.assertFalse(self.loop._selector.unregister.called)
+ self.assertEqual(
+ (1, selectors.EVENT_READ, (reader, None)),
+ self.loop._selector.modify.call_args[0])
+
+ def test_remove_writer_unknown(self):
+ self.loop._selector.get_key.side_effect = KeyError
+ self.assertFalse(
+ self.loop.remove_writer(1))
+
+ def test_process_events_read(self):
+ reader = mock.Mock()
+ reader._cancelled = False
+
+ self.loop._add_callback = mock.Mock()
+ self.loop._process_events(
+ [(selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ, (reader, None)),
+ selectors.EVENT_READ)])
+ self.assertTrue(self.loop._add_callback.called)
+ self.loop._add_callback.assert_called_with(reader)
+
+ def test_process_events_read_cancelled(self):
+ reader = mock.Mock()
+ reader.cancelled = True
+
+ self.loop.remove_reader = mock.Mock()
+ self.loop._process_events(
+ [(selectors.SelectorKey(
+ 1, 1, selectors.EVENT_READ, (reader, None)),
+ selectors.EVENT_READ)])
+ self.loop.remove_reader.assert_called_with(1)
+
+ def test_process_events_write(self):
+ writer = mock.Mock()
+ writer._cancelled = False
+
+ self.loop._add_callback = mock.Mock()
+ self.loop._process_events(
+ [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE,
+ (None, writer)),
+ selectors.EVENT_WRITE)])
+ self.loop._add_callback.assert_called_with(writer)
+
+ def test_process_events_write_cancelled(self):
+ writer = mock.Mock()
+ writer.cancelled = True
+ self.loop.remove_writer = mock.Mock()
+
+ self.loop._process_events(
+ [(selectors.SelectorKey(1, 1, selectors.EVENT_WRITE,
+ (None, writer)),
+ selectors.EVENT_WRITE)])
+ self.loop.remove_writer.assert_called_with(1)
+
+
+class SelectorTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
+ self.sock = mock.Mock(socket.socket)
+ self.sock.fileno.return_value = 7
+
+ def test_ctor(self):
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ self.assertIs(tr._loop, self.loop)
+ self.assertIs(tr._sock, self.sock)
+ self.assertIs(tr._sock_fd, 7)
+
+ def test_abort(self):
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr._force_close = mock.Mock()
+
+ tr.abort()
+ tr._force_close.assert_called_with(None)
+
+ def test_close(self):
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr.close()
+
+ self.assertTrue(tr._closing)
+ self.assertEqual(1, self.loop.remove_reader_count[7])
+ self.protocol.connection_lost(None)
+ self.assertEqual(tr._conn_lost, 1)
+
+ tr.close()
+ self.assertEqual(tr._conn_lost, 1)
+ self.assertEqual(1, self.loop.remove_reader_count[7])
+
+ def test_close_write_buffer(self):
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr._buffer.extend(b'data')
+ tr.close()
+
+ self.assertFalse(self.loop.readers)
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+ def test_force_close(self):
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr._buffer.extend(b'1')
+ self.loop.add_reader(7, mock.sentinel)
+ self.loop.add_writer(7, mock.sentinel)
+ tr._force_close(None)
+
+ self.assertTrue(tr._closing)
+ self.assertEqual(tr._buffer, list_to_buffer())
+ self.assertFalse(self.loop.readers)
+ self.assertFalse(self.loop.writers)
+
+ # second close should not remove reader
+ tr._force_close(None)
+ self.assertFalse(self.loop.readers)
+ self.assertEqual(1, self.loop.remove_reader_count[7])
+
+ @mock.patch('asyncio.log.logger.error')
+ def test_fatal_error(self, m_exc):
+ exc = OSError()
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr._force_close = mock.Mock()
+ tr._fatal_error(exc)
+
+ m_exc.assert_called_with(
+ test_utils.MockPattern(
+ 'Fatal error on transport\nprotocol:.*\ntransport:.*'),
+ exc_info=(OSError, MOCK_ANY, MOCK_ANY))
+
+ tr._force_close.assert_called_with(exc)
+
+ def test_connection_lost(self):
+ exc = OSError()
+ tr = _SelectorTransport(self.loop, self.sock, self.protocol, None)
+ tr._call_connection_lost(exc)
+
+ self.protocol.connection_lost.assert_called_with(exc)
+ self.sock.close.assert_called_with()
+ self.assertIsNone(tr._sock)
+
+ self.assertIsNone(tr._protocol)
+ self.assertEqual(2, sys.getrefcount(self.protocol),
+ pprint.pformat(gc.get_referrers(self.protocol)))
+ self.assertIsNone(tr._loop)
+ self.assertEqual(2, sys.getrefcount(self.loop),
+ pprint.pformat(gc.get_referrers(self.loop)))
+
+
+class SelectorSocketTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
+ self.sock = mock.Mock(socket.socket)
+ self.sock_fd = self.sock.fileno.return_value = 7
+
+ def test_ctor(self):
+ tr = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.loop.assert_reader(7, tr._read_ready)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_made.assert_called_with(tr)
+
+ def test_ctor_with_waiter(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol, fut)
+ test_utils.run_briefly(self.loop)
+ self.assertIsNone(fut.result())
+
+ def test_pause_resume_reading(self):
+ tr = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertFalse(tr._paused)
+ self.loop.assert_reader(7, tr._read_ready)
+ tr.pause_reading()
+ self.assertTrue(tr._paused)
+ self.assertFalse(7 in self.loop.readers)
+ tr.resume_reading()
+ self.assertFalse(tr._paused)
+ self.loop.assert_reader(7, tr._read_ready)
+
+ def test_read_ready(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+
+ self.sock.recv.return_value = b'data'
+ transport._read_ready()
+
+ self.protocol.data_received.assert_called_with(b'data')
+
+ def test_read_ready_eof(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.close = mock.Mock()
+
+ self.sock.recv.return_value = b''
+ transport._read_ready()
+
+ self.protocol.eof_received.assert_called_with()
+ transport.close.assert_called_with()
+
+ def test_read_ready_eof_keep_open(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.close = mock.Mock()
+
+ self.sock.recv.return_value = b''
+ self.protocol.eof_received.return_value = True
+ transport._read_ready()
+
+ self.protocol.eof_received.assert_called_with()
+ self.assertFalse(transport.close.called)
+
+ @mock.patch('logging.exception')
+ def test_read_ready_tryagain(self, m_exc):
+ self.sock.recv.side_effect = BlockingIOError
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+
+ @mock.patch('logging.exception')
+ def test_read_ready_tryagain_interrupted(self, m_exc):
+ self.sock.recv.side_effect = InterruptedError
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+
+ @mock.patch('logging.exception')
+ def test_read_ready_conn_reset(self, m_exc):
+ err = self.sock.recv.side_effect = ConnectionResetError()
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._force_close = mock.Mock()
+ transport._read_ready()
+ transport._force_close.assert_called_with(err)
+
+ @mock.patch('logging.exception')
+ def test_read_ready_err(self, m_exc):
+ err = self.sock.recv.side_effect = OSError()
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal read error on socket transport')
+
+ def test_write(self):
+ data = b'data'
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+ self.sock.send.assert_called_with(data)
+
+ def test_write_bytearray(self):
+ data = bytearray(b'data')
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+ self.sock.send.assert_called_with(data)
+ self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated.
+
+ def test_write_memoryview(self):
+ data = memoryview(b'data')
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+ self.sock.send.assert_called_with(data)
+
+ def test_write_no_data(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend(b'data')
+ transport.write(b'')
+ self.assertFalse(self.sock.send.called)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_buffer(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend(b'data1')
+ transport.write(b'data2')
+ self.assertFalse(self.sock.send.called)
+ self.assertEqual(list_to_buffer([b'data1', b'data2']),
+ transport._buffer)
+
+ def test_write_partial(self):
+ data = b'data'
+ self.sock.send.return_value = 2
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
+
+ def test_write_partial_bytearray(self):
+ data = bytearray(b'data')
+ self.sock.send.return_value = 2
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
+ self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated.
+
+ def test_write_partial_memoryview(self):
+ data = memoryview(b'data')
+ self.sock.send.return_value = 2
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
+
+ def test_write_partial_none(self):
+ data = b'data'
+ self.sock.send.return_value = 0
+ self.sock.fileno.return_value = 7
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_tryagain(self):
+ self.sock.send.side_effect = BlockingIOError
+
+ data = b'data'
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.write(data)
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ @mock.patch('asyncio.selector_events.logger')
+ def test_write_exception(self, m_log):
+ err = self.sock.send.side_effect = OSError()
+
+ data = b'data'
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport.write(data)
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on socket transport')
+ transport._conn_lost = 1
+
+ self.sock.reset_mock()
+ transport.write(data)
+ self.assertFalse(self.sock.send.called)
+ self.assertEqual(transport._conn_lost, 2)
+ transport.write(data)
+ transport.write(data)
+ transport.write(data)
+ transport.write(data)
+ m_log.warning.assert_called_with('socket.send() raised exception.')
+
+ def test_write_str(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertRaises(TypeError, transport.write, 'str')
+
+ def test_write_closing(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.close()
+ self.assertEqual(transport._conn_lost, 1)
+ transport.write(b'data')
+ self.assertEqual(transport._conn_lost, 2)
+
+ def test_write_ready(self):
+ data = b'data'
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend(data)
+ self.loop.add_writer(7, transport._write_ready)
+ transport._write_ready()
+ self.assertTrue(self.sock.send.called)
+ self.assertFalse(self.loop.writers)
+
+ def test_write_ready_closing(self):
+ data = b'data'
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._closing = True
+ transport._buffer.extend(data)
+ self.loop.add_writer(7, transport._write_ready)
+ transport._write_ready()
+ self.assertTrue(self.sock.send.called)
+ self.assertFalse(self.loop.writers)
+ self.sock.close.assert_called_with()
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_write_ready_no_data(self):
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ # This is an internal error.
+ self.assertRaises(AssertionError, transport._write_ready)
+
+ def test_write_ready_partial(self):
+ data = b'data'
+ self.sock.send.return_value = 2
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend(data)
+ self.loop.add_writer(7, transport._write_ready)
+ transport._write_ready()
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'ta']), transport._buffer)
+
+ def test_write_ready_partial_none(self):
+ data = b'data'
+ self.sock.send.return_value = 0
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend(data)
+ self.loop.add_writer(7, transport._write_ready)
+ transport._write_ready()
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_ready_tryagain(self):
+ self.sock.send.side_effect = BlockingIOError
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer = list_to_buffer([b'data1', b'data2'])
+ self.loop.add_writer(7, transport._write_ready)
+ transport._write_ready()
+
+ self.loop.assert_writer(7, transport._write_ready)
+ self.assertEqual(list_to_buffer([b'data1data2']), transport._buffer)
+
+ def test_write_ready_exception(self):
+ err = self.sock.send.side_effect = OSError()
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._buffer.extend(b'data')
+ transport._write_ready()
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on socket transport')
+
+ @mock.patch('asyncio.base_events.logger')
+ def test_write_ready_exception_and_close(self, m_log):
+ self.sock.send.side_effect = OSError()
+ remove_writer = self.loop.remove_writer = mock.Mock()
+
+ transport = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ transport.close()
+ transport._buffer.extend(b'data')
+ transport._write_ready()
+ remove_writer.assert_called_with(self.sock_fd)
+
+ def test_write_eof(self):
+ tr = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertTrue(tr.can_write_eof())
+ tr.write_eof()
+ self.sock.shutdown.assert_called_with(socket.SHUT_WR)
+ tr.write_eof()
+ self.assertEqual(self.sock.shutdown.call_count, 1)
+ tr.close()
+
+ def test_write_eof_buffer(self):
+ tr = _SelectorSocketTransport(
+ self.loop, self.sock, self.protocol)
+ self.sock.send.side_effect = BlockingIOError
+ tr.write(b'data')
+ tr.write_eof()
+ self.assertEqual(tr._buffer, list_to_buffer([b'data']))
+ self.assertTrue(tr._eof)
+ self.assertFalse(self.sock.shutdown.called)
+ self.sock.send.side_effect = lambda _: 4
+ tr._write_ready()
+ self.assertTrue(self.sock.send.called)
+ self.sock.shutdown.assert_called_with(socket.SHUT_WR)
+ tr.close()
+
+
+@unittest.skipIf(ssl is None, 'No ssl module')
+class SelectorSslTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
+ self.sock = mock.Mock(socket.socket)
+ self.sock.fileno.return_value = 7
+ self.sslsock = mock.Mock()
+ self.sslsock.fileno.return_value = 1
+ self.sslcontext = mock.Mock()
+ self.sslcontext.wrap_socket.return_value = self.sslsock
+
+ def _make_one(self, create_waiter=None):
+ transport = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext)
+ self.sock.reset_mock()
+ self.sslsock.reset_mock()
+ self.sslcontext.reset_mock()
+ self.loop.reset_counters()
+ return transport
+
+ def test_on_handshake(self):
+ waiter = asyncio.Future(loop=self.loop)
+ tr = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext,
+ waiter=waiter)
+ self.assertTrue(self.sslsock.do_handshake.called)
+ self.loop.assert_reader(1, tr._read_ready)
+ test_utils.run_briefly(self.loop)
+ self.assertIsNone(waiter.result())
+
+ def test_on_handshake_reader_retry(self):
+ self.sslsock.do_handshake.side_effect = ssl.SSLWantReadError
+ transport = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext)
+ transport._on_handshake()
+ self.loop.assert_reader(1, transport._on_handshake)
+
+ def test_on_handshake_writer_retry(self):
+ self.sslsock.do_handshake.side_effect = ssl.SSLWantWriteError
+ transport = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext)
+ transport._on_handshake()
+ self.loop.assert_writer(1, transport._on_handshake)
+
+ def test_on_handshake_exc(self):
+ exc = ValueError()
+ self.sslsock.do_handshake.side_effect = exc
+ transport = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext)
+ transport._waiter = asyncio.Future(loop=self.loop)
+ transport._on_handshake()
+ self.assertTrue(self.sslsock.close.called)
+ self.assertTrue(transport._waiter.done())
+ self.assertIs(exc, transport._waiter.exception())
+
+ def test_on_handshake_base_exc(self):
+ transport = _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext)
+ transport._waiter = asyncio.Future(loop=self.loop)
+ exc = BaseException()
+ self.sslsock.do_handshake.side_effect = exc
+ self.assertRaises(BaseException, transport._on_handshake)
+ self.assertTrue(self.sslsock.close.called)
+ self.assertTrue(transport._waiter.done())
+ self.assertIs(exc, transport._waiter.exception())
+
+ def test_pause_resume_reading(self):
+ tr = self._make_one()
+ self.assertFalse(tr._paused)
+ self.loop.assert_reader(1, tr._read_ready)
+ tr.pause_reading()
+ self.assertTrue(tr._paused)
+ self.assertFalse(1 in self.loop.readers)
+ tr.resume_reading()
+ self.assertFalse(tr._paused)
+ self.loop.assert_reader(1, tr._read_ready)
+
+ def test_write(self):
+ transport = self._make_one()
+ transport.write(b'data')
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_bytearray(self):
+ transport = self._make_one()
+ data = bytearray(b'data')
+ transport.write(data)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+ self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated.
+ self.assertIsNot(data, transport._buffer) # Hasn't been incorporated.
+
+ def test_write_memoryview(self):
+ transport = self._make_one()
+ data = memoryview(b'data')
+ transport.write(data)
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_no_data(self):
+ transport = self._make_one()
+ transport._buffer.extend(b'data')
+ transport.write(b'')
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_str(self):
+ transport = self._make_one()
+ self.assertRaises(TypeError, transport.write, 'str')
+
+ def test_write_closing(self):
+ transport = self._make_one()
+ transport.close()
+ self.assertEqual(transport._conn_lost, 1)
+ transport.write(b'data')
+ self.assertEqual(transport._conn_lost, 2)
+
+ @mock.patch('asyncio.selector_events.logger')
+ def test_write_exception(self, m_log):
+ transport = self._make_one()
+ transport._conn_lost = 1
+ transport.write(b'data')
+ self.assertEqual(transport._buffer, list_to_buffer())
+ transport.write(b'data')
+ transport.write(b'data')
+ transport.write(b'data')
+ transport.write(b'data')
+ m_log.warning.assert_called_with('socket.send() raised exception.')
+
+ def test_read_ready_recv(self):
+ self.sslsock.recv.return_value = b'data'
+ transport = self._make_one()
+ transport._read_ready()
+ self.assertTrue(self.sslsock.recv.called)
+ self.assertEqual((b'data',), self.protocol.data_received.call_args[0])
+
+ def test_read_ready_write_wants_read(self):
+ self.loop.add_writer = mock.Mock()
+ self.sslsock.recv.side_effect = BlockingIOError
+ transport = self._make_one()
+ transport._write_wants_read = True
+ transport._write_ready = mock.Mock()
+ transport._buffer.extend(b'data')
+ transport._read_ready()
+
+ self.assertFalse(transport._write_wants_read)
+ transport._write_ready.assert_called_with()
+ self.loop.add_writer.assert_called_with(
+ transport._sock_fd, transport._write_ready)
+
+ def test_read_ready_recv_eof(self):
+ self.sslsock.recv.return_value = b''
+ transport = self._make_one()
+ transport.close = mock.Mock()
+ transport._read_ready()
+ transport.close.assert_called_with()
+ self.protocol.eof_received.assert_called_with()
+
+ def test_read_ready_recv_conn_reset(self):
+ err = self.sslsock.recv.side_effect = ConnectionResetError()
+ transport = self._make_one()
+ transport._force_close = mock.Mock()
+ transport._read_ready()
+ transport._force_close.assert_called_with(err)
+
+ def test_read_ready_recv_retry(self):
+ self.sslsock.recv.side_effect = ssl.SSLWantReadError
+ transport = self._make_one()
+ transport._read_ready()
+ self.assertTrue(self.sslsock.recv.called)
+ self.assertFalse(self.protocol.data_received.called)
+
+ self.sslsock.recv.side_effect = BlockingIOError
+ transport._read_ready()
+ self.assertFalse(self.protocol.data_received.called)
+
+ self.sslsock.recv.side_effect = InterruptedError
+ transport._read_ready()
+ self.assertFalse(self.protocol.data_received.called)
+
+ def test_read_ready_recv_write(self):
+ self.loop.remove_reader = mock.Mock()
+ self.loop.add_writer = mock.Mock()
+ self.sslsock.recv.side_effect = ssl.SSLWantWriteError
+ transport = self._make_one()
+ transport._read_ready()
+ self.assertFalse(self.protocol.data_received.called)
+ self.assertTrue(transport._read_wants_write)
+
+ self.loop.remove_reader.assert_called_with(transport._sock_fd)
+ self.loop.add_writer.assert_called_with(
+ transport._sock_fd, transport._write_ready)
+
+ def test_read_ready_recv_exc(self):
+ err = self.sslsock.recv.side_effect = OSError()
+ transport = self._make_one()
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal read error on SSL transport')
+
+ def test_write_ready_send(self):
+ self.sslsock.send.return_value = 4
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data'])
+ transport._write_ready()
+ self.assertEqual(list_to_buffer(), transport._buffer)
+ self.assertTrue(self.sslsock.send.called)
+
+ def test_write_ready_send_none(self):
+ self.sslsock.send.return_value = 0
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data1', b'data2'])
+ transport._write_ready()
+ self.assertTrue(self.sslsock.send.called)
+ self.assertEqual(list_to_buffer([b'data1data2']), transport._buffer)
+
+ def test_write_ready_send_partial(self):
+ self.sslsock.send.return_value = 2
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data1', b'data2'])
+ transport._write_ready()
+ self.assertTrue(self.sslsock.send.called)
+ self.assertEqual(list_to_buffer([b'ta1data2']), transport._buffer)
+
+ def test_write_ready_send_closing_partial(self):
+ self.sslsock.send.return_value = 2
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data1', b'data2'])
+ transport._write_ready()
+ self.assertTrue(self.sslsock.send.called)
+ self.assertFalse(self.sslsock.close.called)
+
+ def test_write_ready_send_closing(self):
+ self.sslsock.send.return_value = 4
+ transport = self._make_one()
+ transport.close()
+ transport._buffer = list_to_buffer([b'data'])
+ transport._write_ready()
+ self.assertFalse(self.loop.writers)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_write_ready_send_closing_empty_buffer(self):
+ self.sslsock.send.return_value = 4
+ transport = self._make_one()
+ transport.close()
+ transport._buffer = list_to_buffer()
+ transport._write_ready()
+ self.assertFalse(self.loop.writers)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_write_ready_send_retry(self):
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data'])
+
+ self.sslsock.send.side_effect = ssl.SSLWantWriteError
+ transport._write_ready()
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ self.sslsock.send.side_effect = BlockingIOError()
+ transport._write_ready()
+ self.assertEqual(list_to_buffer([b'data']), transport._buffer)
+
+ def test_write_ready_send_read(self):
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data'])
+
+ self.loop.remove_writer = mock.Mock()
+ self.sslsock.send.side_effect = ssl.SSLWantReadError
+ transport._write_ready()
+ self.assertFalse(self.protocol.data_received.called)
+ self.assertTrue(transport._write_wants_read)
+ self.loop.remove_writer.assert_called_with(transport._sock_fd)
+
+ def test_write_ready_send_exc(self):
+ err = self.sslsock.send.side_effect = OSError()
+
+ transport = self._make_one()
+ transport._buffer = list_to_buffer([b'data'])
+ transport._fatal_error = mock.Mock()
+ transport._write_ready()
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on SSL transport')
+ self.assertEqual(list_to_buffer(), transport._buffer)
+
+ def test_write_ready_read_wants_write(self):
+ self.loop.add_reader = mock.Mock()
+ self.sslsock.send.side_effect = BlockingIOError
+ transport = self._make_one()
+ transport._read_wants_write = True
+ transport._read_ready = mock.Mock()
+ transport._write_ready()
+
+ self.assertFalse(transport._read_wants_write)
+ transport._read_ready.assert_called_with()
+ self.loop.add_reader.assert_called_with(
+ transport._sock_fd, transport._read_ready)
+
+ def test_write_eof(self):
+ tr = self._make_one()
+ self.assertFalse(tr.can_write_eof())
+ self.assertRaises(NotImplementedError, tr.write_eof)
+
+ def test_close(self):
+ tr = self._make_one()
+ tr.close()
+
+ self.assertTrue(tr._closing)
+ self.assertEqual(1, self.loop.remove_reader_count[1])
+ self.assertEqual(tr._conn_lost, 1)
+
+ tr.close()
+ self.assertEqual(tr._conn_lost, 1)
+ self.assertEqual(1, self.loop.remove_reader_count[1])
+
+ @unittest.skipIf(ssl is None or not ssl.HAS_SNI, 'No SNI support')
+ def test_server_hostname(self):
+ _SelectorSslTransport(
+ self.loop, self.sock, self.protocol, self.sslcontext,
+ server_hostname='localhost')
+ self.sslcontext.wrap_socket.assert_called_with(
+ self.sock, do_handshake_on_connect=False, server_side=False,
+ server_hostname='localhost')
+
+
+class SelectorSslWithoutSslTransportTests(unittest.TestCase):
+
+ @mock.patch('asyncio.selector_events.ssl', None)
+ def test_ssl_transport_requires_ssl_module(self):
+ Mock = mock.Mock
+ with self.assertRaises(RuntimeError):
+ _SelectorSslTransport(Mock(), Mock(), Mock(), Mock())
+
+
+class SelectorDatagramTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.DatagramProtocol)
+ self.sock = mock.Mock(spec_set=socket.socket)
+ self.sock.fileno.return_value = 7
+
+ def test_read_ready(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+
+ self.sock.recvfrom.return_value = (b'data', ('0.0.0.0', 1234))
+ transport._read_ready()
+
+ self.protocol.datagram_received.assert_called_with(
+ b'data', ('0.0.0.0', 1234))
+
+ def test_read_ready_tryagain(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+
+ self.sock.recvfrom.side_effect = BlockingIOError
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+
+ def test_read_ready_err(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+
+ err = self.sock.recvfrom.side_effect = RuntimeError()
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal read error on datagram transport')
+
+ def test_read_ready_oserr(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+
+ err = self.sock.recvfrom.side_effect = OSError()
+ transport._fatal_error = mock.Mock()
+ transport._read_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+ self.protocol.error_received.assert_called_with(err)
+
+ def test_sendto(self):
+ data = b'data'
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport.sendto(data, ('0.0.0.0', 1234))
+ self.assertTrue(self.sock.sendto.called)
+ self.assertEqual(
+ self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234)))
+
+ def test_sendto_bytearray(self):
+ data = bytearray(b'data')
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport.sendto(data, ('0.0.0.0', 1234))
+ self.assertTrue(self.sock.sendto.called)
+ self.assertEqual(
+ self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234)))
+
+ def test_sendto_memoryview(self):
+ data = memoryview(b'data')
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport.sendto(data, ('0.0.0.0', 1234))
+ self.assertTrue(self.sock.sendto.called)
+ self.assertEqual(
+ self.sock.sendto.call_args[0], (data, ('0.0.0.0', 1234)))
+
+ def test_sendto_no_data(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.append((b'data', ('0.0.0.0', 12345)))
+ transport.sendto(b'', ())
+ self.assertFalse(self.sock.sendto.called)
+ self.assertEqual(
+ [(b'data', ('0.0.0.0', 12345))], list(transport._buffer))
+
+ def test_sendto_buffer(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.append((b'data1', ('0.0.0.0', 12345)))
+ transport.sendto(b'data2', ('0.0.0.0', 12345))
+ self.assertFalse(self.sock.sendto.called)
+ self.assertEqual(
+ [(b'data1', ('0.0.0.0', 12345)),
+ (b'data2', ('0.0.0.0', 12345))],
+ list(transport._buffer))
+
+ def test_sendto_buffer_bytearray(self):
+ data2 = bytearray(b'data2')
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.append((b'data1', ('0.0.0.0', 12345)))
+ transport.sendto(data2, ('0.0.0.0', 12345))
+ self.assertFalse(self.sock.sendto.called)
+ self.assertEqual(
+ [(b'data1', ('0.0.0.0', 12345)),
+ (b'data2', ('0.0.0.0', 12345))],
+ list(transport._buffer))
+ self.assertIsInstance(transport._buffer[1][0], bytes)
+
+ def test_sendto_buffer_memoryview(self):
+ data2 = memoryview(b'data2')
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.append((b'data1', ('0.0.0.0', 12345)))
+ transport.sendto(data2, ('0.0.0.0', 12345))
+ self.assertFalse(self.sock.sendto.called)
+ self.assertEqual(
+ [(b'data1', ('0.0.0.0', 12345)),
+ (b'data2', ('0.0.0.0', 12345))],
+ list(transport._buffer))
+ self.assertIsInstance(transport._buffer[1][0], bytes)
+
+ def test_sendto_tryagain(self):
+ data = b'data'
+
+ self.sock.sendto.side_effect = BlockingIOError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport.sendto(data, ('0.0.0.0', 12345))
+
+ self.loop.assert_writer(7, transport._sendto_ready)
+ self.assertEqual(
+ [(b'data', ('0.0.0.0', 12345))], list(transport._buffer))
+
+ @mock.patch('asyncio.selector_events.logger')
+ def test_sendto_exception(self, m_log):
+ data = b'data'
+ err = self.sock.sendto.side_effect = RuntimeError()
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport.sendto(data, ())
+
+ self.assertTrue(transport._fatal_error.called)
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on datagram transport')
+ transport._conn_lost = 1
+
+ transport._address = ('123',)
+ transport.sendto(data)
+ transport.sendto(data)
+ transport.sendto(data)
+ transport.sendto(data)
+ transport.sendto(data)
+ m_log.warning.assert_called_with('socket.send() raised exception.')
+
+ def test_sendto_error_received(self):
+ data = b'data'
+
+ self.sock.sendto.side_effect = ConnectionRefusedError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport.sendto(data, ())
+
+ self.assertEqual(transport._conn_lost, 0)
+ self.assertFalse(transport._fatal_error.called)
+
+ def test_sendto_error_received_connected(self):
+ data = b'data'
+
+ self.sock.send.side_effect = ConnectionRefusedError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol, ('0.0.0.0', 1))
+ transport._fatal_error = mock.Mock()
+ transport.sendto(data)
+
+ self.assertFalse(transport._fatal_error.called)
+ self.assertTrue(self.protocol.error_received.called)
+
+ def test_sendto_str(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ self.assertRaises(TypeError, transport.sendto, 'str', ())
+
+ def test_sendto_connected_addr(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol, ('0.0.0.0', 1))
+ self.assertRaises(
+ ValueError, transport.sendto, b'str', ('0.0.0.0', 2))
+
+ def test_sendto_closing(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol, address=(1,))
+ transport.close()
+ self.assertEqual(transport._conn_lost, 1)
+ transport.sendto(b'data', (1,))
+ self.assertEqual(transport._conn_lost, 2)
+
+ def test_sendto_ready(self):
+ data = b'data'
+ self.sock.sendto.return_value = len(data)
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.append((data, ('0.0.0.0', 12345)))
+ self.loop.add_writer(7, transport._sendto_ready)
+ transport._sendto_ready()
+ self.assertTrue(self.sock.sendto.called)
+ self.assertEqual(
+ self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345)))
+ self.assertFalse(self.loop.writers)
+
+ def test_sendto_ready_closing(self):
+ data = b'data'
+ self.sock.send.return_value = len(data)
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._closing = True
+ transport._buffer.append((data, ()))
+ self.loop.add_writer(7, transport._sendto_ready)
+ transport._sendto_ready()
+ self.sock.sendto.assert_called_with(data, ())
+ self.assertFalse(self.loop.writers)
+ self.sock.close.assert_called_with()
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_sendto_ready_no_data(self):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ self.loop.add_writer(7, transport._sendto_ready)
+ transport._sendto_ready()
+ self.assertFalse(self.sock.sendto.called)
+ self.assertFalse(self.loop.writers)
+
+ def test_sendto_ready_tryagain(self):
+ self.sock.sendto.side_effect = BlockingIOError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._buffer.extend([(b'data1', ()), (b'data2', ())])
+ self.loop.add_writer(7, transport._sendto_ready)
+ transport._sendto_ready()
+
+ self.loop.assert_writer(7, transport._sendto_ready)
+ self.assertEqual(
+ [(b'data1', ()), (b'data2', ())],
+ list(transport._buffer))
+
+ def test_sendto_ready_exception(self):
+ err = self.sock.sendto.side_effect = RuntimeError()
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._buffer.append((b'data', ()))
+ transport._sendto_ready()
+
+ transport._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on datagram transport')
+
+ def test_sendto_ready_error_received(self):
+ self.sock.sendto.side_effect = ConnectionRefusedError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol)
+ transport._fatal_error = mock.Mock()
+ transport._buffer.append((b'data', ()))
+ transport._sendto_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+
+ def test_sendto_ready_error_received_connection(self):
+ self.sock.send.side_effect = ConnectionRefusedError
+
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol, ('0.0.0.0', 1))
+ transport._fatal_error = mock.Mock()
+ transport._buffer.append((b'data', ()))
+ transport._sendto_ready()
+
+ self.assertFalse(transport._fatal_error.called)
+ self.assertTrue(self.protocol.error_received.called)
+
+ @mock.patch('asyncio.base_events.logger.error')
+ def test_fatal_error_connected(self, m_exc):
+ transport = _SelectorDatagramTransport(
+ self.loop, self.sock, self.protocol, ('0.0.0.0', 1))
+ err = ConnectionRefusedError()
+ transport._fatal_error(err)
+ self.assertFalse(self.protocol.error_received.called)
+ m_exc.assert_called_with(
+ test_utils.MockPattern(
+ 'Fatal error on transport\nprotocol:.*\ntransport:.*'),
+ exc_info=(ConnectionRefusedError, MOCK_ANY, MOCK_ANY))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py
new file mode 100644
index 0000000..031499e
--- /dev/null
+++ b/Lib/test/test_asyncio/test_streams.py
@@ -0,0 +1,588 @@
+"""Tests for streams.py."""
+
+import gc
+import socket
+import unittest
+from unittest import mock
+try:
+ import ssl
+except ImportError:
+ ssl = None
+
+import asyncio
+from asyncio import test_utils
+
+
+class StreamReaderTests(unittest.TestCase):
+
+ DATA = b'line1\nline2\nline3\n'
+
+ def setUp(self):
+ self.loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ # just in case if we have transport close callbacks
+ test_utils.run_briefly(self.loop)
+
+ self.loop.close()
+ gc.collect()
+
+ @mock.patch('asyncio.streams.events')
+ def test_ctor_global_loop(self, m_events):
+ stream = asyncio.StreamReader()
+ self.assertIs(stream._loop, m_events.get_event_loop.return_value)
+
+ def _basetest_open_connection(self, open_connection_fut):
+ reader, writer = self.loop.run_until_complete(open_connection_fut)
+ writer.write(b'GET / HTTP/1.0\r\n\r\n')
+ f = reader.readline()
+ data = self.loop.run_until_complete(f)
+ self.assertEqual(data, b'HTTP/1.0 200 OK\r\n')
+ f = reader.read()
+ data = self.loop.run_until_complete(f)
+ self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
+ writer.close()
+
+ def test_open_connection(self):
+ with test_utils.run_test_server() as httpd:
+ conn_fut = asyncio.open_connection(*httpd.address,
+ loop=self.loop)
+ self._basetest_open_connection(conn_fut)
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_open_unix_connection(self):
+ with test_utils.run_test_unix_server() as httpd:
+ conn_fut = asyncio.open_unix_connection(httpd.address,
+ loop=self.loop)
+ self._basetest_open_connection(conn_fut)
+
+ def _basetest_open_connection_no_loop_ssl(self, open_connection_fut):
+ try:
+ reader, writer = self.loop.run_until_complete(open_connection_fut)
+ finally:
+ asyncio.set_event_loop(None)
+ writer.write(b'GET / HTTP/1.0\r\n\r\n')
+ f = reader.read()
+ data = self.loop.run_until_complete(f)
+ self.assertTrue(data.endswith(b'\r\n\r\nTest message'))
+
+ writer.close()
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ def test_open_connection_no_loop_ssl(self):
+ with test_utils.run_test_server(use_ssl=True) as httpd:
+ conn_fut = asyncio.open_connection(
+ *httpd.address,
+ ssl=test_utils.dummy_ssl_context(),
+ loop=self.loop)
+
+ self._basetest_open_connection_no_loop_ssl(conn_fut)
+
+ @unittest.skipIf(ssl is None, 'No ssl module')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_open_unix_connection_no_loop_ssl(self):
+ with test_utils.run_test_unix_server(use_ssl=True) as httpd:
+ conn_fut = asyncio.open_unix_connection(
+ httpd.address,
+ ssl=test_utils.dummy_ssl_context(),
+ server_hostname='',
+ loop=self.loop)
+
+ self._basetest_open_connection_no_loop_ssl(conn_fut)
+
+ def _basetest_open_connection_error(self, open_connection_fut):
+ reader, writer = self.loop.run_until_complete(open_connection_fut)
+ writer._protocol.connection_lost(ZeroDivisionError())
+ f = reader.read()
+ with self.assertRaises(ZeroDivisionError):
+ self.loop.run_until_complete(f)
+ writer.close()
+ test_utils.run_briefly(self.loop)
+
+ def test_open_connection_error(self):
+ with test_utils.run_test_server() as httpd:
+ conn_fut = asyncio.open_connection(*httpd.address,
+ loop=self.loop)
+ self._basetest_open_connection_error(conn_fut)
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_open_unix_connection_error(self):
+ with test_utils.run_test_unix_server() as httpd:
+ conn_fut = asyncio.open_unix_connection(httpd.address,
+ loop=self.loop)
+ self._basetest_open_connection_error(conn_fut)
+
+ def test_feed_empty_data(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+
+ stream.feed_data(b'')
+ self.assertEqual(b'', stream._buffer)
+
+ def test_feed_nonempty_data(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+
+ stream.feed_data(self.DATA)
+ self.assertEqual(self.DATA, stream._buffer)
+
+ def test_read_zero(self):
+ # Read zero bytes.
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(self.DATA)
+
+ data = self.loop.run_until_complete(stream.read(0))
+ self.assertEqual(b'', data)
+ self.assertEqual(self.DATA, stream._buffer)
+
+ def test_read(self):
+ # Read bytes.
+ stream = asyncio.StreamReader(loop=self.loop)
+ read_task = asyncio.Task(stream.read(30), loop=self.loop)
+
+ def cb():
+ stream.feed_data(self.DATA)
+ self.loop.call_soon(cb)
+
+ data = self.loop.run_until_complete(read_task)
+ self.assertEqual(self.DATA, data)
+ self.assertEqual(b'', stream._buffer)
+
+ def test_read_line_breaks(self):
+ # Read bytes without line breaks.
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'line1')
+ stream.feed_data(b'line2')
+
+ data = self.loop.run_until_complete(stream.read(5))
+
+ self.assertEqual(b'line1', data)
+ self.assertEqual(b'line2', stream._buffer)
+
+ def test_read_eof(self):
+ # Read bytes, stop at eof.
+ stream = asyncio.StreamReader(loop=self.loop)
+ read_task = asyncio.Task(stream.read(1024), loop=self.loop)
+
+ def cb():
+ stream.feed_eof()
+ self.loop.call_soon(cb)
+
+ data = self.loop.run_until_complete(read_task)
+ self.assertEqual(b'', data)
+ self.assertEqual(b'', stream._buffer)
+
+ def test_read_until_eof(self):
+ # Read all bytes until eof.
+ stream = asyncio.StreamReader(loop=self.loop)
+ read_task = asyncio.Task(stream.read(-1), loop=self.loop)
+
+ def cb():
+ stream.feed_data(b'chunk1\n')
+ stream.feed_data(b'chunk2')
+ stream.feed_eof()
+ self.loop.call_soon(cb)
+
+ data = self.loop.run_until_complete(read_task)
+
+ self.assertEqual(b'chunk1\nchunk2', data)
+ self.assertEqual(b'', stream._buffer)
+
+ def test_read_exception(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'line\n')
+
+ data = self.loop.run_until_complete(stream.read(2))
+ self.assertEqual(b'li', data)
+
+ stream.set_exception(ValueError())
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.read(2))
+
+ def test_readline(self):
+ # Read one line. 'readline' will need to wait for the data
+ # to come from 'cb'
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'chunk1 ')
+ read_task = asyncio.Task(stream.readline(), loop=self.loop)
+
+ def cb():
+ stream.feed_data(b'chunk2 ')
+ stream.feed_data(b'chunk3 ')
+ stream.feed_data(b'\n chunk4')
+ self.loop.call_soon(cb)
+
+ line = self.loop.run_until_complete(read_task)
+ self.assertEqual(b'chunk1 chunk2 chunk3 \n', line)
+ self.assertEqual(b' chunk4', stream._buffer)
+
+ def test_readline_limit_with_existing_data(self):
+ # Read one line. The data is in StreamReader's buffer
+ # before the event loop is run.
+
+ stream = asyncio.StreamReader(limit=3, loop=self.loop)
+ stream.feed_data(b'li')
+ stream.feed_data(b'ne1\nline2\n')
+
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readline())
+ # The buffer should contain the remaining data after exception
+ self.assertEqual(b'line2\n', stream._buffer)
+
+ stream = asyncio.StreamReader(limit=3, loop=self.loop)
+ stream.feed_data(b'li')
+ stream.feed_data(b'ne1')
+ stream.feed_data(b'li')
+
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readline())
+ # No b'\n' at the end. The 'limit' is set to 3. So before
+ # waiting for the new data in buffer, 'readline' will consume
+ # the entire buffer, and since the length of the consumed data
+ # is more than 3, it will raise a ValueError. The buffer is
+ # expected to be empty now.
+ self.assertEqual(b'', stream._buffer)
+
+ def test_at_eof(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ self.assertFalse(stream.at_eof())
+
+ stream.feed_data(b'some data\n')
+ self.assertFalse(stream.at_eof())
+
+ self.loop.run_until_complete(stream.readline())
+ self.assertFalse(stream.at_eof())
+
+ stream.feed_data(b'some data\n')
+ stream.feed_eof()
+ self.loop.run_until_complete(stream.readline())
+ self.assertTrue(stream.at_eof())
+
+ def test_readline_limit(self):
+ # Read one line. StreamReaders are fed with data after
+ # their 'readline' methods are called.
+
+ stream = asyncio.StreamReader(limit=7, loop=self.loop)
+ def cb():
+ stream.feed_data(b'chunk1')
+ stream.feed_data(b'chunk2')
+ stream.feed_data(b'chunk3\n')
+ stream.feed_eof()
+ self.loop.call_soon(cb)
+
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readline())
+ # The buffer had just one line of data, and after raising
+ # a ValueError it should be empty.
+ self.assertEqual(b'', stream._buffer)
+
+ stream = asyncio.StreamReader(limit=7, loop=self.loop)
+ def cb():
+ stream.feed_data(b'chunk1')
+ stream.feed_data(b'chunk2\n')
+ stream.feed_data(b'chunk3\n')
+ stream.feed_eof()
+ self.loop.call_soon(cb)
+
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readline())
+ self.assertEqual(b'chunk3\n', stream._buffer)
+
+ def test_readline_nolimit_nowait(self):
+ # All needed data for the first 'readline' call will be
+ # in the buffer.
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(self.DATA[:6])
+ stream.feed_data(self.DATA[6:])
+
+ line = self.loop.run_until_complete(stream.readline())
+
+ self.assertEqual(b'line1\n', line)
+ self.assertEqual(b'line2\nline3\n', stream._buffer)
+
+ def test_readline_eof(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'some data')
+ stream.feed_eof()
+
+ line = self.loop.run_until_complete(stream.readline())
+ self.assertEqual(b'some data', line)
+
+ def test_readline_empty_eof(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_eof()
+
+ line = self.loop.run_until_complete(stream.readline())
+ self.assertEqual(b'', line)
+
+ def test_readline_read_byte_count(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(self.DATA)
+
+ self.loop.run_until_complete(stream.readline())
+
+ data = self.loop.run_until_complete(stream.read(7))
+
+ self.assertEqual(b'line2\nl', data)
+ self.assertEqual(b'ine3\n', stream._buffer)
+
+ def test_readline_exception(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'line\n')
+
+ data = self.loop.run_until_complete(stream.readline())
+ self.assertEqual(b'line\n', data)
+
+ stream.set_exception(ValueError())
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readline())
+ self.assertEqual(b'', stream._buffer)
+
+ def test_readexactly_zero_or_less(self):
+ # Read exact number of bytes (zero or less).
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(self.DATA)
+
+ data = self.loop.run_until_complete(stream.readexactly(0))
+ self.assertEqual(b'', data)
+ self.assertEqual(self.DATA, stream._buffer)
+
+ data = self.loop.run_until_complete(stream.readexactly(-1))
+ self.assertEqual(b'', data)
+ self.assertEqual(self.DATA, stream._buffer)
+
+ def test_readexactly(self):
+ # Read exact number of bytes.
+ stream = asyncio.StreamReader(loop=self.loop)
+
+ n = 2 * len(self.DATA)
+ read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+
+ def cb():
+ stream.feed_data(self.DATA)
+ stream.feed_data(self.DATA)
+ stream.feed_data(self.DATA)
+ self.loop.call_soon(cb)
+
+ data = self.loop.run_until_complete(read_task)
+ self.assertEqual(self.DATA + self.DATA, data)
+ self.assertEqual(self.DATA, stream._buffer)
+
+ def test_readexactly_eof(self):
+ # Read exact number of bytes (eof).
+ stream = asyncio.StreamReader(loop=self.loop)
+ n = 2 * len(self.DATA)
+ read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)
+
+ def cb():
+ stream.feed_data(self.DATA)
+ stream.feed_eof()
+ self.loop.call_soon(cb)
+
+ with self.assertRaises(asyncio.IncompleteReadError) as cm:
+ self.loop.run_until_complete(read_task)
+ self.assertEqual(cm.exception.partial, self.DATA)
+ self.assertEqual(cm.exception.expected, n)
+ self.assertEqual(str(cm.exception),
+ '18 bytes read on a total of 36 expected bytes')
+ self.assertEqual(b'', stream._buffer)
+
+ def test_readexactly_exception(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ stream.feed_data(b'line\n')
+
+ data = self.loop.run_until_complete(stream.readexactly(2))
+ self.assertEqual(b'li', data)
+
+ stream.set_exception(ValueError())
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete, stream.readexactly(2))
+
+ def test_exception(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+ self.assertIsNone(stream.exception())
+
+ exc = ValueError()
+ stream.set_exception(exc)
+ self.assertIs(stream.exception(), exc)
+
+ def test_exception_waiter(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+
+ @asyncio.coroutine
+ def set_err():
+ stream.set_exception(ValueError())
+
+ @asyncio.coroutine
+ def readline():
+ yield from stream.readline()
+
+ t1 = asyncio.Task(stream.readline(), loop=self.loop)
+ t2 = asyncio.Task(set_err(), loop=self.loop)
+
+ self.loop.run_until_complete(asyncio.wait([t1, t2], loop=self.loop))
+
+ self.assertRaises(ValueError, t1.result)
+
+ def test_exception_cancel(self):
+ stream = asyncio.StreamReader(loop=self.loop)
+
+ @asyncio.coroutine
+ def read_a_line():
+ yield from stream.readline()
+
+ t = asyncio.Task(read_a_line(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ t.cancel()
+ test_utils.run_briefly(self.loop)
+ # The following line fails if set_exception() isn't careful.
+ stream.set_exception(RuntimeError('message'))
+ test_utils.run_briefly(self.loop)
+ self.assertIs(stream._waiter, None)
+
+ def test_start_server(self):
+
+ class MyServer:
+
+ def __init__(self, loop):
+ self.server = None
+ self.loop = loop
+
+ @asyncio.coroutine
+ def handle_client(self, client_reader, client_writer):
+ data = yield from client_reader.readline()
+ client_writer.write(data)
+
+ def start(self):
+ sock = socket.socket()
+ sock.bind(('127.0.0.1', 0))
+ self.server = self.loop.run_until_complete(
+ asyncio.start_server(self.handle_client,
+ sock=sock,
+ loop=self.loop))
+ return sock.getsockname()
+
+ def handle_client_callback(self, client_reader, client_writer):
+ task = asyncio.Task(client_reader.readline(), loop=self.loop)
+
+ def done(task):
+ client_writer.write(task.result())
+
+ task.add_done_callback(done)
+
+ def start_callback(self):
+ sock = socket.socket()
+ sock.bind(('127.0.0.1', 0))
+ addr = sock.getsockname()
+ sock.close()
+ self.server = self.loop.run_until_complete(
+ asyncio.start_server(self.handle_client_callback,
+ host=addr[0], port=addr[1],
+ loop=self.loop))
+ return addr
+
+ def stop(self):
+ if self.server is not None:
+ self.server.close()
+ self.loop.run_until_complete(self.server.wait_closed())
+ self.server = None
+
+ @asyncio.coroutine
+ def client(addr):
+ reader, writer = yield from asyncio.open_connection(
+ *addr, loop=self.loop)
+ # send a line
+ writer.write(b"hello world!\n")
+ # read it back
+ msgback = yield from reader.readline()
+ writer.close()
+ return msgback
+
+ # test the server variant with a coroutine as client handler
+ server = MyServer(self.loop)
+ addr = server.start()
+ msg = self.loop.run_until_complete(asyncio.Task(client(addr),
+ loop=self.loop))
+ server.stop()
+ self.assertEqual(msg, b"hello world!\n")
+
+ # test the server variant with a callback as client handler
+ server = MyServer(self.loop)
+ addr = server.start_callback()
+ msg = self.loop.run_until_complete(asyncio.Task(client(addr),
+ loop=self.loop))
+ server.stop()
+ self.assertEqual(msg, b"hello world!\n")
+
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets')
+ def test_start_unix_server(self):
+
+ class MyServer:
+
+ def __init__(self, loop, path):
+ self.server = None
+ self.loop = loop
+ self.path = path
+
+ @asyncio.coroutine
+ def handle_client(self, client_reader, client_writer):
+ data = yield from client_reader.readline()
+ client_writer.write(data)
+
+ def start(self):
+ self.server = self.loop.run_until_complete(
+ asyncio.start_unix_server(self.handle_client,
+ path=self.path,
+ loop=self.loop))
+
+ def handle_client_callback(self, client_reader, client_writer):
+ task = asyncio.Task(client_reader.readline(), loop=self.loop)
+
+ def done(task):
+ client_writer.write(task.result())
+
+ task.add_done_callback(done)
+
+ def start_callback(self):
+ self.server = self.loop.run_until_complete(
+ asyncio.start_unix_server(self.handle_client_callback,
+ path=self.path,
+ loop=self.loop))
+
+ def stop(self):
+ if self.server is not None:
+ self.server.close()
+ self.loop.run_until_complete(self.server.wait_closed())
+ self.server = None
+
+ @asyncio.coroutine
+ def client(path):
+ reader, writer = yield from asyncio.open_unix_connection(
+ path, loop=self.loop)
+ # send a line
+ writer.write(b"hello world!\n")
+ # read it back
+ msgback = yield from reader.readline()
+ writer.close()
+ return msgback
+
+ # test the server variant with a coroutine as client handler
+ with test_utils.unix_socket_path() as path:
+ server = MyServer(self.loop, path)
+ server.start()
+ msg = self.loop.run_until_complete(asyncio.Task(client(path),
+ loop=self.loop))
+ server.stop()
+ self.assertEqual(msg, b"hello world!\n")
+
+ # test the server variant with a callback as client handler
+ with test_utils.unix_socket_path() as path:
+ server = MyServer(self.loop, path)
+ server.start_callback()
+ msg = self.loop.run_until_complete(asyncio.Task(client(path),
+ loop=self.loop))
+ server.stop()
+ self.assertEqual(msg, b"hello world!\n")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py
new file mode 100644
index 0000000..14fd17e
--- /dev/null
+++ b/Lib/test/test_asyncio/test_subprocess.py
@@ -0,0 +1,184 @@
+from asyncio import subprocess
+import asyncio
+import signal
+import sys
+import unittest
+from test import support
+if sys.platform != 'win32':
+ from asyncio import unix_events
+
+# Program blocking
+PROGRAM_BLOCKED = [sys.executable, '-c', 'import time; time.sleep(3600)']
+
+# Program sleeping during 1 second
+PROGRAM_SLEEP_1SEC = [sys.executable, '-c', 'import time; time.sleep(1)']
+
+# Program copying input to output
+PROGRAM_CAT = [
+ sys.executable, '-c',
+ ';'.join(('import sys',
+ 'data = sys.stdin.buffer.read()',
+ 'sys.stdout.buffer.write(data)'))]
+
+class SubprocessMixin:
+
+ def test_stdin_stdout(self):
+ args = PROGRAM_CAT
+
+ @asyncio.coroutine
+ def run(data):
+ proc = yield from asyncio.create_subprocess_exec(
+ *args,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ loop=self.loop)
+
+ # feed data
+ proc.stdin.write(data)
+ yield from proc.stdin.drain()
+ proc.stdin.close()
+
+ # get output and exitcode
+ data = yield from proc.stdout.read()
+ exitcode = yield from proc.wait()
+ return (exitcode, data)
+
+ task = run(b'some data')
+ task = asyncio.wait_for(task, 10.0, loop=self.loop)
+ exitcode, stdout = self.loop.run_until_complete(task)
+ self.assertEqual(exitcode, 0)
+ self.assertEqual(stdout, b'some data')
+
+ def test_communicate(self):
+ args = PROGRAM_CAT
+
+ @asyncio.coroutine
+ def run(data):
+ proc = yield from asyncio.create_subprocess_exec(
+ *args,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ loop=self.loop)
+ stdout, stderr = yield from proc.communicate(data)
+ return proc.returncode, stdout
+
+ task = run(b'some data')
+ task = asyncio.wait_for(task, 10.0, loop=self.loop)
+ exitcode, stdout = self.loop.run_until_complete(task)
+ self.assertEqual(exitcode, 0)
+ self.assertEqual(stdout, b'some data')
+
+ def test_shell(self):
+ create = asyncio.create_subprocess_shell('exit 7',
+ loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ exitcode = self.loop.run_until_complete(proc.wait())
+ self.assertEqual(exitcode, 7)
+
+ def test_start_new_session(self):
+ # start the new process in a new session
+ create = asyncio.create_subprocess_shell('exit 8',
+ start_new_session=True,
+ loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ exitcode = self.loop.run_until_complete(proc.wait())
+ self.assertEqual(exitcode, 8)
+
+ def test_kill(self):
+ args = PROGRAM_BLOCKED
+ create = asyncio.create_subprocess_exec(*args, loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ proc.kill()
+ returncode = self.loop.run_until_complete(proc.wait())
+ if sys.platform == 'win32':
+ self.assertIsInstance(returncode, int)
+ # expect 1 but sometimes get 0
+ else:
+ self.assertEqual(-signal.SIGKILL, returncode)
+
+ def test_terminate(self):
+ args = PROGRAM_BLOCKED
+ create = asyncio.create_subprocess_exec(*args, loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ proc.terminate()
+ returncode = self.loop.run_until_complete(proc.wait())
+ if sys.platform == 'win32':
+ self.assertIsInstance(returncode, int)
+ # expect 1 but sometimes get 0
+ else:
+ self.assertEqual(-signal.SIGTERM, returncode)
+
+ @unittest.skipIf(sys.platform == 'win32', "Don't have SIGHUP")
+ def test_send_signal(self):
+ args = PROGRAM_BLOCKED
+ create = asyncio.create_subprocess_exec(*args, loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ proc.send_signal(signal.SIGHUP)
+ returncode = self.loop.run_until_complete(proc.wait())
+ self.assertEqual(-signal.SIGHUP, returncode)
+
+ def test_broken_pipe(self):
+ large_data = b'x' * support.PIPE_MAX_SIZE
+
+ create = asyncio.create_subprocess_exec(
+ *PROGRAM_SLEEP_1SEC,
+ stdin=subprocess.PIPE,
+ loop=self.loop)
+ proc = self.loop.run_until_complete(create)
+ with self.assertRaises(BrokenPipeError):
+ self.loop.run_until_complete(proc.communicate(large_data))
+ self.loop.run_until_complete(proc.wait())
+
+
+if sys.platform != 'win32':
+ # Unix
+ class SubprocessWatcherMixin(SubprocessMixin):
+
+ Watcher = None
+
+ def setUp(self):
+ policy = asyncio.get_event_loop_policy()
+ self.loop = policy.new_event_loop()
+
+ # ensure that the event loop is passed explicitly in the code
+ policy.set_event_loop(None)
+
+ watcher = self.Watcher()
+ watcher.attach_loop(self.loop)
+ policy.set_child_watcher(watcher)
+
+ def tearDown(self):
+ policy = asyncio.get_event_loop_policy()
+ policy.set_child_watcher(None)
+ self.loop.close()
+ policy.set_event_loop(None)
+
+ class SubprocessSafeWatcherTests(SubprocessWatcherMixin,
+ unittest.TestCase):
+
+ Watcher = unix_events.SafeChildWatcher
+
+ class SubprocessFastWatcherTests(SubprocessWatcherMixin,
+ unittest.TestCase):
+
+ Watcher = unix_events.FastChildWatcher
+
+else:
+ # Windows
+ class SubprocessProactorTests(SubprocessMixin, unittest.TestCase):
+
+ def setUp(self):
+ policy = asyncio.get_event_loop_policy()
+ self.loop = asyncio.ProactorEventLoop()
+
+ # ensure that the event loop is passed explicitly in the code
+ policy.set_event_loop(None)
+
+ def tearDown(self):
+ policy = asyncio.get_event_loop_policy()
+ self.loop.close()
+ policy.set_event_loop(None)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py
new file mode 100644
index 0000000..ced3431
--- /dev/null
+++ b/Lib/test/test_asyncio/test_tasks.py
@@ -0,0 +1,1677 @@
+"""Tests for tasks.py."""
+
+import gc
+import os.path
+import unittest
+from test.script_helper import assert_python_ok
+
+import asyncio
+from asyncio import test_utils
+
+
+@asyncio.coroutine
+def coroutine_function():
+ pass
+
+
+class Dummy:
+
+ def __repr__(self):
+ return 'Dummy()'
+
+ def __call__(self, *args):
+ pass
+
+
+class TaskTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+ gc.collect()
+
+ def test_task_class(self):
+ @asyncio.coroutine
+ def notmuch():
+ return 'ok'
+ t = asyncio.Task(notmuch(), loop=self.loop)
+ self.loop.run_until_complete(t)
+ self.assertTrue(t.done())
+ self.assertEqual(t.result(), 'ok')
+ self.assertIs(t._loop, self.loop)
+
+ loop = asyncio.new_event_loop()
+ t = asyncio.Task(notmuch(), loop=loop)
+ self.assertIs(t._loop, loop)
+ loop.close()
+
+ def test_async_coroutine(self):
+ @asyncio.coroutine
+ def notmuch():
+ return 'ok'
+ t = asyncio.async(notmuch(), loop=self.loop)
+ self.loop.run_until_complete(t)
+ self.assertTrue(t.done())
+ self.assertEqual(t.result(), 'ok')
+ self.assertIs(t._loop, self.loop)
+
+ loop = asyncio.new_event_loop()
+ t = asyncio.async(notmuch(), loop=loop)
+ self.assertIs(t._loop, loop)
+ loop.close()
+
+ def test_async_future(self):
+ f_orig = asyncio.Future(loop=self.loop)
+ f_orig.set_result('ko')
+
+ f = asyncio.async(f_orig)
+ self.loop.run_until_complete(f)
+ self.assertTrue(f.done())
+ self.assertEqual(f.result(), 'ko')
+ self.assertIs(f, f_orig)
+
+ loop = asyncio.new_event_loop()
+
+ with self.assertRaises(ValueError):
+ f = asyncio.async(f_orig, loop=loop)
+
+ loop.close()
+
+ f = asyncio.async(f_orig, loop=self.loop)
+ self.assertIs(f, f_orig)
+
+ def test_async_task(self):
+ @asyncio.coroutine
+ def notmuch():
+ return 'ok'
+ t_orig = asyncio.Task(notmuch(), loop=self.loop)
+ t = asyncio.async(t_orig)
+ self.loop.run_until_complete(t)
+ self.assertTrue(t.done())
+ self.assertEqual(t.result(), 'ok')
+ self.assertIs(t, t_orig)
+
+ loop = asyncio.new_event_loop()
+
+ with self.assertRaises(ValueError):
+ t = asyncio.async(t_orig, loop=loop)
+
+ loop.close()
+
+ t = asyncio.async(t_orig, loop=self.loop)
+ self.assertIs(t, t_orig)
+
+ def test_async_neither(self):
+ with self.assertRaises(TypeError):
+ asyncio.async('ok')
+
+ def test_task_repr(self):
+ @asyncio.coroutine
+ def notmuch():
+ yield from []
+ return 'abc'
+
+ t = asyncio.Task(notmuch(), loop=self.loop)
+ t.add_done_callback(Dummy())
+ self.assertEqual(repr(t), 'Task(<notmuch>)<PENDING, [Dummy()]>')
+ t.cancel() # Does not take immediate effect!
+ self.assertEqual(repr(t), 'Task(<notmuch>)<CANCELLING, [Dummy()]>')
+ self.assertRaises(asyncio.CancelledError,
+ self.loop.run_until_complete, t)
+ self.assertEqual(repr(t), 'Task(<notmuch>)<CANCELLED>')
+ t = asyncio.Task(notmuch(), loop=self.loop)
+ self.loop.run_until_complete(t)
+ self.assertEqual(repr(t), "Task(<notmuch>)<result='abc'>")
+
+ def test_task_repr_custom(self):
+ @asyncio.coroutine
+ def coro():
+ pass
+
+ class T(asyncio.Future):
+ def __repr__(self):
+ return 'T[]'
+
+ class MyTask(asyncio.Task, T):
+ def __repr__(self):
+ return super().__repr__()
+
+ gen = coro()
+ t = MyTask(gen, loop=self.loop)
+ self.assertEqual(repr(t), 'T[](<coro>)')
+ gen.close()
+
+ def test_task_basics(self):
+ @asyncio.coroutine
+ def outer():
+ a = yield from inner1()
+ b = yield from inner2()
+ return a+b
+
+ @asyncio.coroutine
+ def inner1():
+ return 42
+
+ @asyncio.coroutine
+ def inner2():
+ return 1000
+
+ t = outer()
+ self.assertEqual(self.loop.run_until_complete(t), 1042)
+
+ def test_cancel(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def task():
+ yield from asyncio.sleep(10.0, loop=loop)
+ return 12
+
+ t = asyncio.Task(task(), loop=loop)
+ loop.call_soon(t.cancel)
+ with self.assertRaises(asyncio.CancelledError):
+ loop.run_until_complete(t)
+ self.assertTrue(t.done())
+ self.assertTrue(t.cancelled())
+ self.assertFalse(t.cancel())
+
+ def test_cancel_yield(self):
+ @asyncio.coroutine
+ def task():
+ yield
+ yield
+ return 12
+
+ t = asyncio.Task(task(), loop=self.loop)
+ test_utils.run_briefly(self.loop) # start coro
+ t.cancel()
+ self.assertRaises(
+ asyncio.CancelledError, self.loop.run_until_complete, t)
+ self.assertTrue(t.done())
+ self.assertTrue(t.cancelled())
+ self.assertFalse(t.cancel())
+
+ def test_cancel_inner_future(self):
+ f = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def task():
+ yield from f
+ return 12
+
+ t = asyncio.Task(task(), loop=self.loop)
+ test_utils.run_briefly(self.loop) # start task
+ f.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ self.loop.run_until_complete(t)
+ self.assertTrue(f.cancelled())
+ self.assertTrue(t.cancelled())
+
+ def test_cancel_both_task_and_inner_future(self):
+ f = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def task():
+ yield from f
+ return 12
+
+ t = asyncio.Task(task(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+
+ f.cancel()
+ t.cancel()
+
+ with self.assertRaises(asyncio.CancelledError):
+ self.loop.run_until_complete(t)
+
+ self.assertTrue(t.done())
+ self.assertTrue(f.cancelled())
+ self.assertTrue(t.cancelled())
+
+ def test_cancel_task_catching(self):
+ fut1 = asyncio.Future(loop=self.loop)
+ fut2 = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def task():
+ yield from fut1
+ try:
+ yield from fut2
+ except asyncio.CancelledError:
+ return 42
+
+ t = asyncio.Task(task(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(t._fut_waiter, fut1) # White-box test.
+ fut1.set_result(None)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(t._fut_waiter, fut2) # White-box test.
+ t.cancel()
+ self.assertTrue(fut2.cancelled())
+ res = self.loop.run_until_complete(t)
+ self.assertEqual(res, 42)
+ self.assertFalse(t.cancelled())
+
+ def test_cancel_task_ignoring(self):
+ fut1 = asyncio.Future(loop=self.loop)
+ fut2 = asyncio.Future(loop=self.loop)
+ fut3 = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def task():
+ yield from fut1
+ try:
+ yield from fut2
+ except asyncio.CancelledError:
+ pass
+ res = yield from fut3
+ return res
+
+ t = asyncio.Task(task(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(t._fut_waiter, fut1) # White-box test.
+ fut1.set_result(None)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(t._fut_waiter, fut2) # White-box test.
+ t.cancel()
+ self.assertTrue(fut2.cancelled())
+ test_utils.run_briefly(self.loop)
+ self.assertIs(t._fut_waiter, fut3) # White-box test.
+ fut3.set_result(42)
+ res = self.loop.run_until_complete(t)
+ self.assertEqual(res, 42)
+ self.assertFalse(fut3.cancelled())
+ self.assertFalse(t.cancelled())
+
+ def test_cancel_current_task(self):
+ loop = asyncio.new_event_loop()
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def task():
+ t.cancel()
+ self.assertTrue(t._must_cancel) # White-box test.
+ # The sleep should be cancelled immediately.
+ yield from asyncio.sleep(100, loop=loop)
+ return 12
+
+ t = asyncio.Task(task(), loop=loop)
+ self.assertRaises(
+ asyncio.CancelledError, loop.run_until_complete, t)
+ self.assertTrue(t.done())
+ self.assertFalse(t._must_cancel) # White-box test.
+ self.assertFalse(t.cancel())
+
+ def test_stop_while_run_in_complete(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0.1
+ self.assertAlmostEqual(0.2, when)
+ when = yield 0.1
+ self.assertAlmostEqual(0.3, when)
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ x = 0
+ waiters = []
+
+ @asyncio.coroutine
+ def task():
+ nonlocal x
+ while x < 10:
+ waiters.append(asyncio.sleep(0.1, loop=loop))
+ yield from waiters[-1]
+ x += 1
+ if x == 2:
+ loop.stop()
+
+ t = asyncio.Task(task(), loop=loop)
+ self.assertRaises(
+ RuntimeError, loop.run_until_complete, t)
+ self.assertFalse(t.done())
+ self.assertEqual(x, 2)
+ self.assertAlmostEqual(0.3, loop.time())
+
+ # close generators
+ for w in waiters:
+ w.close()
+
+ def test_wait_for(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.2, when)
+ when = yield 0
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ foo_running = None
+
+ @asyncio.coroutine
+ def foo():
+ nonlocal foo_running
+ foo_running = True
+ try:
+ yield from asyncio.sleep(0.2, loop=loop)
+ finally:
+ foo_running = False
+ return 'done'
+
+ fut = asyncio.Task(foo(), loop=loop)
+
+ with self.assertRaises(asyncio.TimeoutError):
+ loop.run_until_complete(asyncio.wait_for(fut, 0.1, loop=loop))
+ self.assertTrue(fut.done())
+ # it should have been cancelled due to the timeout
+ self.assertTrue(fut.cancelled())
+ self.assertAlmostEqual(0.1, loop.time())
+ self.assertEqual(foo_running, False)
+
+ def test_wait_for_blocking(self):
+ loop = test_utils.TestLoop()
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def coro():
+ return 'done'
+
+ res = loop.run_until_complete(asyncio.wait_for(coro(),
+ timeout=None,
+ loop=loop))
+ self.assertEqual(res, 'done')
+
+ def test_wait_for_with_global_loop(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.2, when)
+ when = yield 0
+ self.assertAlmostEqual(0.01, when)
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def foo():
+ yield from asyncio.sleep(0.2, loop=loop)
+ return 'done'
+
+ asyncio.set_event_loop(loop)
+ try:
+ fut = asyncio.Task(foo(), loop=loop)
+ with self.assertRaises(asyncio.TimeoutError):
+ loop.run_until_complete(asyncio.wait_for(fut, 0.01))
+ finally:
+ asyncio.set_event_loop(None)
+
+ self.assertAlmostEqual(0.01, loop.time())
+ self.assertTrue(fut.done())
+ self.assertTrue(fut.cancelled())
+
+ def test_wait(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0
+ self.assertAlmostEqual(0.15, when)
+ yield 0.15
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
+ b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ done, pending = yield from asyncio.wait([b, a], loop=loop)
+ self.assertEqual(done, set([a, b]))
+ self.assertEqual(pending, set())
+ return 42
+
+ res = loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertEqual(res, 42)
+ self.assertAlmostEqual(0.15, loop.time())
+
+ # Doing it again should take no time and exercise a different path.
+ res = loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.15, loop.time())
+ self.assertEqual(res, 42)
+
+ def test_wait_with_global_loop(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.01, when)
+ when = yield 0
+ self.assertAlmostEqual(0.015, when)
+ yield 0.015
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(0.01, loop=loop), loop=loop)
+ b = asyncio.Task(asyncio.sleep(0.015, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ done, pending = yield from asyncio.wait([b, a])
+ self.assertEqual(done, set([a, b]))
+ self.assertEqual(pending, set())
+ return 42
+
+ asyncio.set_event_loop(loop)
+ try:
+ res = loop.run_until_complete(
+ asyncio.Task(foo(), loop=loop))
+ finally:
+ asyncio.set_event_loop(None)
+
+ self.assertEqual(res, 42)
+
+ def test_wait_duplicate_coroutines(self):
+ @asyncio.coroutine
+ def coro(s):
+ return s
+ c = coro('test')
+
+ task = asyncio.Task(
+ asyncio.wait([c, c, coro('spam')], loop=self.loop),
+ loop=self.loop)
+
+ done, pending = self.loop.run_until_complete(task)
+
+ self.assertFalse(pending)
+ self.assertEqual(set(f.result() for f in done), {'test', 'spam'})
+
+ def test_wait_errors(self):
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete,
+ asyncio.wait(set(), loop=self.loop))
+
+ self.assertRaises(
+ ValueError, self.loop.run_until_complete,
+ asyncio.wait([asyncio.sleep(10.0, loop=self.loop)],
+ return_when=-1, loop=self.loop))
+
+ def test_wait_first_completed(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ when = yield 0
+ self.assertAlmostEqual(0.1, when)
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop)
+ b = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
+ task = asyncio.Task(
+ asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED,
+ loop=loop),
+ loop=loop)
+
+ done, pending = loop.run_until_complete(task)
+ self.assertEqual({b}, done)
+ self.assertEqual({a}, pending)
+ self.assertFalse(a.done())
+ self.assertTrue(b.done())
+ self.assertIsNone(b.result())
+ self.assertAlmostEqual(0.1, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_wait_really_done(self):
+ # there is possibility that some tasks in the pending list
+ # became done but their callbacks haven't all been called yet
+
+ @asyncio.coroutine
+ def coro1():
+ yield
+
+ @asyncio.coroutine
+ def coro2():
+ yield
+ yield
+
+ a = asyncio.Task(coro1(), loop=self.loop)
+ b = asyncio.Task(coro2(), loop=self.loop)
+ task = asyncio.Task(
+ asyncio.wait([b, a], return_when=asyncio.FIRST_COMPLETED,
+ loop=self.loop),
+ loop=self.loop)
+
+ done, pending = self.loop.run_until_complete(task)
+ self.assertEqual({a, b}, done)
+ self.assertTrue(a.done())
+ self.assertIsNone(a.result())
+ self.assertTrue(b.done())
+ self.assertIsNone(b.result())
+
+ def test_wait_first_exception(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ # first_exception, task already has exception
+ a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def exc():
+ raise ZeroDivisionError('err')
+
+ b = asyncio.Task(exc(), loop=loop)
+ task = asyncio.Task(
+ asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION,
+ loop=loop),
+ loop=loop)
+
+ done, pending = loop.run_until_complete(task)
+ self.assertEqual({b}, done)
+ self.assertEqual({a}, pending)
+ self.assertAlmostEqual(0, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_wait_first_exception_in_wait(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ when = yield 0
+ self.assertAlmostEqual(0.01, when)
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ # first_exception, exception during waiting
+ a = asyncio.Task(asyncio.sleep(10.0, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def exc():
+ yield from asyncio.sleep(0.01, loop=loop)
+ raise ZeroDivisionError('err')
+
+ b = asyncio.Task(exc(), loop=loop)
+ task = asyncio.wait([b, a], return_when=asyncio.FIRST_EXCEPTION,
+ loop=loop)
+
+ done, pending = loop.run_until_complete(task)
+ self.assertEqual({b}, done)
+ self.assertEqual({a}, pending)
+ self.assertAlmostEqual(0.01, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_wait_with_exception(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0
+ self.assertAlmostEqual(0.15, when)
+ yield 0.15
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def sleeper():
+ yield from asyncio.sleep(0.15, loop=loop)
+ raise ZeroDivisionError('really')
+
+ b = asyncio.Task(sleeper(), loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ done, pending = yield from asyncio.wait([b, a], loop=loop)
+ self.assertEqual(len(done), 2)
+ self.assertEqual(pending, set())
+ errors = set(f for f in done if f.exception() is not None)
+ self.assertEqual(len(errors), 1)
+
+ loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.15, loop.time())
+
+ loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.15, loop.time())
+
+ def test_wait_with_timeout(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0
+ self.assertAlmostEqual(0.15, when)
+ when = yield 0
+ self.assertAlmostEqual(0.11, when)
+ yield 0.11
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
+ b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ done, pending = yield from asyncio.wait([b, a], timeout=0.11,
+ loop=loop)
+ self.assertEqual(done, set([a]))
+ self.assertEqual(pending, set([b]))
+
+ loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.11, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_wait_concurrent_complete(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0
+ self.assertAlmostEqual(0.15, when)
+ when = yield 0
+ self.assertAlmostEqual(0.1, when)
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.Task(asyncio.sleep(0.1, loop=loop), loop=loop)
+ b = asyncio.Task(asyncio.sleep(0.15, loop=loop), loop=loop)
+
+ done, pending = loop.run_until_complete(
+ asyncio.wait([b, a], timeout=0.1, loop=loop))
+
+ self.assertEqual(done, set([a]))
+ self.assertEqual(pending, set([b]))
+ self.assertAlmostEqual(0.1, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_as_completed(self):
+
+ def gen():
+ yield 0
+ yield 0
+ yield 0.01
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+ completed = set()
+ time_shifted = False
+
+ @asyncio.coroutine
+ def sleeper(dt, x):
+ nonlocal time_shifted
+ yield from asyncio.sleep(dt, loop=loop)
+ completed.add(x)
+ if not time_shifted and 'a' in completed and 'b' in completed:
+ time_shifted = True
+ loop.advance_time(0.14)
+ return x
+
+ a = sleeper(0.01, 'a')
+ b = sleeper(0.01, 'b')
+ c = sleeper(0.15, 'c')
+
+ @asyncio.coroutine
+ def foo():
+ values = []
+ for f in asyncio.as_completed([b, c, a], loop=loop):
+ values.append((yield from f))
+ return values
+
+ res = loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.15, loop.time())
+ self.assertTrue('a' in res[:2])
+ self.assertTrue('b' in res[:2])
+ self.assertEqual(res[2], 'c')
+
+ # Doing it again should take no time and exercise a different path.
+ res = loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertAlmostEqual(0.15, loop.time())
+
+ def test_as_completed_with_timeout(self):
+
+ def gen():
+ yield
+ yield 0
+ yield 0
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.sleep(0.1, 'a', loop=loop)
+ b = asyncio.sleep(0.15, 'b', loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ values = []
+ for f in asyncio.as_completed([a, b], timeout=0.12, loop=loop):
+ if values:
+ loop.advance_time(0.02)
+ try:
+ v = yield from f
+ values.append((1, v))
+ except asyncio.TimeoutError as exc:
+ values.append((2, exc))
+ return values
+
+ res = loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+ self.assertEqual(len(res), 2, res)
+ self.assertEqual(res[0], (1, 'a'))
+ self.assertEqual(res[1][0], 2)
+ self.assertIsInstance(res[1][1], asyncio.TimeoutError)
+ self.assertAlmostEqual(0.12, loop.time())
+
+ # move forward to close generator
+ loop.advance_time(10)
+ loop.run_until_complete(asyncio.wait([a, b], loop=loop))
+
+ def test_as_completed_with_unused_timeout(self):
+
+ def gen():
+ yield
+ yield 0
+ yield 0.01
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.sleep(0.01, 'a', loop=loop)
+
+ @asyncio.coroutine
+ def foo():
+ for f in asyncio.as_completed([a], timeout=1, loop=loop):
+ v = yield from f
+ self.assertEqual(v, 'a')
+
+ loop.run_until_complete(asyncio.Task(foo(), loop=loop))
+
+ def test_as_completed_reverse_wait(self):
+
+ def gen():
+ yield 0
+ yield 0.05
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.sleep(0.05, 'a', loop=loop)
+ b = asyncio.sleep(0.10, 'b', loop=loop)
+ fs = {a, b}
+ futs = list(asyncio.as_completed(fs, loop=loop))
+ self.assertEqual(len(futs), 2)
+
+ x = loop.run_until_complete(futs[1])
+ self.assertEqual(x, 'a')
+ self.assertAlmostEqual(0.05, loop.time())
+ loop.advance_time(0.05)
+ y = loop.run_until_complete(futs[0])
+ self.assertEqual(y, 'b')
+ self.assertAlmostEqual(0.10, loop.time())
+
+ def test_as_completed_concurrent(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.05, when)
+ when = yield 0
+ self.assertAlmostEqual(0.05, when)
+ yield 0.05
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ a = asyncio.sleep(0.05, 'a', loop=loop)
+ b = asyncio.sleep(0.05, 'b', loop=loop)
+ fs = {a, b}
+ futs = list(asyncio.as_completed(fs, loop=loop))
+ self.assertEqual(len(futs), 2)
+ waiter = asyncio.wait(futs, loop=loop)
+ done, pending = loop.run_until_complete(waiter)
+ self.assertEqual(set(f.result() for f in done), {'a', 'b'})
+
+ def test_as_completed_duplicate_coroutines(self):
+
+ @asyncio.coroutine
+ def coro(s):
+ return s
+
+ @asyncio.coroutine
+ def runner():
+ result = []
+ c = coro('ham')
+ for f in asyncio.as_completed([c, c, coro('spam')],
+ loop=self.loop):
+ result.append((yield from f))
+ return result
+
+ fut = asyncio.Task(runner(), loop=self.loop)
+ self.loop.run_until_complete(fut)
+ result = fut.result()
+ self.assertEqual(set(result), {'ham', 'spam'})
+ self.assertEqual(len(result), 2)
+
+ def test_sleep(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.05, when)
+ when = yield 0.05
+ self.assertAlmostEqual(0.1, when)
+ yield 0.05
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def sleeper(dt, arg):
+ yield from asyncio.sleep(dt/2, loop=loop)
+ res = yield from asyncio.sleep(dt/2, arg, loop=loop)
+ return res
+
+ t = asyncio.Task(sleeper(0.1, 'yeah'), loop=loop)
+ loop.run_until_complete(t)
+ self.assertTrue(t.done())
+ self.assertEqual(t.result(), 'yeah')
+ self.assertAlmostEqual(0.1, loop.time())
+
+ def test_sleep_cancel(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ t = asyncio.Task(asyncio.sleep(10.0, 'yeah', loop=loop),
+ loop=loop)
+
+ handle = None
+ orig_call_later = loop.call_later
+
+ def call_later(self, delay, callback, *args):
+ nonlocal handle
+ handle = orig_call_later(self, delay, callback, *args)
+ return handle
+
+ loop.call_later = call_later
+ test_utils.run_briefly(loop)
+
+ self.assertFalse(handle._cancelled)
+
+ t.cancel()
+ test_utils.run_briefly(loop)
+ self.assertTrue(handle._cancelled)
+
+ def test_task_cancel_sleeping_task(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(0.1, when)
+ when = yield 0
+ self.assertAlmostEqual(5000, when)
+ yield 0.1
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def sleep(dt):
+ yield from asyncio.sleep(dt, loop=loop)
+
+ @asyncio.coroutine
+ def doit():
+ sleeper = asyncio.Task(sleep(5000), loop=loop)
+ loop.call_later(0.1, sleeper.cancel)
+ try:
+ yield from sleeper
+ except asyncio.CancelledError:
+ return 'cancelled'
+ else:
+ return 'slept in'
+
+ doer = doit()
+ self.assertEqual(loop.run_until_complete(doer), 'cancelled')
+ self.assertAlmostEqual(0.1, loop.time())
+
+ def test_task_cancel_waiter_future(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def coro():
+ yield from fut
+
+ task = asyncio.Task(coro(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(task._fut_waiter, fut)
+
+ task.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertRaises(
+ asyncio.CancelledError, self.loop.run_until_complete, task)
+ self.assertIsNone(task._fut_waiter)
+ self.assertTrue(fut.cancelled())
+
+ def test_step_in_completed_task(self):
+ @asyncio.coroutine
+ def notmuch():
+ return 'ko'
+
+ gen = notmuch()
+ task = asyncio.Task(gen, loop=self.loop)
+ task.set_result('ok')
+
+ self.assertRaises(AssertionError, task._step)
+ gen.close()
+
+ def test_step_result(self):
+ @asyncio.coroutine
+ def notmuch():
+ yield None
+ yield 1
+ return 'ko'
+
+ self.assertRaises(
+ RuntimeError, self.loop.run_until_complete, notmuch())
+
+ def test_step_result_future(self):
+ # If coroutine returns future, task waits on this future.
+
+ class Fut(asyncio.Future):
+ def __init__(self, *args, **kwds):
+ self.cb_added = False
+ super().__init__(*args, **kwds)
+
+ def add_done_callback(self, fn):
+ self.cb_added = True
+ super().add_done_callback(fn)
+
+ fut = Fut(loop=self.loop)
+ result = None
+
+ @asyncio.coroutine
+ def wait_for_future():
+ nonlocal result
+ result = yield from fut
+
+ t = asyncio.Task(wait_for_future(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(fut.cb_added)
+
+ res = object()
+ fut.set_result(res)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(res, result)
+ self.assertTrue(t.done())
+ self.assertIsNone(t.result())
+
+ def test_step_with_baseexception(self):
+ @asyncio.coroutine
+ def notmutch():
+ raise BaseException()
+
+ task = asyncio.Task(notmutch(), loop=self.loop)
+ self.assertRaises(BaseException, task._step)
+
+ self.assertTrue(task.done())
+ self.assertIsInstance(task.exception(), BaseException)
+
+ def test_baseexception_during_cancel(self):
+
+ def gen():
+ when = yield
+ self.assertAlmostEqual(10.0, when)
+ yield 0
+
+ loop = test_utils.TestLoop(gen)
+ self.addCleanup(loop.close)
+
+ @asyncio.coroutine
+ def sleeper():
+ yield from asyncio.sleep(10, loop=loop)
+
+ base_exc = BaseException()
+
+ @asyncio.coroutine
+ def notmutch():
+ try:
+ yield from sleeper()
+ except asyncio.CancelledError:
+ raise base_exc
+
+ task = asyncio.Task(notmutch(), loop=loop)
+ test_utils.run_briefly(loop)
+
+ task.cancel()
+ self.assertFalse(task.done())
+
+ self.assertRaises(BaseException, test_utils.run_briefly, loop)
+
+ self.assertTrue(task.done())
+ self.assertFalse(task.cancelled())
+ self.assertIs(task.exception(), base_exc)
+
+ def test_iscoroutinefunction(self):
+ def fn():
+ pass
+
+ self.assertFalse(asyncio.iscoroutinefunction(fn))
+
+ def fn1():
+ yield
+ self.assertFalse(asyncio.iscoroutinefunction(fn1))
+
+ @asyncio.coroutine
+ def fn2():
+ yield
+ self.assertTrue(asyncio.iscoroutinefunction(fn2))
+
+ def test_yield_vs_yield_from(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def wait_for_future():
+ yield fut
+
+ task = wait_for_future()
+ with self.assertRaises(RuntimeError):
+ self.loop.run_until_complete(task)
+
+ self.assertFalse(fut.done())
+
+ def test_yield_vs_yield_from_generator(self):
+ @asyncio.coroutine
+ def coro():
+ yield
+
+ @asyncio.coroutine
+ def wait_for_future():
+ gen = coro()
+ try:
+ yield gen
+ finally:
+ gen.close()
+
+ task = wait_for_future()
+ self.assertRaises(
+ RuntimeError,
+ self.loop.run_until_complete, task)
+
+ def test_coroutine_non_gen_function(self):
+ @asyncio.coroutine
+ def func():
+ return 'test'
+
+ self.assertTrue(asyncio.iscoroutinefunction(func))
+
+ coro = func()
+ self.assertTrue(asyncio.iscoroutine(coro))
+
+ res = self.loop.run_until_complete(coro)
+ self.assertEqual(res, 'test')
+
+ def test_coroutine_non_gen_function_return_future(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def func():
+ return fut
+
+ @asyncio.coroutine
+ def coro():
+ fut.set_result('test')
+
+ t1 = asyncio.Task(func(), loop=self.loop)
+ t2 = asyncio.Task(coro(), loop=self.loop)
+ res = self.loop.run_until_complete(t1)
+ self.assertEqual(res, 'test')
+ self.assertIsNone(t2.result())
+
+ def test_current_task(self):
+ self.assertIsNone(asyncio.Task.current_task(loop=self.loop))
+
+ @asyncio.coroutine
+ def coro(loop):
+ self.assertTrue(asyncio.Task.current_task(loop=loop) is task)
+
+ task = asyncio.Task(coro(self.loop), loop=self.loop)
+ self.loop.run_until_complete(task)
+ self.assertIsNone(asyncio.Task.current_task(loop=self.loop))
+
+ def test_current_task_with_interleaving_tasks(self):
+ self.assertIsNone(asyncio.Task.current_task(loop=self.loop))
+
+ fut1 = asyncio.Future(loop=self.loop)
+ fut2 = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def coro1(loop):
+ self.assertTrue(asyncio.Task.current_task(loop=loop) is task1)
+ yield from fut1
+ self.assertTrue(asyncio.Task.current_task(loop=loop) is task1)
+ fut2.set_result(True)
+
+ @asyncio.coroutine
+ def coro2(loop):
+ self.assertTrue(asyncio.Task.current_task(loop=loop) is task2)
+ fut1.set_result(True)
+ yield from fut2
+ self.assertTrue(asyncio.Task.current_task(loop=loop) is task2)
+
+ task1 = asyncio.Task(coro1(self.loop), loop=self.loop)
+ task2 = asyncio.Task(coro2(self.loop), loop=self.loop)
+
+ self.loop.run_until_complete(asyncio.wait((task1, task2),
+ loop=self.loop))
+ self.assertIsNone(asyncio.Task.current_task(loop=self.loop))
+
+ # Some thorough tests for cancellation propagation through
+ # coroutines, tasks and wait().
+
+ def test_yield_future_passes_cancel(self):
+ # Cancelling outer() cancels inner() cancels waiter.
+ proof = 0
+ waiter = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def inner():
+ nonlocal proof
+ try:
+ yield from waiter
+ except asyncio.CancelledError:
+ proof += 1
+ raise
+ else:
+ self.fail('got past sleep() in inner()')
+
+ @asyncio.coroutine
+ def outer():
+ nonlocal proof
+ try:
+ yield from inner()
+ except asyncio.CancelledError:
+ proof += 100 # Expect this path.
+ else:
+ proof += 10
+
+ f = asyncio.async(outer(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ f.cancel()
+ self.loop.run_until_complete(f)
+ self.assertEqual(proof, 101)
+ self.assertTrue(waiter.cancelled())
+
+ def test_yield_wait_does_not_shield_cancel(self):
+ # Cancelling outer() makes wait() return early, leaves inner()
+ # running.
+ proof = 0
+ waiter = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def inner():
+ nonlocal proof
+ yield from waiter
+ proof += 1
+
+ @asyncio.coroutine
+ def outer():
+ nonlocal proof
+ d, p = yield from asyncio.wait([inner()], loop=self.loop)
+ proof += 100
+
+ f = asyncio.async(outer(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ f.cancel()
+ self.assertRaises(
+ asyncio.CancelledError, self.loop.run_until_complete, f)
+ waiter.set_result(None)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(proof, 1)
+
+ def test_shield_result(self):
+ inner = asyncio.Future(loop=self.loop)
+ outer = asyncio.shield(inner)
+ inner.set_result(42)
+ res = self.loop.run_until_complete(outer)
+ self.assertEqual(res, 42)
+
+ def test_shield_exception(self):
+ inner = asyncio.Future(loop=self.loop)
+ outer = asyncio.shield(inner)
+ test_utils.run_briefly(self.loop)
+ exc = RuntimeError('expected')
+ inner.set_exception(exc)
+ test_utils.run_briefly(self.loop)
+ self.assertIs(outer.exception(), exc)
+
+ def test_shield_cancel(self):
+ inner = asyncio.Future(loop=self.loop)
+ outer = asyncio.shield(inner)
+ test_utils.run_briefly(self.loop)
+ inner.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(outer.cancelled())
+
+ def test_shield_shortcut(self):
+ fut = asyncio.Future(loop=self.loop)
+ fut.set_result(42)
+ res = self.loop.run_until_complete(asyncio.shield(fut))
+ self.assertEqual(res, 42)
+
+ def test_shield_effect(self):
+ # Cancelling outer() does not affect inner().
+ proof = 0
+ waiter = asyncio.Future(loop=self.loop)
+
+ @asyncio.coroutine
+ def inner():
+ nonlocal proof
+ yield from waiter
+ proof += 1
+
+ @asyncio.coroutine
+ def outer():
+ nonlocal proof
+ yield from asyncio.shield(inner(), loop=self.loop)
+ proof += 100
+
+ f = asyncio.async(outer(), loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ f.cancel()
+ with self.assertRaises(asyncio.CancelledError):
+ self.loop.run_until_complete(f)
+ waiter.set_result(None)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(proof, 1)
+
+ def test_shield_gather(self):
+ child1 = asyncio.Future(loop=self.loop)
+ child2 = asyncio.Future(loop=self.loop)
+ parent = asyncio.gather(child1, child2, loop=self.loop)
+ outer = asyncio.shield(parent, loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ outer.cancel()
+ test_utils.run_briefly(self.loop)
+ self.assertTrue(outer.cancelled())
+ child1.set_result(1)
+ child2.set_result(2)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(parent.result(), [1, 2])
+
+ def test_gather_shield(self):
+ child1 = asyncio.Future(loop=self.loop)
+ child2 = asyncio.Future(loop=self.loop)
+ inner1 = asyncio.shield(child1, loop=self.loop)
+ inner2 = asyncio.shield(child2, loop=self.loop)
+ parent = asyncio.gather(inner1, inner2, loop=self.loop)
+ test_utils.run_briefly(self.loop)
+ parent.cancel()
+ # This should cancel inner1 and inner2 but bot child1 and child2.
+ test_utils.run_briefly(self.loop)
+ self.assertIsInstance(parent.exception(), asyncio.CancelledError)
+ self.assertTrue(inner1.cancelled())
+ self.assertTrue(inner2.cancelled())
+ child1.set_result(1)
+ child2.set_result(2)
+ test_utils.run_briefly(self.loop)
+
+ def test_as_completed_invalid_args(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ # as_completed() expects a list of futures, not a future instance
+ self.assertRaises(TypeError, self.loop.run_until_complete,
+ asyncio.as_completed(fut, loop=self.loop))
+ self.assertRaises(TypeError, self.loop.run_until_complete,
+ asyncio.as_completed(coroutine_function(), loop=self.loop))
+
+ def test_wait_invalid_args(self):
+ fut = asyncio.Future(loop=self.loop)
+
+ # wait() expects a list of futures, not a future instance
+ self.assertRaises(TypeError, self.loop.run_until_complete,
+ asyncio.wait(fut, loop=self.loop))
+ self.assertRaises(TypeError, self.loop.run_until_complete,
+ asyncio.wait(coroutine_function(), loop=self.loop))
+
+ # wait() expects at least a future
+ self.assertRaises(ValueError, self.loop.run_until_complete,
+ asyncio.wait([], loop=self.loop))
+
+class GatherTestsBase:
+
+ def setUp(self):
+ self.one_loop = test_utils.TestLoop()
+ self.other_loop = test_utils.TestLoop()
+
+ def tearDown(self):
+ self.one_loop.close()
+ self.other_loop.close()
+
+ def _run_loop(self, loop):
+ while loop._ready:
+ test_utils.run_briefly(loop)
+
+ def _check_success(self, **kwargs):
+ a, b, c = [asyncio.Future(loop=self.one_loop) for i in range(3)]
+ fut = asyncio.gather(*self.wrap_futures(a, b, c), **kwargs)
+ cb = test_utils.MockCallback()
+ fut.add_done_callback(cb)
+ b.set_result(1)
+ a.set_result(2)
+ self._run_loop(self.one_loop)
+ self.assertEqual(cb.called, False)
+ self.assertFalse(fut.done())
+ c.set_result(3)
+ self._run_loop(self.one_loop)
+ cb.assert_called_once_with(fut)
+ self.assertEqual(fut.result(), [2, 1, 3])
+
+ def test_success(self):
+ self._check_success()
+ self._check_success(return_exceptions=False)
+
+ def test_result_exception_success(self):
+ self._check_success(return_exceptions=True)
+
+ def test_one_exception(self):
+ a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+ fut = asyncio.gather(*self.wrap_futures(a, b, c, d, e))
+ cb = test_utils.MockCallback()
+ fut.add_done_callback(cb)
+ exc = ZeroDivisionError()
+ a.set_result(1)
+ b.set_exception(exc)
+ self._run_loop(self.one_loop)
+ self.assertTrue(fut.done())
+ cb.assert_called_once_with(fut)
+ self.assertIs(fut.exception(), exc)
+ # Does nothing
+ c.set_result(3)
+ d.cancel()
+ e.set_exception(RuntimeError())
+ e.exception()
+
+ def test_return_exceptions(self):
+ a, b, c, d = [asyncio.Future(loop=self.one_loop) for i in range(4)]
+ fut = asyncio.gather(*self.wrap_futures(a, b, c, d),
+ return_exceptions=True)
+ cb = test_utils.MockCallback()
+ fut.add_done_callback(cb)
+ exc = ZeroDivisionError()
+ exc2 = RuntimeError()
+ b.set_result(1)
+ c.set_exception(exc)
+ a.set_result(3)
+ self._run_loop(self.one_loop)
+ self.assertFalse(fut.done())
+ d.set_exception(exc2)
+ self._run_loop(self.one_loop)
+ self.assertTrue(fut.done())
+ cb.assert_called_once_with(fut)
+ self.assertEqual(fut.result(), [3, 1, exc, exc2])
+
+ def test_env_var_debug(self):
+ path = os.path.dirname(asyncio.__file__)
+ path = os.path.normpath(os.path.join(path, '..'))
+ code = '\n'.join((
+ 'import sys',
+ 'sys.path.insert(0, %r)' % path,
+ 'import asyncio.tasks',
+ 'print(asyncio.tasks._DEBUG)'))
+
+ # Test with -E to not fail if the unit test was run with
+ # PYTHONASYNCIODEBUG set to a non-empty string
+ sts, stdout, stderr = assert_python_ok('-E', '-c', code)
+ self.assertEqual(stdout.rstrip(), b'False')
+
+ sts, stdout, stderr = assert_python_ok('-c', code,
+ PYTHONASYNCIODEBUG='')
+ self.assertEqual(stdout.rstrip(), b'False')
+
+ sts, stdout, stderr = assert_python_ok('-c', code,
+ PYTHONASYNCIODEBUG='1')
+ self.assertEqual(stdout.rstrip(), b'True')
+
+ sts, stdout, stderr = assert_python_ok('-E', '-c', code,
+ PYTHONASYNCIODEBUG='1')
+ self.assertEqual(stdout.rstrip(), b'False')
+
+
+class FutureGatherTests(GatherTestsBase, unittest.TestCase):
+
+ def wrap_futures(self, *futures):
+ return futures
+
+ def _check_empty_sequence(self, seq_or_iter):
+ asyncio.set_event_loop(self.one_loop)
+ self.addCleanup(asyncio.set_event_loop, None)
+ fut = asyncio.gather(*seq_or_iter)
+ self.assertIsInstance(fut, asyncio.Future)
+ self.assertIs(fut._loop, self.one_loop)
+ self._run_loop(self.one_loop)
+ self.assertTrue(fut.done())
+ self.assertEqual(fut.result(), [])
+ fut = asyncio.gather(*seq_or_iter, loop=self.other_loop)
+ self.assertIs(fut._loop, self.other_loop)
+
+ def test_constructor_empty_sequence(self):
+ self._check_empty_sequence([])
+ self._check_empty_sequence(())
+ self._check_empty_sequence(set())
+ self._check_empty_sequence(iter(""))
+
+ def test_constructor_heterogenous_futures(self):
+ fut1 = asyncio.Future(loop=self.one_loop)
+ fut2 = asyncio.Future(loop=self.other_loop)
+ with self.assertRaises(ValueError):
+ asyncio.gather(fut1, fut2)
+ with self.assertRaises(ValueError):
+ asyncio.gather(fut1, loop=self.other_loop)
+
+ def test_constructor_homogenous_futures(self):
+ children = [asyncio.Future(loop=self.other_loop) for i in range(3)]
+ fut = asyncio.gather(*children)
+ self.assertIs(fut._loop, self.other_loop)
+ self._run_loop(self.other_loop)
+ self.assertFalse(fut.done())
+ fut = asyncio.gather(*children, loop=self.other_loop)
+ self.assertIs(fut._loop, self.other_loop)
+ self._run_loop(self.other_loop)
+ self.assertFalse(fut.done())
+
+ def test_one_cancellation(self):
+ a, b, c, d, e = [asyncio.Future(loop=self.one_loop) for i in range(5)]
+ fut = asyncio.gather(a, b, c, d, e)
+ cb = test_utils.MockCallback()
+ fut.add_done_callback(cb)
+ a.set_result(1)
+ b.cancel()
+ self._run_loop(self.one_loop)
+ self.assertTrue(fut.done())
+ cb.assert_called_once_with(fut)
+ self.assertFalse(fut.cancelled())
+ self.assertIsInstance(fut.exception(), asyncio.CancelledError)
+ # Does nothing
+ c.set_result(3)
+ d.cancel()
+ e.set_exception(RuntimeError())
+ e.exception()
+
+ def test_result_exception_one_cancellation(self):
+ a, b, c, d, e, f = [asyncio.Future(loop=self.one_loop)
+ for i in range(6)]
+ fut = asyncio.gather(a, b, c, d, e, f, return_exceptions=True)
+ cb = test_utils.MockCallback()
+ fut.add_done_callback(cb)
+ a.set_result(1)
+ zde = ZeroDivisionError()
+ b.set_exception(zde)
+ c.cancel()
+ self._run_loop(self.one_loop)
+ self.assertFalse(fut.done())
+ d.set_result(3)
+ e.cancel()
+ rte = RuntimeError()
+ f.set_exception(rte)
+ res = self.one_loop.run_until_complete(fut)
+ self.assertIsInstance(res[2], asyncio.CancelledError)
+ self.assertIsInstance(res[4], asyncio.CancelledError)
+ res[2] = res[4] = None
+ self.assertEqual(res, [1, zde, None, 3, None, rte])
+ cb.assert_called_once_with(fut)
+
+
+class CoroutineGatherTests(GatherTestsBase, unittest.TestCase):
+
+ def setUp(self):
+ super().setUp()
+ asyncio.set_event_loop(self.one_loop)
+
+ def tearDown(self):
+ asyncio.set_event_loop(None)
+ super().tearDown()
+
+ def wrap_futures(self, *futures):
+ coros = []
+ for fut in futures:
+ @asyncio.coroutine
+ def coro(fut=fut):
+ return (yield from fut)
+ coros.append(coro())
+ return coros
+
+ def test_constructor_loop_selection(self):
+ @asyncio.coroutine
+ def coro():
+ return 'abc'
+ gen1 = coro()
+ gen2 = coro()
+ fut = asyncio.gather(gen1, gen2)
+ self.assertIs(fut._loop, self.one_loop)
+ gen1.close()
+ gen2.close()
+ gen3 = coro()
+ gen4 = coro()
+ fut = asyncio.gather(gen3, gen4, loop=self.other_loop)
+ self.assertIs(fut._loop, self.other_loop)
+ gen3.close()
+ gen4.close()
+
+ def test_duplicate_coroutines(self):
+ @asyncio.coroutine
+ def coro(s):
+ return s
+ c = coro('abc')
+ fut = asyncio.gather(c, c, coro('def'), c, loop=self.one_loop)
+ self._run_loop(self.one_loop)
+ self.assertEqual(fut.result(), ['abc', 'abc', 'def', 'abc'])
+
+ def test_cancellation_broadcast(self):
+ # Cancelling outer() cancels all children.
+ proof = 0
+ waiter = asyncio.Future(loop=self.one_loop)
+
+ @asyncio.coroutine
+ def inner():
+ nonlocal proof
+ yield from waiter
+ proof += 1
+
+ child1 = asyncio.async(inner(), loop=self.one_loop)
+ child2 = asyncio.async(inner(), loop=self.one_loop)
+ gatherer = None
+
+ @asyncio.coroutine
+ def outer():
+ nonlocal proof, gatherer
+ gatherer = asyncio.gather(child1, child2, loop=self.one_loop)
+ yield from gatherer
+ proof += 100
+
+ f = asyncio.async(outer(), loop=self.one_loop)
+ test_utils.run_briefly(self.one_loop)
+ self.assertTrue(f.cancel())
+ with self.assertRaises(asyncio.CancelledError):
+ self.one_loop.run_until_complete(f)
+ self.assertFalse(gatherer.cancel())
+ self.assertTrue(waiter.cancelled())
+ self.assertTrue(child1.cancelled())
+ self.assertTrue(child2.cancelled())
+ test_utils.run_briefly(self.one_loop)
+ self.assertEqual(proof, 0)
+
+ def test_exception_marking(self):
+ # Test for the first line marked "Mark exception retrieved."
+
+ @asyncio.coroutine
+ def inner(f):
+ yield from f
+ raise RuntimeError('should not be ignored')
+
+ a = asyncio.Future(loop=self.one_loop)
+ b = asyncio.Future(loop=self.one_loop)
+
+ @asyncio.coroutine
+ def outer():
+ yield from asyncio.gather(inner(a), inner(b), loop=self.one_loop)
+
+ f = asyncio.async(outer(), loop=self.one_loop)
+ test_utils.run_briefly(self.one_loop)
+ a.set_result(None)
+ test_utils.run_briefly(self.one_loop)
+ b.set_result(None)
+ test_utils.run_briefly(self.one_loop)
+ self.assertIsInstance(f.exception(), RuntimeError)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py
new file mode 100644
index 0000000..cfbdf3e
--- /dev/null
+++ b/Lib/test/test_asyncio/test_transports.py
@@ -0,0 +1,88 @@
+"""Tests for transports.py."""
+
+import unittest
+from unittest import mock
+
+import asyncio
+from asyncio import transports
+
+
+class TransportTests(unittest.TestCase):
+
+ def test_ctor_extra_is_none(self):
+ transport = asyncio.Transport()
+ self.assertEqual(transport._extra, {})
+
+ def test_get_extra_info(self):
+ transport = asyncio.Transport({'extra': 'info'})
+ self.assertEqual('info', transport.get_extra_info('extra'))
+ self.assertIsNone(transport.get_extra_info('unknown'))
+
+ default = object()
+ self.assertIs(default, transport.get_extra_info('unknown', default))
+
+ def test_writelines(self):
+ transport = asyncio.Transport()
+ transport.write = mock.Mock()
+
+ transport.writelines([b'line1',
+ bytearray(b'line2'),
+ memoryview(b'line3')])
+ self.assertEqual(1, transport.write.call_count)
+ transport.write.assert_called_with(b'line1line2line3')
+
+ def test_not_implemented(self):
+ transport = asyncio.Transport()
+
+ self.assertRaises(NotImplementedError,
+ transport.set_write_buffer_limits)
+ self.assertRaises(NotImplementedError, transport.get_write_buffer_size)
+ self.assertRaises(NotImplementedError, transport.write, 'data')
+ self.assertRaises(NotImplementedError, transport.write_eof)
+ self.assertRaises(NotImplementedError, transport.can_write_eof)
+ self.assertRaises(NotImplementedError, transport.pause_reading)
+ self.assertRaises(NotImplementedError, transport.resume_reading)
+ self.assertRaises(NotImplementedError, transport.close)
+ self.assertRaises(NotImplementedError, transport.abort)
+
+ def test_dgram_not_implemented(self):
+ transport = asyncio.DatagramTransport()
+
+ self.assertRaises(NotImplementedError, transport.sendto, 'data')
+ self.assertRaises(NotImplementedError, transport.abort)
+
+ def test_subprocess_transport_not_implemented(self):
+ transport = asyncio.SubprocessTransport()
+
+ self.assertRaises(NotImplementedError, transport.get_pid)
+ self.assertRaises(NotImplementedError, transport.get_returncode)
+ self.assertRaises(NotImplementedError, transport.get_pipe_transport, 1)
+ self.assertRaises(NotImplementedError, transport.send_signal, 1)
+ self.assertRaises(NotImplementedError, transport.terminate)
+ self.assertRaises(NotImplementedError, transport.kill)
+
+ def test_flowcontrol_mixin_set_write_limits(self):
+
+ class MyTransport(transports._FlowControlMixin,
+ transports.Transport):
+
+ def get_write_buffer_size(self):
+ return 512
+
+ transport = MyTransport()
+ transport._protocol = mock.Mock()
+
+ self.assertFalse(transport._protocol_paused)
+
+ with self.assertRaisesRegex(ValueError, 'high.*must be >= low'):
+ transport.set_write_buffer_limits(high=0, low=1)
+
+ transport.set_write_buffer_limits(high=1024, low=128)
+ self.assertFalse(transport._protocol_paused)
+
+ transport.set_write_buffer_limits(high=256, low=128)
+ self.assertTrue(transport._protocol_paused)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py
new file mode 100644
index 0000000..cc74383
--- /dev/null
+++ b/Lib/test/test_asyncio/test_unix_events.py
@@ -0,0 +1,1580 @@
+"""Tests for unix_events.py."""
+
+import collections
+import gc
+import errno
+import io
+import os
+import pprint
+import signal
+import socket
+import stat
+import sys
+import tempfile
+import threading
+import unittest
+from unittest import mock
+
+if sys.platform == 'win32':
+ raise unittest.SkipTest('UNIX only')
+
+
+import asyncio
+from asyncio import log
+from asyncio import test_utils
+from asyncio import unix_events
+
+
+MOCK_ANY = mock.ANY
+
+
+@unittest.skipUnless(signal, 'Signals are not supported')
+class SelectorEventLoopSignalTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = asyncio.SelectorEventLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_check_signal(self):
+ self.assertRaises(
+ TypeError, self.loop._check_signal, '1')
+ self.assertRaises(
+ ValueError, self.loop._check_signal, signal.NSIG + 1)
+
+ def test_handle_signal_no_handler(self):
+ self.loop._handle_signal(signal.NSIG + 1, ())
+
+ def test_handle_signal_cancelled_handler(self):
+ h = asyncio.Handle(mock.Mock(), (),
+ loop=mock.Mock())
+ h.cancel()
+ self.loop._signal_handlers[signal.NSIG + 1] = h
+ self.loop.remove_signal_handler = mock.Mock()
+ self.loop._handle_signal(signal.NSIG + 1, ())
+ self.loop.remove_signal_handler.assert_called_with(signal.NSIG + 1)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_add_signal_handler_setup_error(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+ m_signal.set_wakeup_fd.side_effect = ValueError
+
+ self.assertRaises(
+ RuntimeError,
+ self.loop.add_signal_handler,
+ signal.SIGINT, lambda: True)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_add_signal_handler(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+
+ cb = lambda: True
+ self.loop.add_signal_handler(signal.SIGHUP, cb)
+ h = self.loop._signal_handlers.get(signal.SIGHUP)
+ self.assertIsInstance(h, asyncio.Handle)
+ self.assertEqual(h._callback, cb)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_add_signal_handler_install_error(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+
+ def set_wakeup_fd(fd):
+ if fd == -1:
+ raise ValueError()
+ m_signal.set_wakeup_fd = set_wakeup_fd
+
+ class Err(OSError):
+ errno = errno.EFAULT
+ m_signal.signal.side_effect = Err
+
+ self.assertRaises(
+ Err,
+ self.loop.add_signal_handler,
+ signal.SIGINT, lambda: True)
+
+ @mock.patch('asyncio.unix_events.signal')
+ @mock.patch('asyncio.base_events.logger')
+ def test_add_signal_handler_install_error2(self, m_logging, m_signal):
+ m_signal.NSIG = signal.NSIG
+
+ class Err(OSError):
+ errno = errno.EINVAL
+ m_signal.signal.side_effect = Err
+
+ self.loop._signal_handlers[signal.SIGHUP] = lambda: True
+ self.assertRaises(
+ RuntimeError,
+ self.loop.add_signal_handler,
+ signal.SIGINT, lambda: True)
+ self.assertFalse(m_logging.info.called)
+ self.assertEqual(1, m_signal.set_wakeup_fd.call_count)
+
+ @mock.patch('asyncio.unix_events.signal')
+ @mock.patch('asyncio.base_events.logger')
+ def test_add_signal_handler_install_error3(self, m_logging, m_signal):
+ class Err(OSError):
+ errno = errno.EINVAL
+ m_signal.signal.side_effect = Err
+ m_signal.NSIG = signal.NSIG
+
+ self.assertRaises(
+ RuntimeError,
+ self.loop.add_signal_handler,
+ signal.SIGINT, lambda: True)
+ self.assertFalse(m_logging.info.called)
+ self.assertEqual(2, m_signal.set_wakeup_fd.call_count)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_remove_signal_handler(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+
+ self.loop.add_signal_handler(signal.SIGHUP, lambda: True)
+
+ self.assertTrue(
+ self.loop.remove_signal_handler(signal.SIGHUP))
+ self.assertTrue(m_signal.set_wakeup_fd.called)
+ self.assertTrue(m_signal.signal.called)
+ self.assertEqual(
+ (signal.SIGHUP, m_signal.SIG_DFL), m_signal.signal.call_args[0])
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_remove_signal_handler_2(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+ m_signal.SIGINT = signal.SIGINT
+
+ self.loop.add_signal_handler(signal.SIGINT, lambda: True)
+ self.loop._signal_handlers[signal.SIGHUP] = object()
+ m_signal.set_wakeup_fd.reset_mock()
+
+ self.assertTrue(
+ self.loop.remove_signal_handler(signal.SIGINT))
+ self.assertFalse(m_signal.set_wakeup_fd.called)
+ self.assertTrue(m_signal.signal.called)
+ self.assertEqual(
+ (signal.SIGINT, m_signal.default_int_handler),
+ m_signal.signal.call_args[0])
+
+ @mock.patch('asyncio.unix_events.signal')
+ @mock.patch('asyncio.base_events.logger')
+ def test_remove_signal_handler_cleanup_error(self, m_logging, m_signal):
+ m_signal.NSIG = signal.NSIG
+ self.loop.add_signal_handler(signal.SIGHUP, lambda: True)
+
+ m_signal.set_wakeup_fd.side_effect = ValueError
+
+ self.loop.remove_signal_handler(signal.SIGHUP)
+ self.assertTrue(m_logging.info)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_remove_signal_handler_error(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+ self.loop.add_signal_handler(signal.SIGHUP, lambda: True)
+
+ m_signal.signal.side_effect = OSError
+
+ self.assertRaises(
+ OSError, self.loop.remove_signal_handler, signal.SIGHUP)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_remove_signal_handler_error2(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+ self.loop.add_signal_handler(signal.SIGHUP, lambda: True)
+
+ class Err(OSError):
+ errno = errno.EINVAL
+ m_signal.signal.side_effect = Err
+
+ self.assertRaises(
+ RuntimeError, self.loop.remove_signal_handler, signal.SIGHUP)
+
+ @mock.patch('asyncio.unix_events.signal')
+ def test_close(self, m_signal):
+ m_signal.NSIG = signal.NSIG
+
+ self.loop.add_signal_handler(signal.SIGHUP, lambda: True)
+ self.loop.add_signal_handler(signal.SIGCHLD, lambda: True)
+
+ self.assertEqual(len(self.loop._signal_handlers), 2)
+
+ m_signal.set_wakeup_fd.reset_mock()
+
+ self.loop.close()
+
+ self.assertEqual(len(self.loop._signal_handlers), 0)
+ m_signal.set_wakeup_fd.assert_called_once_with(-1)
+
+
+@unittest.skipUnless(hasattr(socket, 'AF_UNIX'),
+ 'UNIX Sockets are not supported')
+class SelectorEventLoopUnixSocketTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = asyncio.SelectorEventLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+
+ def test_create_unix_server_existing_path_sock(self):
+ with test_utils.unix_socket_path() as path:
+ sock = socket.socket(socket.AF_UNIX)
+ sock.bind(path)
+ with sock:
+ coro = self.loop.create_unix_server(lambda: None, path)
+ with self.assertRaisesRegex(OSError,
+ 'Address.*is already in use'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_server_existing_path_nonsock(self):
+ with tempfile.NamedTemporaryFile() as file:
+ coro = self.loop.create_unix_server(lambda: None, file.name)
+ with self.assertRaisesRegex(OSError,
+ 'Address.*is already in use'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_server_ssl_bool(self):
+ coro = self.loop.create_unix_server(lambda: None, path='spam',
+ ssl=True)
+ with self.assertRaisesRegex(TypeError,
+ 'ssl argument must be an SSLContext'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_server_nopath_nosock(self):
+ coro = self.loop.create_unix_server(lambda: None, path=None)
+ with self.assertRaisesRegex(ValueError,
+ 'path was not specified, and no sock'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_server_path_inetsock(self):
+ sock = socket.socket()
+ with sock:
+ coro = self.loop.create_unix_server(lambda: None, path=None,
+ sock=sock)
+ with self.assertRaisesRegex(ValueError,
+ 'A UNIX Domain Socket was expected'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_connection_path_sock(self):
+ coro = self.loop.create_unix_connection(
+ lambda: None, '/dev/null', sock=object())
+ with self.assertRaisesRegex(ValueError, 'path and sock can not be'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_connection_nopath_nosock(self):
+ coro = self.loop.create_unix_connection(
+ lambda: None, None)
+ with self.assertRaisesRegex(ValueError,
+ 'no path and sock were specified'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_connection_nossl_serverhost(self):
+ coro = self.loop.create_unix_connection(
+ lambda: None, '/dev/null', server_hostname='spam')
+ with self.assertRaisesRegex(ValueError,
+ 'server_hostname is only meaningful'):
+ self.loop.run_until_complete(coro)
+
+ def test_create_unix_connection_ssl_noserverhost(self):
+ coro = self.loop.create_unix_connection(
+ lambda: None, '/dev/null', ssl=True)
+
+ with self.assertRaisesRegex(
+ ValueError, 'you have to pass server_hostname when using ssl'):
+
+ self.loop.run_until_complete(coro)
+
+
+class UnixReadPipeTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.Protocol)
+ self.pipe = mock.Mock(spec_set=io.RawIOBase)
+ self.pipe.fileno.return_value = 5
+
+ fcntl_patcher = mock.patch('fcntl.fcntl')
+ fcntl_patcher.start()
+ self.addCleanup(fcntl_patcher.stop)
+
+ fstat_patcher = mock.patch('os.fstat')
+ m_fstat = fstat_patcher.start()
+ st = mock.Mock()
+ st.st_mode = stat.S_IFIFO
+ m_fstat.return_value = st
+ self.addCleanup(fstat_patcher.stop)
+
+ def test_ctor(self):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+ self.loop.assert_reader(5, tr._read_ready)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_made.assert_called_with(tr)
+
+ def test_ctor_with_waiter(self):
+ fut = asyncio.Future(loop=self.loop)
+ unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol, fut)
+ test_utils.run_briefly(self.loop)
+ self.assertIsNone(fut.result())
+
+ @mock.patch('os.read')
+ def test__read_ready(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+ m_read.return_value = b'data'
+ tr._read_ready()
+
+ m_read.assert_called_with(5, tr.max_size)
+ self.protocol.data_received.assert_called_with(b'data')
+
+ @mock.patch('os.read')
+ def test__read_ready_eof(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+ m_read.return_value = b''
+ tr._read_ready()
+
+ m_read.assert_called_with(5, tr.max_size)
+ self.assertFalse(self.loop.readers)
+ test_utils.run_briefly(self.loop)
+ self.protocol.eof_received.assert_called_with()
+ self.protocol.connection_lost.assert_called_with(None)
+
+ @mock.patch('os.read')
+ def test__read_ready_blocked(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+ m_read.side_effect = BlockingIOError
+ tr._read_ready()
+
+ m_read.assert_called_with(5, tr.max_size)
+ test_utils.run_briefly(self.loop)
+ self.assertFalse(self.protocol.data_received.called)
+
+ @mock.patch('asyncio.log.logger.error')
+ @mock.patch('os.read')
+ def test__read_ready_error(self, m_read, m_logexc):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+ err = OSError()
+ m_read.side_effect = err
+ tr._close = mock.Mock()
+ tr._read_ready()
+
+ m_read.assert_called_with(5, tr.max_size)
+ tr._close.assert_called_with(err)
+ m_logexc.assert_called_with(
+ test_utils.MockPattern(
+ 'Fatal read error on pipe transport'
+ '\nprotocol:.*\ntransport:.*'),
+ exc_info=(OSError, MOCK_ANY, MOCK_ANY))
+
+ @mock.patch('os.read')
+ def test_pause_reading(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ m = mock.Mock()
+ self.loop.add_reader(5, m)
+ tr.pause_reading()
+ self.assertFalse(self.loop.readers)
+
+ @mock.patch('os.read')
+ def test_resume_reading(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr.resume_reading()
+ self.loop.assert_reader(5, tr._read_ready)
+
+ @mock.patch('os.read')
+ def test_close(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr._close = mock.Mock()
+ tr.close()
+ tr._close.assert_called_with(None)
+
+ @mock.patch('os.read')
+ def test_close_already_closing(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr._closing = True
+ tr._close = mock.Mock()
+ tr.close()
+ self.assertFalse(tr._close.called)
+
+ @mock.patch('os.read')
+ def test__close(self, m_read):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = object()
+ tr._close(err)
+ self.assertTrue(tr._closing)
+ self.assertFalse(self.loop.readers)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(err)
+
+ def test__call_connection_lost(self):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = None
+ tr._call_connection_lost(err)
+ self.protocol.connection_lost.assert_called_with(err)
+ self.pipe.close.assert_called_with()
+
+ self.assertIsNone(tr._protocol)
+ self.assertEqual(2, sys.getrefcount(self.protocol),
+ pprint.pformat(gc.get_referrers(self.protocol)))
+ self.assertIsNone(tr._loop)
+ self.assertEqual(4, sys.getrefcount(self.loop),
+ pprint.pformat(gc.get_referrers(self.loop)))
+
+ def test__call_connection_lost_with_err(self):
+ tr = unix_events._UnixReadPipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = OSError()
+ tr._call_connection_lost(err)
+ self.protocol.connection_lost.assert_called_with(err)
+ self.pipe.close.assert_called_with()
+
+ self.assertIsNone(tr._protocol)
+
+ self.assertEqual(2, sys.getrefcount(self.protocol),
+ pprint.pformat(gc.get_referrers(self.protocol)))
+ self.assertIsNone(tr._loop)
+ self.assertEqual(4, sys.getrefcount(self.loop),
+ pprint.pformat(gc.get_referrers(self.loop)))
+
+
+class UnixWritePipeTransportTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.protocol = test_utils.make_test_protocol(asyncio.BaseProtocol)
+ self.pipe = mock.Mock(spec_set=io.RawIOBase)
+ self.pipe.fileno.return_value = 5
+
+ fcntl_patcher = mock.patch('fcntl.fcntl')
+ fcntl_patcher.start()
+ self.addCleanup(fcntl_patcher.stop)
+
+ fstat_patcher = mock.patch('os.fstat')
+ m_fstat = fstat_patcher.start()
+ st = mock.Mock()
+ st.st_mode = stat.S_IFSOCK
+ m_fstat.return_value = st
+ self.addCleanup(fstat_patcher.stop)
+
+ def test_ctor(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+ self.loop.assert_reader(5, tr._read_ready)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_made.assert_called_with(tr)
+
+ def test_ctor_with_waiter(self):
+ fut = asyncio.Future(loop=self.loop)
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol, fut)
+ self.loop.assert_reader(5, tr._read_ready)
+ test_utils.run_briefly(self.loop)
+ self.assertEqual(None, fut.result())
+
+ def test_can_write_eof(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+ self.assertTrue(tr.can_write_eof())
+
+ @mock.patch('os.write')
+ def test_write(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ m_write.return_value = 4
+ tr.write(b'data')
+ m_write.assert_called_with(5, b'data')
+ self.assertFalse(self.loop.writers)
+ self.assertEqual([], tr._buffer)
+
+ @mock.patch('os.write')
+ def test_write_no_data(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr.write(b'')
+ self.assertFalse(m_write.called)
+ self.assertFalse(self.loop.writers)
+ self.assertEqual([], tr._buffer)
+
+ @mock.patch('os.write')
+ def test_write_partial(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ m_write.return_value = 2
+ tr.write(b'data')
+ m_write.assert_called_with(5, b'data')
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'ta'], tr._buffer)
+
+ @mock.patch('os.write')
+ def test_write_buffer(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'previous']
+ tr.write(b'data')
+ self.assertFalse(m_write.called)
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'previous', b'data'], tr._buffer)
+
+ @mock.patch('os.write')
+ def test_write_again(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ m_write.side_effect = BlockingIOError()
+ tr.write(b'data')
+ m_write.assert_called_with(5, b'data')
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'data'], tr._buffer)
+
+ @mock.patch('asyncio.unix_events.logger')
+ @mock.patch('os.write')
+ def test_write_err(self, m_write, m_log):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = OSError()
+ m_write.side_effect = err
+ tr._fatal_error = mock.Mock()
+ tr.write(b'data')
+ m_write.assert_called_with(5, b'data')
+ self.assertFalse(self.loop.writers)
+ self.assertEqual([], tr._buffer)
+ tr._fatal_error.assert_called_with(
+ err,
+ 'Fatal write error on pipe transport')
+ self.assertEqual(1, tr._conn_lost)
+
+ tr.write(b'data')
+ self.assertEqual(2, tr._conn_lost)
+ tr.write(b'data')
+ tr.write(b'data')
+ tr.write(b'data')
+ tr.write(b'data')
+ # This is a bit overspecified. :-(
+ m_log.warning.assert_called_with(
+ 'pipe closed by peer or os.write(pipe, data) raised exception.')
+
+ @mock.patch('os.write')
+ def test_write_close(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+ tr._read_ready() # pipe was closed by peer
+
+ tr.write(b'data')
+ self.assertEqual(tr._conn_lost, 1)
+ tr.write(b'data')
+ self.assertEqual(tr._conn_lost, 2)
+
+ def test__read_ready(self):
+ tr = unix_events._UnixWritePipeTransport(self.loop, self.pipe,
+ self.protocol)
+ tr._read_ready()
+ self.assertFalse(self.loop.readers)
+ self.assertFalse(self.loop.writers)
+ self.assertTrue(tr._closing)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ @mock.patch('os.write')
+ def test__write_ready(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'da', b'ta']
+ m_write.return_value = 4
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.assertFalse(self.loop.writers)
+ self.assertEqual([], tr._buffer)
+
+ @mock.patch('os.write')
+ def test__write_ready_partial(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'da', b'ta']
+ m_write.return_value = 3
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'a'], tr._buffer)
+
+ @mock.patch('os.write')
+ def test__write_ready_again(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'da', b'ta']
+ m_write.side_effect = BlockingIOError()
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'data'], tr._buffer)
+
+ @mock.patch('os.write')
+ def test__write_ready_empty(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'da', b'ta']
+ m_write.return_value = 0
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.loop.assert_writer(5, tr._write_ready)
+ self.assertEqual([b'data'], tr._buffer)
+
+ @mock.patch('asyncio.log.logger.error')
+ @mock.patch('os.write')
+ def test__write_ready_err(self, m_write, m_logexc):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._buffer = [b'da', b'ta']
+ m_write.side_effect = err = OSError()
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.assertFalse(self.loop.writers)
+ self.assertFalse(self.loop.readers)
+ self.assertEqual([], tr._buffer)
+ self.assertTrue(tr._closing)
+ m_logexc.assert_called_with(
+ test_utils.MockPattern(
+ 'Fatal write error on pipe transport'
+ '\nprotocol:.*\ntransport:.*'),
+ exc_info=(OSError, MOCK_ANY, MOCK_ANY))
+ self.assertEqual(1, tr._conn_lost)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(err)
+
+ @mock.patch('os.write')
+ def test__write_ready_closing(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ tr._closing = True
+ tr._buffer = [b'da', b'ta']
+ m_write.return_value = 4
+ tr._write_ready()
+ m_write.assert_called_with(5, b'data')
+ self.assertFalse(self.loop.writers)
+ self.assertFalse(self.loop.readers)
+ self.assertEqual([], tr._buffer)
+ self.protocol.connection_lost.assert_called_with(None)
+ self.pipe.close.assert_called_with()
+
+ @mock.patch('os.write')
+ def test_abort(self, m_write):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ self.loop.add_writer(5, tr._write_ready)
+ self.loop.add_reader(5, tr._read_ready)
+ tr._buffer = [b'da', b'ta']
+ tr.abort()
+ self.assertFalse(m_write.called)
+ self.assertFalse(self.loop.readers)
+ self.assertFalse(self.loop.writers)
+ self.assertEqual([], tr._buffer)
+ self.assertTrue(tr._closing)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test__call_connection_lost(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = None
+ tr._call_connection_lost(err)
+ self.protocol.connection_lost.assert_called_with(err)
+ self.pipe.close.assert_called_with()
+
+ self.assertIsNone(tr._protocol)
+ self.assertEqual(2, sys.getrefcount(self.protocol),
+ pprint.pformat(gc.get_referrers(self.protocol)))
+ self.assertIsNone(tr._loop)
+ self.assertEqual(4, sys.getrefcount(self.loop),
+ pprint.pformat(gc.get_referrers(self.loop)))
+
+ def test__call_connection_lost_with_err(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ err = OSError()
+ tr._call_connection_lost(err)
+ self.protocol.connection_lost.assert_called_with(err)
+ self.pipe.close.assert_called_with()
+
+ self.assertIsNone(tr._protocol)
+ self.assertEqual(2, sys.getrefcount(self.protocol),
+ pprint.pformat(gc.get_referrers(self.protocol)))
+ self.assertIsNone(tr._loop)
+ self.assertEqual(4, sys.getrefcount(self.loop),
+ pprint.pformat(gc.get_referrers(self.loop)))
+
+ def test_close(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr.write_eof = mock.Mock()
+ tr.close()
+ tr.write_eof.assert_called_with()
+
+ def test_close_closing(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr.write_eof = mock.Mock()
+ tr._closing = True
+ tr.close()
+ self.assertFalse(tr.write_eof.called)
+
+ def test_write_eof(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+
+ tr.write_eof()
+ self.assertTrue(tr._closing)
+ self.assertFalse(self.loop.readers)
+ test_utils.run_briefly(self.loop)
+ self.protocol.connection_lost.assert_called_with(None)
+
+ def test_write_eof_pending(self):
+ tr = unix_events._UnixWritePipeTransport(
+ self.loop, self.pipe, self.protocol)
+ tr._buffer = [b'data']
+ tr.write_eof()
+ self.assertTrue(tr._closing)
+ self.assertFalse(self.protocol.connection_lost.called)
+
+
+class AbstractChildWatcherTests(unittest.TestCase):
+
+ def test_not_implemented(self):
+ f = mock.Mock()
+ watcher = asyncio.AbstractChildWatcher()
+ self.assertRaises(
+ NotImplementedError, watcher.add_child_handler, f, f)
+ self.assertRaises(
+ NotImplementedError, watcher.remove_child_handler, f)
+ self.assertRaises(
+ NotImplementedError, watcher.attach_loop, f)
+ self.assertRaises(
+ NotImplementedError, watcher.close)
+ self.assertRaises(
+ NotImplementedError, watcher.__enter__)
+ self.assertRaises(
+ NotImplementedError, watcher.__exit__, f, f, f)
+
+
+class BaseChildWatcherTests(unittest.TestCase):
+
+ def test_not_implemented(self):
+ f = mock.Mock()
+ watcher = unix_events.BaseChildWatcher()
+ self.assertRaises(
+ NotImplementedError, watcher._do_waitpid, f)
+
+
+WaitPidMocks = collections.namedtuple("WaitPidMocks",
+ ("waitpid",
+ "WIFEXITED",
+ "WIFSIGNALED",
+ "WEXITSTATUS",
+ "WTERMSIG",
+ ))
+
+
+class ChildWatcherTestsMixin:
+
+ ignore_warnings = mock.patch.object(log.logger, "warning")
+
+ def setUp(self):
+ self.loop = test_utils.TestLoop()
+ self.running = False
+ self.zombies = {}
+
+ with mock.patch.object(
+ self.loop, "add_signal_handler") as self.m_add_signal_handler:
+ self.watcher = self.create_watcher()
+ self.watcher.attach_loop(self.loop)
+
+ def waitpid(self, pid, flags):
+ if isinstance(self.watcher, asyncio.SafeChildWatcher) or pid != -1:
+ self.assertGreater(pid, 0)
+ try:
+ if pid < 0:
+ return self.zombies.popitem()
+ else:
+ return pid, self.zombies.pop(pid)
+ except KeyError:
+ pass
+ if self.running:
+ return 0, 0
+ else:
+ raise ChildProcessError()
+
+ def add_zombie(self, pid, returncode):
+ self.zombies[pid] = returncode + 32768
+
+ def WIFEXITED(self, status):
+ return status >= 32768
+
+ def WIFSIGNALED(self, status):
+ return 32700 < status < 32768
+
+ def WEXITSTATUS(self, status):
+ self.assertTrue(self.WIFEXITED(status))
+ return status - 32768
+
+ def WTERMSIG(self, status):
+ self.assertTrue(self.WIFSIGNALED(status))
+ return 32768 - status
+
+ def test_create_watcher(self):
+ self.m_add_signal_handler.assert_called_once_with(
+ signal.SIGCHLD, self.watcher._sig_chld)
+
+ def waitpid_mocks(func):
+ def wrapped_func(self):
+ def patch(target, wrapper):
+ return mock.patch(target, wraps=wrapper,
+ new_callable=mock.Mock)
+
+ with patch('os.WTERMSIG', self.WTERMSIG) as m_WTERMSIG, \
+ patch('os.WEXITSTATUS', self.WEXITSTATUS) as m_WEXITSTATUS, \
+ patch('os.WIFSIGNALED', self.WIFSIGNALED) as m_WIFSIGNALED, \
+ patch('os.WIFEXITED', self.WIFEXITED) as m_WIFEXITED, \
+ patch('os.waitpid', self.waitpid) as m_waitpid:
+ func(self, WaitPidMocks(m_waitpid,
+ m_WIFEXITED, m_WIFSIGNALED,
+ m_WEXITSTATUS, m_WTERMSIG,
+ ))
+ return wrapped_func
+
+ @waitpid_mocks
+ def test_sigchld(self, m):
+ # register a child
+ callback = mock.Mock()
+
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(42, callback, 9, 10, 14)
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child is running
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child terminates (returncode 12)
+ self.running = False
+ self.add_zombie(42, 12)
+ self.watcher._sig_chld()
+
+ self.assertTrue(m.WIFEXITED.called)
+ self.assertTrue(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+ callback.assert_called_once_with(42, 12, 9, 10, 14)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WEXITSTATUS.reset_mock()
+ callback.reset_mock()
+
+ # ensure that the child is effectively reaped
+ self.add_zombie(42, 13)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WEXITSTATUS.reset_mock()
+
+ # sigchld called again
+ self.zombies.clear()
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ @waitpid_mocks
+ def test_sigchld_two_children(self, m):
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+
+ # register child 1
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(43, callback1, 7, 8)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # register child 2
+ with self.watcher:
+ self.watcher.add_child_handler(44, callback2, 147, 18)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # children are running
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child 1 terminates (signal 3)
+ self.add_zombie(43, -3)
+ self.watcher._sig_chld()
+
+ callback1.assert_called_once_with(43, -3, 7, 8)
+ self.assertFalse(callback2.called)
+ self.assertTrue(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertTrue(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WTERMSIG.reset_mock()
+ callback1.reset_mock()
+
+ # child 2 still running
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child 2 terminates (code 108)
+ self.add_zombie(44, 108)
+ self.running = False
+ self.watcher._sig_chld()
+
+ callback2.assert_called_once_with(44, 108, 147, 18)
+ self.assertFalse(callback1.called)
+ self.assertTrue(m.WIFEXITED.called)
+ self.assertTrue(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WEXITSTATUS.reset_mock()
+ callback2.reset_mock()
+
+ # ensure that the children are effectively reaped
+ self.add_zombie(43, 14)
+ self.add_zombie(44, 15)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WEXITSTATUS.reset_mock()
+
+ # sigchld called again
+ self.zombies.clear()
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ @waitpid_mocks
+ def test_sigchld_two_children_terminating_together(self, m):
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+
+ # register child 1
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(45, callback1, 17, 8)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # register child 2
+ with self.watcher:
+ self.watcher.add_child_handler(46, callback2, 1147, 18)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # children are running
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child 1 terminates (code 78)
+ # child 2 terminates (signal 5)
+ self.add_zombie(45, 78)
+ self.add_zombie(46, -5)
+ self.running = False
+ self.watcher._sig_chld()
+
+ callback1.assert_called_once_with(45, 78, 17, 8)
+ callback2.assert_called_once_with(46, -5, 1147, 18)
+ self.assertTrue(m.WIFSIGNALED.called)
+ self.assertTrue(m.WIFEXITED.called)
+ self.assertTrue(m.WEXITSTATUS.called)
+ self.assertTrue(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WTERMSIG.reset_mock()
+ m.WEXITSTATUS.reset_mock()
+ callback1.reset_mock()
+ callback2.reset_mock()
+
+ # ensure that the children are effectively reaped
+ self.add_zombie(45, 14)
+ self.add_zombie(46, 15)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ @waitpid_mocks
+ def test_sigchld_race_condition(self, m):
+ # register a child
+ callback = mock.Mock()
+
+ with self.watcher:
+ # child terminates before being registered
+ self.add_zombie(50, 4)
+ self.watcher._sig_chld()
+
+ self.watcher.add_child_handler(50, callback, 1, 12)
+
+ callback.assert_called_once_with(50, 4, 1, 12)
+ callback.reset_mock()
+
+ # ensure that the child is effectively reaped
+ self.add_zombie(50, -1)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+
+ @waitpid_mocks
+ def test_sigchld_replace_handler(self, m):
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+
+ # register a child
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(51, callback1, 19)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # register the same child again
+ with self.watcher:
+ self.watcher.add_child_handler(51, callback2, 21)
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child terminates (signal 8)
+ self.running = False
+ self.add_zombie(51, -8)
+ self.watcher._sig_chld()
+
+ callback2.assert_called_once_with(51, -8, 21)
+ self.assertFalse(callback1.called)
+ self.assertTrue(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertTrue(m.WTERMSIG.called)
+
+ m.WIFSIGNALED.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WTERMSIG.reset_mock()
+ callback2.reset_mock()
+
+ # ensure that the child is effectively reaped
+ self.add_zombie(51, 13)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ @waitpid_mocks
+ def test_sigchld_remove_handler(self, m):
+ callback = mock.Mock()
+
+ # register a child
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(52, callback, 1984)
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # unregister the child
+ self.watcher.remove_child_handler(52)
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child terminates (code 99)
+ self.running = False
+ self.add_zombie(52, 99)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+
+ @waitpid_mocks
+ def test_sigchld_unknown_status(self, m):
+ callback = mock.Mock()
+
+ # register a child
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(53, callback, -19)
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # terminate with unknown status
+ self.zombies[53] = 1178
+ self.running = False
+ self.watcher._sig_chld()
+
+ callback.assert_called_once_with(53, 1178, -19)
+ self.assertTrue(m.WIFEXITED.called)
+ self.assertTrue(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ callback.reset_mock()
+ m.WIFEXITED.reset_mock()
+ m.WIFSIGNALED.reset_mock()
+
+ # ensure that the child is effectively reaped
+ self.add_zombie(53, 101)
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback.called)
+
+ @waitpid_mocks
+ def test_remove_child_handler(self, m):
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+ callback3 = mock.Mock()
+
+ # register children
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(54, callback1, 1)
+ self.watcher.add_child_handler(55, callback2, 2)
+ self.watcher.add_child_handler(56, callback3, 3)
+
+ # remove child handler 1
+ self.assertTrue(self.watcher.remove_child_handler(54))
+
+ # remove child handler 2 multiple times
+ self.assertTrue(self.watcher.remove_child_handler(55))
+ self.assertFalse(self.watcher.remove_child_handler(55))
+ self.assertFalse(self.watcher.remove_child_handler(55))
+
+ # all children terminate
+ self.add_zombie(54, 0)
+ self.add_zombie(55, 1)
+ self.add_zombie(56, 2)
+ self.running = False
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ callback3.assert_called_once_with(56, 2, 3)
+
+ @waitpid_mocks
+ def test_sigchld_unhandled_exception(self, m):
+ callback = mock.Mock()
+
+ # register a child
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(57, callback)
+
+ # raise an exception
+ m.waitpid.side_effect = ValueError
+
+ with mock.patch.object(log.logger,
+ 'error') as m_error:
+
+ self.assertEqual(self.watcher._sig_chld(), None)
+ self.assertTrue(m_error.called)
+
+ @waitpid_mocks
+ def test_sigchld_child_reaped_elsewhere(self, m):
+ # register a child
+ callback = mock.Mock()
+
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(58, callback)
+
+ self.assertFalse(callback.called)
+ self.assertFalse(m.WIFEXITED.called)
+ self.assertFalse(m.WIFSIGNALED.called)
+ self.assertFalse(m.WEXITSTATUS.called)
+ self.assertFalse(m.WTERMSIG.called)
+
+ # child terminates
+ self.running = False
+ self.add_zombie(58, 4)
+
+ # waitpid is called elsewhere
+ os.waitpid(58, os.WNOHANG)
+
+ m.waitpid.reset_mock()
+
+ # sigchld
+ with self.ignore_warnings:
+ self.watcher._sig_chld()
+
+ callback.assert_called(m.waitpid)
+ if isinstance(self.watcher, asyncio.FastChildWatcher):
+ # here the FastChildWatche enters a deadlock
+ # (there is no way to prevent it)
+ self.assertFalse(callback.called)
+ else:
+ callback.assert_called_once_with(58, 255)
+
+ @waitpid_mocks
+ def test_sigchld_unknown_pid_during_registration(self, m):
+ # register two children
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+
+ with self.ignore_warnings, self.watcher:
+ self.running = True
+ # child 1 terminates
+ self.add_zombie(591, 7)
+ # an unknown child terminates
+ self.add_zombie(593, 17)
+
+ self.watcher._sig_chld()
+
+ self.watcher.add_child_handler(591, callback1)
+ self.watcher.add_child_handler(592, callback2)
+
+ callback1.assert_called_once_with(591, 7)
+ self.assertFalse(callback2.called)
+
+ @waitpid_mocks
+ def test_set_loop(self, m):
+ # register a child
+ callback = mock.Mock()
+
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(60, callback)
+
+ # attach a new loop
+ old_loop = self.loop
+ self.loop = test_utils.TestLoop()
+ patch = mock.patch.object
+
+ with patch(old_loop, "remove_signal_handler") as m_old_remove, \
+ patch(self.loop, "add_signal_handler") as m_new_add:
+
+ self.watcher.attach_loop(self.loop)
+
+ m_old_remove.assert_called_once_with(
+ signal.SIGCHLD)
+ m_new_add.assert_called_once_with(
+ signal.SIGCHLD, self.watcher._sig_chld)
+
+ # child terminates
+ self.running = False
+ self.add_zombie(60, 9)
+ self.watcher._sig_chld()
+
+ callback.assert_called_once_with(60, 9)
+
+ @waitpid_mocks
+ def test_set_loop_race_condition(self, m):
+ # register 3 children
+ callback1 = mock.Mock()
+ callback2 = mock.Mock()
+ callback3 = mock.Mock()
+
+ with self.watcher:
+ self.running = True
+ self.watcher.add_child_handler(61, callback1)
+ self.watcher.add_child_handler(62, callback2)
+ self.watcher.add_child_handler(622, callback3)
+
+ # detach the loop
+ old_loop = self.loop
+ self.loop = None
+
+ with mock.patch.object(
+ old_loop, "remove_signal_handler") as m_remove_signal_handler:
+
+ self.watcher.attach_loop(None)
+
+ m_remove_signal_handler.assert_called_once_with(
+ signal.SIGCHLD)
+
+ # child 1 & 2 terminate
+ self.add_zombie(61, 11)
+ self.add_zombie(62, -5)
+
+ # SIGCHLD was not caught
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ self.assertFalse(callback3.called)
+
+ # attach a new loop
+ self.loop = test_utils.TestLoop()
+
+ with mock.patch.object(
+ self.loop, "add_signal_handler") as m_add_signal_handler:
+
+ self.watcher.attach_loop(self.loop)
+
+ m_add_signal_handler.assert_called_once_with(
+ signal.SIGCHLD, self.watcher._sig_chld)
+ callback1.assert_called_once_with(61, 11) # race condition!
+ callback2.assert_called_once_with(62, -5) # race condition!
+ self.assertFalse(callback3.called)
+
+ callback1.reset_mock()
+ callback2.reset_mock()
+
+ # child 3 terminates
+ self.running = False
+ self.add_zombie(622, 19)
+ self.watcher._sig_chld()
+
+ self.assertFalse(callback1.called)
+ self.assertFalse(callback2.called)
+ callback3.assert_called_once_with(622, 19)
+
+ @waitpid_mocks
+ def test_close(self, m):
+ # register two children
+ callback1 = mock.Mock()
+
+ with self.watcher:
+ self.running = True
+ # child 1 terminates
+ self.add_zombie(63, 9)
+ # other child terminates
+ self.add_zombie(65, 18)
+ self.watcher._sig_chld()
+
+ self.watcher.add_child_handler(63, callback1)
+ self.watcher.add_child_handler(64, callback1)
+
+ self.assertEqual(len(self.watcher._callbacks), 1)
+ if isinstance(self.watcher, asyncio.FastChildWatcher):
+ self.assertEqual(len(self.watcher._zombies), 1)
+
+ with mock.patch.object(
+ self.loop,
+ "remove_signal_handler") as m_remove_signal_handler:
+
+ self.watcher.close()
+
+ m_remove_signal_handler.assert_called_once_with(
+ signal.SIGCHLD)
+ self.assertFalse(self.watcher._callbacks)
+ if isinstance(self.watcher, asyncio.FastChildWatcher):
+ self.assertFalse(self.watcher._zombies)
+
+
+class SafeChildWatcherTests (ChildWatcherTestsMixin, unittest.TestCase):
+ def create_watcher(self):
+ return asyncio.SafeChildWatcher()
+
+
+class FastChildWatcherTests (ChildWatcherTestsMixin, unittest.TestCase):
+ def create_watcher(self):
+ return asyncio.FastChildWatcher()
+
+
+class PolicyTests(unittest.TestCase):
+
+ def create_policy(self):
+ return asyncio.DefaultEventLoopPolicy()
+
+ def test_get_child_watcher(self):
+ policy = self.create_policy()
+ self.assertIsNone(policy._watcher)
+
+ watcher = policy.get_child_watcher()
+ self.assertIsInstance(watcher, asyncio.SafeChildWatcher)
+
+ self.assertIs(policy._watcher, watcher)
+
+ self.assertIs(watcher, policy.get_child_watcher())
+ self.assertIsNone(watcher._loop)
+
+ def test_get_child_watcher_after_set(self):
+ policy = self.create_policy()
+ watcher = asyncio.FastChildWatcher()
+
+ policy.set_child_watcher(watcher)
+ self.assertIs(policy._watcher, watcher)
+ self.assertIs(watcher, policy.get_child_watcher())
+
+ def test_get_child_watcher_with_mainloop_existing(self):
+ policy = self.create_policy()
+ loop = policy.get_event_loop()
+
+ self.assertIsNone(policy._watcher)
+ watcher = policy.get_child_watcher()
+
+ self.assertIsInstance(watcher, asyncio.SafeChildWatcher)
+ self.assertIs(watcher._loop, loop)
+
+ loop.close()
+
+ def test_get_child_watcher_thread(self):
+
+ def f():
+ policy.set_event_loop(policy.new_event_loop())
+
+ self.assertIsInstance(policy.get_event_loop(),
+ asyncio.AbstractEventLoop)
+ watcher = policy.get_child_watcher()
+
+ self.assertIsInstance(watcher, asyncio.SafeChildWatcher)
+ self.assertIsNone(watcher._loop)
+
+ policy.get_event_loop().close()
+
+ policy = self.create_policy()
+
+ th = threading.Thread(target=f)
+ th.start()
+ th.join()
+
+ def test_child_watcher_replace_mainloop_existing(self):
+ policy = self.create_policy()
+ loop = policy.get_event_loop()
+
+ watcher = policy.get_child_watcher()
+
+ self.assertIs(watcher._loop, loop)
+
+ new_loop = policy.new_event_loop()
+ policy.set_event_loop(new_loop)
+
+ self.assertIs(watcher._loop, new_loop)
+
+ policy.set_event_loop(None)
+
+ self.assertIs(watcher._loop, None)
+
+ loop.close()
+ new_loop.close()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py
new file mode 100644
index 0000000..f652258
--- /dev/null
+++ b/Lib/test/test_asyncio/test_windows_events.py
@@ -0,0 +1,134 @@
+import os
+import sys
+import unittest
+
+if sys.platform != 'win32':
+ raise unittest.SkipTest('Windows only')
+
+import _winapi
+
+import asyncio
+from asyncio import _overlapped
+from asyncio import windows_events
+
+
+class UpperProto(asyncio.Protocol):
+ def __init__(self):
+ self.buf = []
+
+ def connection_made(self, trans):
+ self.trans = trans
+
+ def data_received(self, data):
+ self.buf.append(data)
+ if b'\n' in data:
+ self.trans.write(b''.join(self.buf).upper())
+ self.trans.close()
+
+
+class ProactorTests(unittest.TestCase):
+
+ def setUp(self):
+ self.loop = asyncio.ProactorEventLoop()
+ asyncio.set_event_loop(None)
+
+ def tearDown(self):
+ self.loop.close()
+ self.loop = None
+
+ def test_close(self):
+ a, b = self.loop._socketpair()
+ trans = self.loop._make_socket_transport(a, asyncio.Protocol())
+ f = asyncio.async(self.loop.sock_recv(b, 100))
+ trans.close()
+ self.loop.run_until_complete(f)
+ self.assertEqual(f.result(), b'')
+ b.close()
+
+ def test_double_bind(self):
+ ADDRESS = r'\\.\pipe\test_double_bind-%s' % os.getpid()
+ server1 = windows_events.PipeServer(ADDRESS)
+ with self.assertRaises(PermissionError):
+ windows_events.PipeServer(ADDRESS)
+ server1.close()
+
+ def test_pipe(self):
+ res = self.loop.run_until_complete(self._test_pipe())
+ self.assertEqual(res, 'done')
+
+ def _test_pipe(self):
+ ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid()
+
+ with self.assertRaises(FileNotFoundError):
+ yield from self.loop.create_pipe_connection(
+ asyncio.Protocol, ADDRESS)
+
+ [server] = yield from self.loop.start_serving_pipe(
+ UpperProto, ADDRESS)
+ self.assertIsInstance(server, windows_events.PipeServer)
+
+ clients = []
+ for i in range(5):
+ stream_reader = asyncio.StreamReader(loop=self.loop)
+ protocol = asyncio.StreamReaderProtocol(stream_reader)
+ trans, proto = yield from self.loop.create_pipe_connection(
+ lambda: protocol, ADDRESS)
+ self.assertIsInstance(trans, asyncio.Transport)
+ self.assertEqual(protocol, proto)
+ clients.append((stream_reader, trans))
+
+ for i, (r, w) in enumerate(clients):
+ w.write('lower-{}\n'.format(i).encode())
+
+ for i, (r, w) in enumerate(clients):
+ response = yield from r.readline()
+ self.assertEqual(response, 'LOWER-{}\n'.format(i).encode())
+ w.close()
+
+ server.close()
+
+ with self.assertRaises(FileNotFoundError):
+ yield from self.loop.create_pipe_connection(
+ asyncio.Protocol, ADDRESS)
+
+ return 'done'
+
+ def test_wait_for_handle(self):
+ event = _overlapped.CreateEvent(None, True, False, None)
+ self.addCleanup(_winapi.CloseHandle, event)
+
+ # Wait for unset event with 0.2s timeout;
+ # result should be False at timeout
+ f = self.loop._proactor.wait_for_handle(event, 0.2)
+ start = self.loop.time()
+ self.loop.run_until_complete(f)
+ elapsed = self.loop.time() - start
+ self.assertFalse(f.result())
+ self.assertTrue(0.18 < elapsed < 0.9, elapsed)
+
+ _overlapped.SetEvent(event)
+
+ # Wait for for set event;
+ # result should be True immediately
+ f = self.loop._proactor.wait_for_handle(event, 10)
+ start = self.loop.time()
+ self.loop.run_until_complete(f)
+ elapsed = self.loop.time() - start
+ self.assertTrue(f.result())
+ self.assertTrue(0 <= elapsed < 0.1, elapsed)
+
+ _overlapped.ResetEvent(event)
+
+ # Wait for unset event with a cancelled future;
+ # CancelledError should be raised immediately
+ f = self.loop._proactor.wait_for_handle(event, 10)
+ f.cancel()
+ start = self.loop.time()
+ with self.assertRaises(asyncio.CancelledError):
+ self.loop.run_until_complete(f)
+ elapsed = self.loop.time() - start
+ self.assertTrue(0 <= elapsed < 0.1, elapsed)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py
new file mode 100644
index 0000000..9daf434
--- /dev/null
+++ b/Lib/test/test_asyncio/test_windows_utils.py
@@ -0,0 +1,160 @@
+"""Tests for window_utils"""
+
+import socket
+import sys
+import test.support
+import unittest
+from test.support import IPV6_ENABLED
+from unittest import mock
+
+if sys.platform != 'win32':
+ raise unittest.SkipTest('Windows only')
+
+import _winapi
+
+from asyncio import windows_utils
+from asyncio import _overlapped
+
+
+class WinsocketpairTests(unittest.TestCase):
+
+ def check_winsocketpair(self, ssock, csock):
+ csock.send(b'xxx')
+ self.assertEqual(b'xxx', ssock.recv(1024))
+ csock.close()
+ ssock.close()
+
+ def test_winsocketpair(self):
+ ssock, csock = windows_utils.socketpair()
+ self.check_winsocketpair(ssock, csock)
+
+ @unittest.skipUnless(IPV6_ENABLED, 'IPv6 not supported or enabled')
+ def test_winsocketpair_ipv6(self):
+ ssock, csock = windows_utils.socketpair(family=socket.AF_INET6)
+ self.check_winsocketpair(ssock, csock)
+
+ @mock.patch('asyncio.windows_utils.socket')
+ def test_winsocketpair_exc(self, m_socket):
+ m_socket.AF_INET = socket.AF_INET
+ m_socket.SOCK_STREAM = socket.SOCK_STREAM
+ m_socket.socket.return_value.getsockname.return_value = ('', 12345)
+ m_socket.socket.return_value.accept.return_value = object(), object()
+ m_socket.socket.return_value.connect.side_effect = OSError()
+
+ self.assertRaises(OSError, windows_utils.socketpair)
+
+ def test_winsocketpair_invalid_args(self):
+ self.assertRaises(ValueError,
+ windows_utils.socketpair, family=socket.AF_UNSPEC)
+ self.assertRaises(ValueError,
+ windows_utils.socketpair, type=socket.SOCK_DGRAM)
+ self.assertRaises(ValueError,
+ windows_utils.socketpair, proto=1)
+
+
+
+class PipeTests(unittest.TestCase):
+
+ def test_pipe_overlapped(self):
+ h1, h2 = windows_utils.pipe(overlapped=(True, True))
+ try:
+ ov1 = _overlapped.Overlapped()
+ self.assertFalse(ov1.pending)
+ self.assertEqual(ov1.error, 0)
+
+ ov1.ReadFile(h1, 100)
+ self.assertTrue(ov1.pending)
+ self.assertEqual(ov1.error, _winapi.ERROR_IO_PENDING)
+ ERROR_IO_INCOMPLETE = 996
+ try:
+ ov1.getresult()
+ except OSError as e:
+ self.assertEqual(e.winerror, ERROR_IO_INCOMPLETE)
+ else:
+ raise RuntimeError('expected ERROR_IO_INCOMPLETE')
+
+ ov2 = _overlapped.Overlapped()
+ self.assertFalse(ov2.pending)
+ self.assertEqual(ov2.error, 0)
+
+ ov2.WriteFile(h2, b"hello")
+ self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})
+
+ res = _winapi.WaitForMultipleObjects([ov2.event], False, 100)
+ self.assertEqual(res, _winapi.WAIT_OBJECT_0)
+
+ self.assertFalse(ov1.pending)
+ self.assertEqual(ov1.error, ERROR_IO_INCOMPLETE)
+ self.assertFalse(ov2.pending)
+ self.assertIn(ov2.error, {0, _winapi.ERROR_IO_PENDING})
+ self.assertEqual(ov1.getresult(), b"hello")
+ finally:
+ _winapi.CloseHandle(h1)
+ _winapi.CloseHandle(h2)
+
+ def test_pipe_handle(self):
+ h, _ = windows_utils.pipe(overlapped=(True, True))
+ _winapi.CloseHandle(_)
+ p = windows_utils.PipeHandle(h)
+ self.assertEqual(p.fileno(), h)
+ self.assertEqual(p.handle, h)
+
+ # check garbage collection of p closes handle
+ del p
+ test.support.gc_collect()
+ try:
+ _winapi.CloseHandle(h)
+ except OSError as e:
+ self.assertEqual(e.winerror, 6) # ERROR_INVALID_HANDLE
+ else:
+ raise RuntimeError('expected ERROR_INVALID_HANDLE')
+
+
+class PopenTests(unittest.TestCase):
+
+ def test_popen(self):
+ command = r"""if 1:
+ import sys
+ s = sys.stdin.readline()
+ sys.stdout.write(s.upper())
+ sys.stderr.write('stderr')
+ """
+ msg = b"blah\n"
+
+ p = windows_utils.Popen([sys.executable, '-c', command],
+ stdin=windows_utils.PIPE,
+ stdout=windows_utils.PIPE,
+ stderr=windows_utils.PIPE)
+
+ for f in [p.stdin, p.stdout, p.stderr]:
+ self.assertIsInstance(f, windows_utils.PipeHandle)
+
+ ovin = _overlapped.Overlapped()
+ ovout = _overlapped.Overlapped()
+ overr = _overlapped.Overlapped()
+
+ ovin.WriteFile(p.stdin.handle, msg)
+ ovout.ReadFile(p.stdout.handle, 100)
+ overr.ReadFile(p.stderr.handle, 100)
+
+ events = [ovin.event, ovout.event, overr.event]
+ # Super-long timeout for slow buildbots.
+ res = _winapi.WaitForMultipleObjects(events, True, 10000)
+ self.assertEqual(res, _winapi.WAIT_OBJECT_0)
+ self.assertFalse(ovout.pending)
+ self.assertFalse(overr.pending)
+ self.assertFalse(ovin.pending)
+
+ self.assertEqual(ovin.getresult(), len(msg))
+ out = ovout.getresult().rstrip()
+ err = overr.getresult().rstrip()
+
+ self.assertGreater(len(out), 0)
+ self.assertGreater(len(err), 0)
+ # allow for partial reads...
+ self.assertTrue(msg.upper().rstrip().startswith(out))
+ self.assertTrue(b"stderr".startswith(err))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_asyncio/tests.txt b/Lib/test/test_asyncio/tests.txt
new file mode 100644
index 0000000..c69582b
--- /dev/null
+++ b/Lib/test/test_asyncio/tests.txt
@@ -0,0 +1,14 @@
+test_asyncio.test_base_events
+test_asyncio.test_events
+test_asyncio.test_futures
+test_asyncio.test_locks
+test_asyncio.test_proactor_events
+test_asyncio.test_queues
+test_asyncio.test_selector_events
+test_asyncio.test_streams
+test_asyncio.test_subprocess
+test_asyncio.test_tasks
+test_asyncio.test_transports
+test_asyncio.test_unix_events
+test_asyncio.test_windows_events
+test_asyncio.test_windows_utils
diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py
index 5d0632e..084d247 100644
--- a/Lib/test/test_asyncore.py
+++ b/Lib/test/test_asyncore.py
@@ -19,6 +19,7 @@ try:
except ImportError:
threading = None
+TIMEOUT = 3
HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX')
class dummysocket:
@@ -395,7 +396,10 @@ class DispatcherWithSendTests(unittest.TestCase):
self.assertEqual(cap.getvalue(), data*2)
finally:
- t.join()
+ t.join(timeout=TIMEOUT)
+ if t.is_alive():
+ self.fail("join() timed out")
+
class DispatcherWithSendTests_UsePoll(DispatcherWithSendTests):
@@ -740,7 +744,12 @@ class BaseTestAPI:
s.create_socket(self.family)
self.assertEqual(s.socket.family, self.family)
SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0)
- self.assertEqual(s.socket.type, socket.SOCK_STREAM | SOCK_NONBLOCK)
+ sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK
+ if hasattr(socket, 'SOCK_CLOEXEC'):
+ self.assertIn(s.socket.type,
+ (sock_type | socket.SOCK_CLOEXEC, sock_type))
+ else:
+ self.assertEqual(s.socket.type, sock_type)
def test_bind(self):
if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
@@ -754,7 +763,7 @@ class BaseTestAPI:
s2 = asyncore.dispatcher()
s2.create_socket(self.family)
# EADDRINUSE indicates the socket was correctly bound
- self.assertRaises(socket.error, s2.bind, (self.addr[0], port))
+ self.assertRaises(OSError, s2.bind, (self.addr[0], port))
def test_set_reuse_addr(self):
if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX:
@@ -762,7 +771,7 @@ class BaseTestAPI:
sock = socket.socket(self.family)
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- except socket.error:
+ except OSError:
unittest.skip("SO_REUSEADDR not supported on this platform")
else:
# if SO_REUSEADDR succeeded for sock we expect asyncore
@@ -787,7 +796,11 @@ class BaseTestAPI:
t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1,
count=500))
t.start()
- self.addCleanup(t.join)
+ def cleanup():
+ t.join(timeout=TIMEOUT)
+ if t.is_alive():
+ self.fail("join() timed out")
+ self.addCleanup(cleanup)
s = socket.socket(self.family, socket.SOCK_STREAM)
s.settimeout(.2)
@@ -795,7 +808,7 @@ class BaseTestAPI:
struct.pack('ii', 1, 0))
try:
s.connect(server.address)
- except socket.error:
+ except OSError:
pass
finally:
s.close()
diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py
index 30c3b4a..70d2f1c 100644
--- a/Lib/test/test_atexit.py
+++ b/Lib/test/test_atexit.py
@@ -23,7 +23,9 @@ def raise1():
def raise2():
raise SystemError
-class TestCase(unittest.TestCase):
+
+class GeneralTest(unittest.TestCase):
+
def setUp(self):
self.save_stdout = sys.stdout
self.save_stderr = sys.stderr
@@ -141,8 +143,43 @@ class TestCase(unittest.TestCase):
self.assertEqual(l, [5])
+class SubinterpreterTest(unittest.TestCase):
+
+ def test_callbacks_leak(self):
+ # This test shows a leak in refleak mode if atexit doesn't
+ # take care to free callbacks in its per-subinterpreter module
+ # state.
+ n = atexit._ncallbacks()
+ code = r"""if 1:
+ import atexit
+ def f():
+ pass
+ atexit.register(f)
+ del atexit
+ """
+ ret = support.run_in_subinterp(code)
+ self.assertEqual(ret, 0)
+ self.assertEqual(atexit._ncallbacks(), n)
+
+ def test_callbacks_leak_refcycle(self):
+ # Similar to the above, but with a refcycle through the atexit
+ # module.
+ n = atexit._ncallbacks()
+ code = r"""if 1:
+ import atexit
+ def f():
+ pass
+ atexit.register(f)
+ atexit.__atexit = atexit
+ """
+ ret = support.run_in_subinterp(code)
+ self.assertEqual(ret, 0)
+ self.assertEqual(atexit._ncallbacks(), n)
+
+
def test_main():
- support.run_unittest(TestCase)
+ support.run_unittest(__name__)
+
if __name__ == "__main__":
test_main()
diff --git a/Lib/test/test_audioop.py b/Lib/test/test_audioop.py
index a92cf87..879adea 100644
--- a/Lib/test/test_audioop.py
+++ b/Lib/test/test_audioop.py
@@ -5,13 +5,18 @@ import unittest
def pack(width, data):
return b''.join(v.to_bytes(width, sys.byteorder, signed=True) for v in data)
-packs = {w: (lambda *data, width=w: pack(width, data)) for w in (1, 2, 4)}
-maxvalues = {w: (1 << (8 * w - 1)) - 1 for w in (1, 2, 4)}
-minvalues = {w: -1 << (8 * w - 1) for w in (1, 2, 4)}
+def unpack(width, data):
+ return [int.from_bytes(data[i: i + width], sys.byteorder, signed=True)
+ for i in range(0, len(data), width)]
+
+packs = {w: (lambda *data, width=w: pack(width, data)) for w in (1, 2, 3, 4)}
+maxvalues = {w: (1 << (8 * w - 1)) - 1 for w in (1, 2, 3, 4)}
+minvalues = {w: -1 << (8 * w - 1) for w in (1, 2, 3, 4)}
datas = {
1: b'\x00\x12\x45\xbb\x7f\x80\xff',
2: packs[2](0, 0x1234, 0x4567, -0x4567, 0x7fff, -0x8000, -1),
+ 3: packs[3](0, 0x123456, 0x456789, -0x456789, 0x7fffff, -0x800000, -1),
4: packs[4](0, 0x12345678, 0x456789ab, -0x456789ab,
0x7fffffff, -0x80000000, -1),
}
@@ -19,6 +24,7 @@ datas = {
INVALID_DATA = [
(b'abc', 0),
(b'abc', 2),
+ (b'ab', 3),
(b'abc', 4),
]
@@ -26,8 +32,10 @@ INVALID_DATA = [
class TestAudioop(unittest.TestCase):
def test_max(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.max(b'', w), 0)
+ self.assertEqual(audioop.max(bytearray(), w), 0)
+ self.assertEqual(audioop.max(memoryview(b''), w), 0)
p = packs[w]
self.assertEqual(audioop.max(p(5), w), 5)
self.assertEqual(audioop.max(p(5, -8, -1), w), 8)
@@ -36,9 +44,13 @@ class TestAudioop(unittest.TestCase):
self.assertEqual(audioop.max(datas[w], w), -minvalues[w])
def test_minmax(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.minmax(b'', w),
(0x7fffffff, -0x80000000))
+ self.assertEqual(audioop.minmax(bytearray(), w),
+ (0x7fffffff, -0x80000000))
+ self.assertEqual(audioop.minmax(memoryview(b''), w),
+ (0x7fffffff, -0x80000000))
p = packs[w]
self.assertEqual(audioop.minmax(p(5), w), (5, 5))
self.assertEqual(audioop.minmax(p(5, -8, -1), w), (-8, 5))
@@ -50,16 +62,20 @@ class TestAudioop(unittest.TestCase):
(minvalues[w], maxvalues[w]))
def test_maxpp(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.maxpp(b'', w), 0)
+ self.assertEqual(audioop.maxpp(bytearray(), w), 0)
+ self.assertEqual(audioop.maxpp(memoryview(b''), w), 0)
self.assertEqual(audioop.maxpp(packs[w](*range(100)), w), 0)
self.assertEqual(audioop.maxpp(packs[w](9, 10, 5, 5, 0, 1), w), 10)
self.assertEqual(audioop.maxpp(datas[w], w),
maxvalues[w] - minvalues[w])
def test_avg(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.avg(b'', w), 0)
+ self.assertEqual(audioop.avg(bytearray(), w), 0)
+ self.assertEqual(audioop.avg(memoryview(b''), w), 0)
p = packs[w]
self.assertEqual(audioop.avg(p(5), w), 5)
self .assertEqual(audioop.avg(p(5, 8), w), 6)
@@ -74,17 +90,22 @@ class TestAudioop(unittest.TestCase):
-0x60000000)
def test_avgpp(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.avgpp(b'', w), 0)
+ self.assertEqual(audioop.avgpp(bytearray(), w), 0)
+ self.assertEqual(audioop.avgpp(memoryview(b''), w), 0)
self.assertEqual(audioop.avgpp(packs[w](*range(100)), w), 0)
self.assertEqual(audioop.avgpp(packs[w](9, 10, 5, 5, 0, 1), w), 10)
self.assertEqual(audioop.avgpp(datas[1], 1), 196)
self.assertEqual(audioop.avgpp(datas[2], 2), 50534)
+ self.assertEqual(audioop.avgpp(datas[3], 3), 12937096)
self.assertEqual(audioop.avgpp(datas[4], 4), 3311897002)
def test_rms(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.rms(b'', w), 0)
+ self.assertEqual(audioop.rms(bytearray(), w), 0)
+ self.assertEqual(audioop.rms(memoryview(b''), w), 0)
p = packs[w]
self.assertEqual(audioop.rms(p(*range(100)), w), 57)
self.assertAlmostEqual(audioop.rms(p(maxvalues[w]) * 5, w),
@@ -93,11 +114,14 @@ class TestAudioop(unittest.TestCase):
-minvalues[w], delta=1)
self.assertEqual(audioop.rms(datas[1], 1), 77)
self.assertEqual(audioop.rms(datas[2], 2), 20001)
+ self.assertEqual(audioop.rms(datas[3], 3), 5120523)
self.assertEqual(audioop.rms(datas[4], 4), 1310854152)
def test_cross(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.cross(b'', w), -1)
+ self.assertEqual(audioop.cross(bytearray(), w), -1)
+ self.assertEqual(audioop.cross(memoryview(b''), w), -1)
p = packs[w]
self.assertEqual(audioop.cross(p(0, 1, 2), w), 0)
self.assertEqual(audioop.cross(p(1, 2, -3, -4), w), 1)
@@ -106,22 +130,29 @@ class TestAudioop(unittest.TestCase):
self.assertEqual(audioop.cross(p(minvalues[w], maxvalues[w]), w), 1)
def test_add(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.add(b'', b'', w), b'')
+ self.assertEqual(audioop.add(bytearray(), bytearray(), w), b'')
+ self.assertEqual(audioop.add(memoryview(b''), memoryview(b''), w), b'')
self.assertEqual(audioop.add(datas[w], b'\0' * len(datas[w]), w),
datas[w])
self.assertEqual(audioop.add(datas[1], datas[1], 1),
b'\x00\x24\x7f\x80\x7f\x80\xfe')
self.assertEqual(audioop.add(datas[2], datas[2], 2),
packs[2](0, 0x2468, 0x7fff, -0x8000, 0x7fff, -0x8000, -2))
+ self.assertEqual(audioop.add(datas[3], datas[3], 3),
+ packs[3](0, 0x2468ac, 0x7fffff, -0x800000,
+ 0x7fffff, -0x800000, -2))
self.assertEqual(audioop.add(datas[4], datas[4], 4),
packs[4](0, 0x2468acf0, 0x7fffffff, -0x80000000,
0x7fffffff, -0x80000000, -2))
def test_bias(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
for bias in 0, 1, -1, 127, -128, 0x7fffffff, -0x80000000:
self.assertEqual(audioop.bias(b'', w, bias), b'')
+ self.assertEqual(audioop.bias(bytearray(), w, bias), b'')
+ self.assertEqual(audioop.bias(memoryview(b''), w, bias), b'')
self.assertEqual(audioop.bias(datas[1], 1, 1),
b'\x01\x13\x46\xbc\x80\x81\x00')
self.assertEqual(audioop.bias(datas[1], 1, -1),
@@ -138,6 +169,17 @@ class TestAudioop(unittest.TestCase):
packs[2](-1, 0x1233, 0x4566, -0x4568, 0x7ffe, 0x7fff, -2))
self.assertEqual(audioop.bias(datas[2], 2, -0x80000000),
datas[2])
+ self.assertEqual(audioop.bias(datas[3], 3, 1),
+ packs[3](1, 0x123457, 0x45678a, -0x456788,
+ -0x800000, -0x7fffff, 0))
+ self.assertEqual(audioop.bias(datas[3], 3, -1),
+ packs[3](-1, 0x123455, 0x456788, -0x45678a,
+ 0x7ffffe, 0x7fffff, -2))
+ self.assertEqual(audioop.bias(datas[3], 3, 0x7fffffff),
+ packs[3](-1, 0x123455, 0x456788, -0x45678a,
+ 0x7ffffe, 0x7fffff, -2))
+ self.assertEqual(audioop.bias(datas[3], 3, -0x80000000),
+ datas[3])
self.assertEqual(audioop.bias(datas[4], 4, 1),
packs[4](1, 0x12345679, 0x456789ac, -0x456789aa,
-0x80000000, -0x7fffffff, 0))
@@ -152,99 +194,146 @@ class TestAudioop(unittest.TestCase):
-1, 0, 0x7fffffff))
def test_lin2lin(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.lin2lin(datas[w], w, w), datas[w])
+ self.assertEqual(audioop.lin2lin(bytearray(datas[w]), w, w),
+ datas[w])
+ self.assertEqual(audioop.lin2lin(memoryview(datas[w]), w, w),
+ datas[w])
self.assertEqual(audioop.lin2lin(datas[1], 1, 2),
packs[2](0, 0x1200, 0x4500, -0x4500, 0x7f00, -0x8000, -0x100))
+ self.assertEqual(audioop.lin2lin(datas[1], 1, 3),
+ packs[3](0, 0x120000, 0x450000, -0x450000,
+ 0x7f0000, -0x800000, -0x10000))
self.assertEqual(audioop.lin2lin(datas[1], 1, 4),
packs[4](0, 0x12000000, 0x45000000, -0x45000000,
0x7f000000, -0x80000000, -0x1000000))
self.assertEqual(audioop.lin2lin(datas[2], 2, 1),
b'\x00\x12\x45\xba\x7f\x80\xff')
+ self.assertEqual(audioop.lin2lin(datas[2], 2, 3),
+ packs[3](0, 0x123400, 0x456700, -0x456700,
+ 0x7fff00, -0x800000, -0x100))
self.assertEqual(audioop.lin2lin(datas[2], 2, 4),
packs[4](0, 0x12340000, 0x45670000, -0x45670000,
0x7fff0000, -0x80000000, -0x10000))
+ self.assertEqual(audioop.lin2lin(datas[3], 3, 1),
+ b'\x00\x12\x45\xba\x7f\x80\xff')
+ self.assertEqual(audioop.lin2lin(datas[3], 3, 2),
+ packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
+ self.assertEqual(audioop.lin2lin(datas[3], 3, 4),
+ packs[4](0, 0x12345600, 0x45678900, -0x45678900,
+ 0x7fffff00, -0x80000000, -0x100))
self.assertEqual(audioop.lin2lin(datas[4], 4, 1),
b'\x00\x12\x45\xba\x7f\x80\xff')
self.assertEqual(audioop.lin2lin(datas[4], 4, 2),
packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
+ self.assertEqual(audioop.lin2lin(datas[4], 4, 3),
+ packs[3](0, 0x123456, 0x456789, -0x45678a,
+ 0x7fffff, -0x800000, -1))
def test_adpcm2lin(self):
self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None),
(b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
+ self.assertEqual(audioop.adpcm2lin(bytearray(b'\x07\x7f\x7f'), 1, None),
+ (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
+ self.assertEqual(audioop.adpcm2lin(memoryview(b'\x07\x7f\x7f'), 1, None),
+ (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 2, None),
(packs[2](0, 0xb, 0x29, -0x16, 0x72, -0xb3), (-179, 40)))
+ self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 3, None),
+ (packs[3](0, 0xb00, 0x2900, -0x1600, 0x7200,
+ -0xb300), (-179, 40)))
self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 4, None),
(packs[4](0, 0xb0000, 0x290000, -0x160000, 0x720000,
-0xb30000), (-179, 40)))
# Very cursory test
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None),
(b'\0' * w * 10, (0, 0)))
def test_lin2adpcm(self):
self.assertEqual(audioop.lin2adpcm(datas[1], 1, None),
(b'\x07\x7f\x7f', (-221, 39)))
- self.assertEqual(audioop.lin2adpcm(datas[2], 2, None),
- (b'\x07\x7f\x7f', (31, 39)))
- self.assertEqual(audioop.lin2adpcm(datas[4], 4, None),
- (b'\x07\x7f\x7f', (31, 39)))
+ self.assertEqual(audioop.lin2adpcm(bytearray(datas[1]), 1, None),
+ (b'\x07\x7f\x7f', (-221, 39)))
+ self.assertEqual(audioop.lin2adpcm(memoryview(datas[1]), 1, None),
+ (b'\x07\x7f\x7f', (-221, 39)))
+ for w in 2, 3, 4:
+ self.assertEqual(audioop.lin2adpcm(datas[w], w, None),
+ (b'\x07\x7f\x7f', (31, 39)))
# Very cursory test
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None),
(b'\0' * 5, (0, 0)))
+ def test_invalid_adpcm_state(self):
+ # state must be a tuple or None, not an integer
+ self.assertRaises(TypeError, audioop.adpcm2lin, b'\0', 1, 555)
+ self.assertRaises(TypeError, audioop.lin2adpcm, b'\0', 1, 555)
+
def test_lin2alaw(self):
self.assertEqual(audioop.lin2alaw(datas[1], 1),
b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
- self.assertEqual(audioop.lin2alaw(datas[2], 2),
- b'\xd5\x87\xa4\x24\xaa\x2a\x55')
- self.assertEqual(audioop.lin2alaw(datas[4], 4),
- b'\xd5\x87\xa4\x24\xaa\x2a\x55')
+ self.assertEqual(audioop.lin2alaw(bytearray(datas[1]), 1),
+ b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
+ self.assertEqual(audioop.lin2alaw(memoryview(datas[1]), 1),
+ b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
+ for w in 2, 3, 4:
+ self.assertEqual(audioop.lin2alaw(datas[w], w),
+ b'\xd5\x87\xa4\x24\xaa\x2a\x55')
def test_alaw2lin(self):
encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\
b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff'
src = [-688, -720, -2240, -4032, -9, -3, -1, -27, -244, -82, -106,
688, 720, 2240, 4032, 9, 3, 1, 27, 244, 82, 106]
- for w in 1, 2, 4:
- self.assertEqual(audioop.alaw2lin(encoded, w),
- packs[w](*(x << (w * 8) >> 13 for x in src)))
+ for w in 1, 2, 3, 4:
+ decoded = packs[w](*(x << (w * 8) >> 13 for x in src))
+ self.assertEqual(audioop.alaw2lin(encoded, w), decoded)
+ self.assertEqual(audioop.alaw2lin(bytearray(encoded), w), decoded)
+ self.assertEqual(audioop.alaw2lin(memoryview(encoded), w), decoded)
encoded = bytes(range(256))
- for w in 2, 4:
+ for w in 2, 3, 4:
decoded = audioop.alaw2lin(encoded, w)
self.assertEqual(audioop.lin2alaw(decoded, w), encoded)
def test_lin2ulaw(self):
self.assertEqual(audioop.lin2ulaw(datas[1], 1),
b'\xff\xad\x8e\x0e\x80\x00\x67')
- self.assertEqual(audioop.lin2ulaw(datas[2], 2),
- b'\xff\xad\x8e\x0e\x80\x00\x7e')
- self.assertEqual(audioop.lin2ulaw(datas[4], 4),
- b'\xff\xad\x8e\x0e\x80\x00\x7e')
+ self.assertEqual(audioop.lin2ulaw(bytearray(datas[1]), 1),
+ b'\xff\xad\x8e\x0e\x80\x00\x67')
+ self.assertEqual(audioop.lin2ulaw(memoryview(datas[1]), 1),
+ b'\xff\xad\x8e\x0e\x80\x00\x67')
+ for w in 2, 3, 4:
+ self.assertEqual(audioop.lin2ulaw(datas[w], w),
+ b'\xff\xad\x8e\x0e\x80\x00\x7e')
def test_ulaw2lin(self):
encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\
b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff'
src = [-8031, -4447, -1471, -495, -163, -53, -18, -6, -2, 0,
8031, 4447, 1471, 495, 163, 53, 18, 6, 2, 0]
- for w in 1, 2, 4:
- self.assertEqual(audioop.ulaw2lin(encoded, w),
- packs[w](*(x << (w * 8) >> 14 for x in src)))
+ for w in 1, 2, 3, 4:
+ decoded = packs[w](*(x << (w * 8) >> 14 for x in src))
+ self.assertEqual(audioop.ulaw2lin(encoded, w), decoded)
+ self.assertEqual(audioop.ulaw2lin(bytearray(encoded), w), decoded)
+ self.assertEqual(audioop.ulaw2lin(memoryview(encoded), w), decoded)
# Current u-law implementation has two codes fo 0: 0x7f and 0xff.
encoded = bytes(range(127)) + bytes(range(128, 256))
- for w in 2, 4:
+ for w in 2, 3, 4:
decoded = audioop.ulaw2lin(encoded, w)
self.assertEqual(audioop.lin2ulaw(decoded, w), encoded)
def test_mul(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.mul(b'', w, 2), b'')
+ self.assertEqual(audioop.mul(bytearray(), w, 2), b'')
+ self.assertEqual(audioop.mul(memoryview(b''), w, 2), b'')
self.assertEqual(audioop.mul(datas[w], w, 0),
b'\0' * len(datas[w]))
self.assertEqual(audioop.mul(datas[w], w, 1),
@@ -253,14 +342,21 @@ class TestAudioop(unittest.TestCase):
b'\x00\x24\x7f\x80\x7f\x80\xfe')
self.assertEqual(audioop.mul(datas[2], 2, 2),
packs[2](0, 0x2468, 0x7fff, -0x8000, 0x7fff, -0x8000, -2))
+ self.assertEqual(audioop.mul(datas[3], 3, 2),
+ packs[3](0, 0x2468ac, 0x7fffff, -0x800000,
+ 0x7fffff, -0x800000, -2))
self.assertEqual(audioop.mul(datas[4], 4, 2),
packs[4](0, 0x2468acf0, 0x7fffffff, -0x80000000,
0x7fffffff, -0x80000000, -2))
def test_ratecv(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.ratecv(b'', w, 1, 8000, 8000, None),
(b'', (-1, ((0, 0),))))
+ self.assertEqual(audioop.ratecv(bytearray(), w, 1, 8000, 8000, None),
+ (b'', (-1, ((0, 0),))))
+ self.assertEqual(audioop.ratecv(memoryview(b''), w, 1, 8000, 8000, None),
+ (b'', (-1, ((0, 0),))))
self.assertEqual(audioop.ratecv(b'', w, 5, 8000, 8000, None),
(b'', (-1, ((0, 0),) * 5)))
self.assertEqual(audioop.ratecv(b'', w, 1, 8000, 16000, None),
@@ -272,7 +368,7 @@ class TestAudioop(unittest.TestCase):
d2, state = audioop.ratecv(b'\x00\x01\x02', 1, 1, 8000, 16000, state)
self.assertEqual(d1 + d2, b'\000\000\001\001\002\001\000\000\001\001\002')
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
d0, state0 = audioop.ratecv(datas[w], w, 1, 8000, 16000, None)
d, state = b'', None
for i in range(0, len(datas[w]), w):
@@ -283,13 +379,15 @@ class TestAudioop(unittest.TestCase):
self.assertEqual(state, state0)
def test_reverse(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
self.assertEqual(audioop.reverse(b'', w), b'')
+ self.assertEqual(audioop.reverse(bytearray(), w), b'')
+ self.assertEqual(audioop.reverse(memoryview(b''), w), b'')
self.assertEqual(audioop.reverse(packs[w](0, 1, 2), w),
packs[w](2, 1, 0))
def test_tomono(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
data1 = datas[w]
data2 = bytearray(2 * len(data1))
for k in range(w):
@@ -299,9 +397,13 @@ class TestAudioop(unittest.TestCase):
for k in range(w):
data2[k+w::2*w] = data1[k::w]
self.assertEqual(audioop.tomono(data2, w, 0.5, 0.5), data1)
+ self.assertEqual(audioop.tomono(bytearray(data2), w, 0.5, 0.5),
+ data1)
+ self.assertEqual(audioop.tomono(memoryview(data2), w, 0.5, 0.5),
+ data1)
def test_tostereo(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
data1 = datas[w]
data2 = bytearray(2 * len(data1))
for k in range(w):
@@ -311,14 +413,25 @@ class TestAudioop(unittest.TestCase):
for k in range(w):
data2[k+w::2*w] = data1[k::w]
self.assertEqual(audioop.tostereo(data1, w, 1, 1), data2)
+ self.assertEqual(audioop.tostereo(bytearray(data1), w, 1, 1), data2)
+ self.assertEqual(audioop.tostereo(memoryview(data1), w, 1, 1),
+ data2)
def test_findfactor(self):
self.assertEqual(audioop.findfactor(datas[2], datas[2]), 1.0)
+ self.assertEqual(audioop.findfactor(bytearray(datas[2]),
+ bytearray(datas[2])), 1.0)
+ self.assertEqual(audioop.findfactor(memoryview(datas[2]),
+ memoryview(datas[2])), 1.0)
self.assertEqual(audioop.findfactor(b'\0' * len(datas[2]), datas[2]),
0.0)
def test_findfit(self):
self.assertEqual(audioop.findfit(datas[2], datas[2]), (0, 1.0))
+ self.assertEqual(audioop.findfit(bytearray(datas[2]),
+ bytearray(datas[2])), (0, 1.0))
+ self.assertEqual(audioop.findfit(memoryview(datas[2]),
+ memoryview(datas[2])), (0, 1.0))
self.assertEqual(audioop.findfit(datas[2], packs[2](1, 2, 0)),
(1, 8038.8))
self.assertEqual(audioop.findfit(datas[2][:-2] * 5 + datas[2], datas[2]),
@@ -326,16 +439,37 @@ class TestAudioop(unittest.TestCase):
def test_findmax(self):
self.assertEqual(audioop.findmax(datas[2], 1), 5)
+ self.assertEqual(audioop.findmax(bytearray(datas[2]), 1), 5)
+ self.assertEqual(audioop.findmax(memoryview(datas[2]), 1), 5)
def test_getsample(self):
- for w in 1, 2, 4:
+ for w in 1, 2, 3, 4:
data = packs[w](0, 1, -1, maxvalues[w], minvalues[w])
self.assertEqual(audioop.getsample(data, w, 0), 0)
+ self.assertEqual(audioop.getsample(bytearray(data), w, 0), 0)
+ self.assertEqual(audioop.getsample(memoryview(data), w, 0), 0)
self.assertEqual(audioop.getsample(data, w, 1), 1)
self.assertEqual(audioop.getsample(data, w, 2), -1)
self.assertEqual(audioop.getsample(data, w, 3), maxvalues[w])
self.assertEqual(audioop.getsample(data, w, 4), minvalues[w])
+ def test_byteswap(self):
+ swapped_datas = {
+ 1: datas[1],
+ 2: packs[2](0, 0x3412, 0x6745, -0x6646, -0x81, 0x80, -1),
+ 3: packs[3](0, 0x563412, -0x7698bb, 0x7798ba, -0x81, 0x80, -1),
+ 4: packs[4](0, 0x78563412, -0x547698bb, 0x557698ba,
+ -0x81, 0x80, -1),
+ }
+ for w in 1, 2, 3, 4:
+ self.assertEqual(audioop.byteswap(b'', w), b'')
+ self.assertEqual(audioop.byteswap(datas[w], w), swapped_datas[w])
+ self.assertEqual(audioop.byteswap(swapped_datas[w], w), datas[w])
+ self.assertEqual(audioop.byteswap(bytearray(datas[w]), w),
+ swapped_datas[w])
+ self.assertEqual(audioop.byteswap(memoryview(datas[w]), w),
+ swapped_datas[w])
+
def test_negativelen(self):
# from issue 3306, previously it segfaulted
self.assertRaises(audioop.error,
@@ -365,10 +499,33 @@ class TestAudioop(unittest.TestCase):
self.assertRaises(audioop.error, audioop.lin2alaw, data, size)
self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state)
+ def test_string(self):
+ data = 'abcd'
+ size = 2
+ self.assertRaises(TypeError, audioop.getsample, data, size, 0)
+ self.assertRaises(TypeError, audioop.max, data, size)
+ self.assertRaises(TypeError, audioop.minmax, data, size)
+ self.assertRaises(TypeError, audioop.avg, data, size)
+ self.assertRaises(TypeError, audioop.rms, data, size)
+ self.assertRaises(TypeError, audioop.avgpp, data, size)
+ self.assertRaises(TypeError, audioop.maxpp, data, size)
+ self.assertRaises(TypeError, audioop.cross, data, size)
+ self.assertRaises(TypeError, audioop.mul, data, size, 1.0)
+ self.assertRaises(TypeError, audioop.tomono, data, size, 0.5, 0.5)
+ self.assertRaises(TypeError, audioop.tostereo, data, size, 0.5, 0.5)
+ self.assertRaises(TypeError, audioop.add, data, data, size)
+ self.assertRaises(TypeError, audioop.bias, data, size, 0)
+ self.assertRaises(TypeError, audioop.reverse, data, size)
+ self.assertRaises(TypeError, audioop.lin2lin, data, size, size)
+ self.assertRaises(TypeError, audioop.ratecv, data, size, 1, 1, 1, None)
+ self.assertRaises(TypeError, audioop.lin2ulaw, data, size)
+ self.assertRaises(TypeError, audioop.lin2alaw, data, size)
+ self.assertRaises(TypeError, audioop.lin2adpcm, data, size, None)
+
def test_wrongsize(self):
data = b'abcdefgh'
state = None
- for size in (-1, 0, 3, 5, 1024):
+ for size in (-1, 0, 5, 1024):
self.assertRaises(audioop.error, audioop.ulaw2lin, data, size)
self.assertRaises(audioop.error, audioop.alaw2lin, data, size)
self.assertRaises(audioop.error, audioop.adpcm2lin, data, size, state)
diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py
index 13695de..b9738dd 100644
--- a/Lib/test/test_base64.py
+++ b/Lib/test/test_base64.py
@@ -5,10 +5,21 @@ import binascii
import os
import sys
import subprocess
-
+import struct
+from array import array
class LegacyBase64TestCase(unittest.TestCase):
+
+ # Legacy API is not as permissive as the modern API
+ def check_type_errors(self, f):
+ self.assertRaises(TypeError, f, "")
+ self.assertRaises(TypeError, f, [])
+ multidimensional = memoryview(b"1234").cast('B', (2, 2))
+ self.assertRaises(TypeError, f, multidimensional)
+ int_data = memoryview(b"1234").cast('I')
+ self.assertRaises(TypeError, f, int_data)
+
def test_encodebytes(self):
eq = self.assertEqual
eq(base64.encodebytes(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=\n")
@@ -24,7 +35,9 @@ class LegacyBase64TestCase(unittest.TestCase):
b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n")
# Non-bytes
eq(base64.encodebytes(bytearray(b'abc')), b'YWJj\n')
- self.assertRaises(TypeError, base64.encodebytes, "")
+ eq(base64.encodebytes(memoryview(b'abc')), b'YWJj\n')
+ eq(base64.encodebytes(array('B', b'abc')), b'YWJj\n')
+ self.check_type_errors(base64.encodebytes)
def test_decodebytes(self):
eq = self.assertEqual
@@ -41,7 +54,9 @@ class LegacyBase64TestCase(unittest.TestCase):
eq(base64.decodebytes(b''), b'')
# Non-bytes
eq(base64.decodebytes(bytearray(b'YWJj\n')), b'abc')
- self.assertRaises(TypeError, base64.decodebytes, "")
+ eq(base64.decodebytes(memoryview(b'YWJj\n')), b'abc')
+ eq(base64.decodebytes(array('B', b'YWJj\n')), b'abc')
+ self.check_type_errors(base64.decodebytes)
def test_encode(self):
eq = self.assertEqual
@@ -73,6 +88,42 @@ class LegacyBase64TestCase(unittest.TestCase):
class BaseXYTestCase(unittest.TestCase):
+
+ # Modern API completely ignores exported dimension and format data and
+ # treats any buffer as a stream of bytes
+ def check_encode_type_errors(self, f):
+ self.assertRaises(TypeError, f, "")
+ self.assertRaises(TypeError, f, [])
+
+ def check_decode_type_errors(self, f):
+ self.assertRaises(TypeError, f, [])
+
+ def check_other_types(self, f, bytes_data, expected):
+ eq = self.assertEqual
+ b = bytearray(bytes_data)
+ eq(f(b), expected)
+ # The bytearray wasn't mutated
+ eq(b, bytes_data)
+ eq(f(memoryview(bytes_data)), expected)
+ eq(f(array('B', bytes_data)), expected)
+ # XXX why is b64encode hardcoded here?
+ self.check_nonbyte_element_format(base64.b64encode, bytes_data)
+ self.check_multidimensional(base64.b64encode, bytes_data)
+
+ def check_multidimensional(self, f, data):
+ padding = b"\x00" if len(data) % 2 else b""
+ bytes_data = data + padding # Make sure cast works
+ shape = (len(bytes_data) // 2, 2)
+ multidimensional = memoryview(bytes_data).cast('B', shape)
+ self.assertEqual(f(multidimensional), f(bytes_data))
+
+ def check_nonbyte_element_format(self, f, data):
+ padding = b"\x00" * ((4 - len(data)) % 4)
+ bytes_data = data + padding # Make sure cast works
+ int_data = memoryview(bytes_data).cast('I')
+ self.assertEqual(f(int_data), f(bytes_data))
+
+
def test_b64encode(self):
eq = self.assertEqual
# Test default alphabet
@@ -90,13 +141,16 @@ class BaseXYTestCase(unittest.TestCase):
b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==")
# Test with arbitrary alternative characters
eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=b'*$'), b'01a*b$cd')
- # Non-bytes
- eq(base64.b64encode(bytearray(b'abcd')), b'YWJjZA==')
eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=bytearray(b'*$')),
b'01a*b$cd')
- # Check if passing a str object raises an error
- self.assertRaises(TypeError, base64.b64encode, "")
- self.assertRaises(TypeError, base64.b64encode, b"", altchars="")
+ eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=memoryview(b'*$')),
+ b'01a*b$cd')
+ eq(base64.b64encode(b'\xd3V\xbeo\xf7\x1d', altchars=array('B', b'*$')),
+ b'01a*b$cd')
+ # Non-bytes
+ self.check_other_types(base64.b64encode, b'abcd', b'YWJjZA==')
+ self.check_encode_type_errors(base64.b64encode)
+ self.assertRaises(TypeError, base64.b64encode, b"", altchars="*$")
# Test standard alphabet
eq(base64.standard_b64encode(b"www.python.org"), b"d3d3LnB5dGhvbi5vcmc=")
eq(base64.standard_b64encode(b"a"), b"YQ==")
@@ -110,15 +164,15 @@ class BaseXYTestCase(unittest.TestCase):
b"RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT"
b"Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==")
# Non-bytes
- eq(base64.standard_b64encode(bytearray(b'abcd')), b'YWJjZA==')
- # Check if passing a str object raises an error
- self.assertRaises(TypeError, base64.standard_b64encode, "")
+ self.check_other_types(base64.standard_b64encode,
+ b'abcd', b'YWJjZA==')
+ self.check_encode_type_errors(base64.standard_b64encode)
# Test with 'URL safe' alternative characters
eq(base64.urlsafe_b64encode(b'\xd3V\xbeo\xf7\x1d'), b'01a-b_cd')
# Non-bytes
- eq(base64.urlsafe_b64encode(bytearray(b'\xd3V\xbeo\xf7\x1d')), b'01a-b_cd')
- # Check if passing a str object raises an error
- self.assertRaises(TypeError, base64.urlsafe_b64encode, "")
+ self.check_other_types(base64.urlsafe_b64encode,
+ b'\xd3V\xbeo\xf7\x1d', b'01a-b_cd')
+ self.check_encode_type_errors(base64.urlsafe_b64encode)
def test_b64decode(self):
eq = self.assertEqual
@@ -141,7 +195,8 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.b64decode(data), res)
eq(base64.b64decode(data.decode('ascii')), res)
# Non-bytes
- eq(base64.b64decode(bytearray(b"YWJj")), b"abc")
+ self.check_other_types(base64.b64decode, b"YWJj", b"abc")
+ self.check_decode_type_errors(base64.b64decode)
# Test with arbitrary alternative characters
tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d',
@@ -160,7 +215,8 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.standard_b64decode(data), res)
eq(base64.standard_b64decode(data.decode('ascii')), res)
# Non-bytes
- eq(base64.standard_b64decode(bytearray(b"YWJj")), b"abc")
+ self.check_other_types(base64.standard_b64decode, b"YWJj", b"abc")
+ self.check_decode_type_errors(base64.standard_b64decode)
# Test with 'URL safe' alternative characters
tests_urlsafe = {b'01a-b_cd': b'\xd3V\xbeo\xf7\x1d',
@@ -170,7 +226,9 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.urlsafe_b64decode(data), res)
eq(base64.urlsafe_b64decode(data.decode('ascii')), res)
# Non-bytes
- eq(base64.urlsafe_b64decode(bytearray(b'01a-b_cd')), b'\xd3V\xbeo\xf7\x1d')
+ self.check_other_types(base64.urlsafe_b64decode, b'01a-b_cd',
+ b'\xd3V\xbeo\xf7\x1d')
+ self.check_decode_type_errors(base64.urlsafe_b64decode)
def test_b64decode_padding_error(self):
self.assertRaises(binascii.Error, base64.b64decode, b'abc')
@@ -205,8 +263,8 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.b32encode(b'abcd'), b'MFRGGZA=')
eq(base64.b32encode(b'abcde'), b'MFRGGZDF')
# Non-bytes
- eq(base64.b32encode(bytearray(b'abcd')), b'MFRGGZA=')
- self.assertRaises(TypeError, base64.b32encode, "")
+ self.check_other_types(base64.b32encode, b'abcd', b'MFRGGZA=')
+ self.check_encode_type_errors(base64.b32encode)
def test_b32decode(self):
eq = self.assertEqual
@@ -222,7 +280,8 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.b32decode(data), res)
eq(base64.b32decode(data.decode('ascii')), res)
# Non-bytes
- eq(base64.b32decode(bytearray(b'MFRGG===')), b'abc')
+ self.check_other_types(base64.b32decode, b'MFRGG===', b"abc")
+ self.check_decode_type_errors(base64.b32decode)
def test_b32decode_casefold(self):
eq = self.assertEqual
@@ -277,8 +336,9 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.b16encode(b'\x01\x02\xab\xcd\xef'), b'0102ABCDEF')
eq(base64.b16encode(b'\x00'), b'00')
# Non-bytes
- eq(base64.b16encode(bytearray(b'\x01\x02\xab\xcd\xef')), b'0102ABCDEF')
- self.assertRaises(TypeError, base64.b16encode, "")
+ self.check_other_types(base64.b16encode, b'\x01\x02\xab\xcd\xef',
+ b'0102ABCDEF')
+ self.check_encode_type_errors(base64.b16encode)
def test_b16decode(self):
eq = self.assertEqual
@@ -293,14 +353,268 @@ class BaseXYTestCase(unittest.TestCase):
eq(base64.b16decode(b'0102abcdef', True), b'\x01\x02\xab\xcd\xef')
eq(base64.b16decode('0102abcdef', True), b'\x01\x02\xab\xcd\xef')
# Non-bytes
- eq(base64.b16decode(bytearray(b"0102ABCDEF")), b'\x01\x02\xab\xcd\xef')
+ self.check_other_types(base64.b16decode, b"0102ABCDEF",
+ b'\x01\x02\xab\xcd\xef')
+ self.check_decode_type_errors(base64.b16decode)
+ eq(base64.b16decode(bytearray(b"0102abcdef"), True),
+ b'\x01\x02\xab\xcd\xef')
+ eq(base64.b16decode(memoryview(b"0102abcdef"), True),
+ b'\x01\x02\xab\xcd\xef')
+ eq(base64.b16decode(array('B', b"0102abcdef"), True),
+ b'\x01\x02\xab\xcd\xef')
+
+ def test_a85encode(self):
+ eq = self.assertEqual
+
+ tests = {
+ b'': b'',
+ b"www.python.org": b'GB\\6`E-ZP=Df.1GEb>',
+ bytes(range(255)): b"""!!*-'"9eu7#RLhG$k3[W&.oNg'GVB"(`=52*$$"""
+ b"""(B+<_pR,UFcb-n-Vr/1iJ-0JP==1c70M3&s#]4?Ykm5X@_(6q'R884cE"""
+ b"""H9MJ8X:f1+h<)lt#=BSg3>[:ZC?t!MSA7]@cBPD3sCi+'.E,fo>FEMbN"""
+ b"""G^4U^I!pHnJ:W<)KS>/9Ll%"IN/`jYOHG]iPa.Q$R$jD4S=Q7DTV8*TU"""
+ b"""nsrdW2ZetXKAY/Yd(L?['d?O\\@K2_]Y2%o^qmn*`5Ta:aN;TJbg"GZd"""
+ b"""*^:jeCE.%f\\,!5gtgiEi8N\\UjQ5OekiqBum-X60nF?)@o_%qPq"ad`"""
+ b"""r;HT""",
+ b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ b"0123456789!@#0^&*();:<>,. []{}":
+ b'@:E_WAS,RgBkhF"D/O92EH6,BF`qtRH$VbC6UX@47n?3D92&&T'
+ b":Jand;cHat='/U/0JP==1c70M3&r-I,;<FN.OZ`-3]oSW/g+A(H[P",
+ b"no padding..": b'DJpY:@:Wn_DJ(RS',
+ b"zero compression\0\0\0\0": b'H=_,8+Cf>,E,oN2F(oQ1z',
+ b"zero compression\0\0\0": b'H=_,8+Cf>,E,oN2F(oQ1!!!!',
+ b"Boundary:\0\0\0\0": b'6>q!aA79M(3WK-[!!',
+ b"Space compr: ": b';fH/TAKYK$D/aMV+<VdL',
+ b'\xff': b'rr',
+ b'\xff'*2: b's8N',
+ b'\xff'*3: b's8W*',
+ b'\xff'*4: b's8W-!',
+ }
+
+ for data, res in tests.items():
+ eq(base64.a85encode(data), res, data)
+ eq(base64.a85encode(data, adobe=False), res, data)
+ eq(base64.a85encode(data, adobe=True), b'<~' + res + b'~>', data)
+
+ self.check_other_types(base64.a85encode, b"www.python.org",
+ b'GB\\6`E-ZP=Df.1GEb>')
+
+ self.assertRaises(TypeError, base64.a85encode, "")
+
+ eq(base64.a85encode(b"www.python.org", wrapcol=7, adobe=False),
+ b'GB\\6`E-\nZP=Df.1\nGEb>')
+ eq(base64.a85encode(b"\0\0\0\0www.python.org", wrapcol=7, adobe=False),
+ b'zGB\\6`E\n-ZP=Df.\n1GEb>')
+ eq(base64.a85encode(b"www.python.org", wrapcol=7, adobe=True),
+ b'<~GB\\6`\nE-ZP=Df\n.1GEb>\n~>')
+
+ eq(base64.a85encode(b' '*8, foldspaces=True, adobe=False), b'yy')
+ eq(base64.a85encode(b' '*7, foldspaces=True, adobe=False), b'y+<Vd')
+ eq(base64.a85encode(b' '*6, foldspaces=True, adobe=False), b'y+<U')
+ eq(base64.a85encode(b' '*5, foldspaces=True, adobe=False), b'y+9')
+
+ def test_b85encode(self):
+ eq = self.assertEqual
+
+ tests = {
+ b'': b'',
+ b'www.python.org': b'cXxL#aCvlSZ*DGca%T',
+ bytes(range(255)): b"""009C61O)~M2nh-c3=Iws5D^j+6crX17#SKH9337X"""
+ b"""AR!_nBqb&%C@Cr{EG;fCFflSSG&MFiI5|2yJUu=?KtV!7L`6nNNJ&ad"""
+ b"""OifNtP*GA-R8>}2SXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%j"""
+ b"""czJqze0_d@fPsR8goTEOh>41ejE#<ukdcy;l$Dm3n3<ZJoSmMZprN9p"""
+ b"""q@|{(sHv)}tgWuEu(7hUw6(UkxVgH!yuH4^z`?@9#Kp$P$jQpf%+1cv"""
+ b"""(9zP<)YaD4*xB0K+}+;a;Njxq<mKk)=;`X~?CtLF@bU8V^!4`l`1$(#"""
+ b"""{Qdp""",
+ b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"""
+ b"""0123456789!@#0^&*();:<>,. []{}""":
+ b"""VPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3r%YLqtVJM@UIZOH55pPf$@("""
+ b"""Q&d$}S6EqEFflSSG&MFiI5{CeBQRbjDkv#CIy^osE+AW7dwl""",
+ b'no padding..': b'Zf_uPVPs@!Zf7no',
+ b'zero compression\x00\x00\x00\x00': b'dS!BNAY*TBaB^jHb7^mG00000',
+ b'zero compression\x00\x00\x00': b'dS!BNAY*TBaB^jHb7^mG0000',
+ b"""Boundary:\x00\x00\x00\x00""": b"""LT`0$WMOi7IsgCw00""",
+ b'Space compr: ': b'Q*dEpWgug3ZE$irARr(h',
+ b'\xff': b'{{',
+ b'\xff'*2: b'|Nj',
+ b'\xff'*3: b'|Ns9',
+ b'\xff'*4: b'|NsC0',
+ }
+
+ for data, res in tests.items():
+ eq(base64.b85encode(data), res)
+
+ self.check_other_types(base64.b85encode, b"www.python.org",
+ b'cXxL#aCvlSZ*DGca%T')
+
+ def test_a85decode(self):
+ eq = self.assertEqual
+
+ tests = {
+ b'': b'',
+ b'GB\\6`E-ZP=Df.1GEb>': b'www.python.org',
+ b"""! ! * -'"\n\t\t9eu\r\n7# RL\vhG$k3[W&.oNg'GVB"(`=52*$$"""
+ b"""(B+<_pR,UFcb-n-Vr/1iJ-0JP==1c70M3&s#]4?Ykm5X@_(6q'R884cE"""
+ b"""H9MJ8X:f1+h<)lt#=BSg3>[:ZC?t!MSA7]@cBPD3sCi+'.E,fo>FEMbN"""
+ b"""G^4U^I!pHnJ:W<)KS>/9Ll%"IN/`jYOHG]iPa.Q$R$jD4S=Q7DTV8*TU"""
+ b"""nsrdW2ZetXKAY/Yd(L?['d?O\\@K2_]Y2%o^qmn*`5Ta:aN;TJbg"GZd"""
+ b"""*^:jeCE.%f\\,!5gtgiEi8N\\UjQ5OekiqBum-X60nF?)@o_%qPq"ad`"""
+ b"""r;HT""": bytes(range(255)),
+ b"""@:E_WAS,RgBkhF"D/O92EH6,BF`qtRH$VbC6UX@47n?3D92&&T:Jand;c"""
+ b"""Hat='/U/0JP==1c70M3&r-I,;<FN.OZ`-3]oSW/g+A(H[P""":
+ b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234'
+ b'56789!@#0^&*();:<>,. []{}',
+ b'DJpY:@:Wn_DJ(RS': b'no padding..',
+ b'H=_,8+Cf>,E,oN2F(oQ1z': b'zero compression\x00\x00\x00\x00',
+ b'H=_,8+Cf>,E,oN2F(oQ1!!!!': b'zero compression\x00\x00\x00',
+ b'6>q!aA79M(3WK-[!!': b"Boundary:\x00\x00\x00\x00",
+ b';fH/TAKYK$D/aMV+<VdL': b'Space compr: ',
+ b'rr': b'\xff',
+ b's8N': b'\xff'*2,
+ b's8W*': b'\xff'*3,
+ b's8W-!': b'\xff'*4,
+ }
+
+ for data, res in tests.items():
+ eq(base64.a85decode(data), res, data)
+ eq(base64.a85decode(data, adobe=False), res, data)
+ eq(base64.a85decode(data.decode("ascii"), adobe=False), res, data)
+ eq(base64.a85decode(b'<~' + data + b'~>', adobe=True), res, data)
+ eq(base64.a85decode('<~%s~>' % data.decode("ascii"), adobe=True),
+ res, data)
+
+ eq(base64.a85decode(b'yy', foldspaces=True, adobe=False), b' '*8)
+ eq(base64.a85decode(b'y+<Vd', foldspaces=True, adobe=False), b' '*7)
+ eq(base64.a85decode(b'y+<U', foldspaces=True, adobe=False), b' '*6)
+ eq(base64.a85decode(b'y+9', foldspaces=True, adobe=False), b' '*5)
+
+ self.check_other_types(base64.a85decode, b'GB\\6`E-ZP=Df.1GEb>',
+ b"www.python.org")
+
+ def test_b85decode(self):
+ eq = self.assertEqual
+
+ tests = {
+ b'': b'',
+ b'cXxL#aCvlSZ*DGca%T': b'www.python.org',
+ b"""009C61O)~M2nh-c3=Iws5D^j+6crX17#SKH9337X"""
+ b"""AR!_nBqb&%C@Cr{EG;fCFflSSG&MFiI5|2yJUu=?KtV!7L`6nNNJ&ad"""
+ b"""OifNtP*GA-R8>}2SXo+ITwPvYU}0ioWMyV&XlZI|Y;A6DaB*^Tbai%j"""
+ b"""czJqze0_d@fPsR8goTEOh>41ejE#<ukdcy;l$Dm3n3<ZJoSmMZprN9p"""
+ b"""q@|{(sHv)}tgWuEu(7hUw6(UkxVgH!yuH4^z`?@9#Kp$P$jQpf%+1cv"""
+ b"""(9zP<)YaD4*xB0K+}+;a;Njxq<mKk)=;`X~?CtLF@bU8V^!4`l`1$(#"""
+ b"""{Qdp""": bytes(range(255)),
+ b"""VPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3r%YLqtVJM@UIZOH55pPf$@("""
+ b"""Q&d$}S6EqEFflSSG&MFiI5{CeBQRbjDkv#CIy^osE+AW7dwl""":
+ b"""abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"""
+ b"""0123456789!@#0^&*();:<>,. []{}""",
+ b'Zf_uPVPs@!Zf7no': b'no padding..',
+ b'dS!BNAY*TBaB^jHb7^mG00000': b'zero compression\x00\x00\x00\x00',
+ b'dS!BNAY*TBaB^jHb7^mG0000': b'zero compression\x00\x00\x00',
+ b"""LT`0$WMOi7IsgCw00""": b"""Boundary:\x00\x00\x00\x00""",
+ b'Q*dEpWgug3ZE$irARr(h': b'Space compr: ',
+ b'{{': b'\xff',
+ b'|Nj': b'\xff'*2,
+ b'|Ns9': b'\xff'*3,
+ b'|NsC0': b'\xff'*4,
+ }
+
+ for data, res in tests.items():
+ eq(base64.b85decode(data), res)
+ eq(base64.b85decode(data.decode("ascii")), res)
+
+ self.check_other_types(base64.b85decode, b'cXxL#aCvlSZ*DGca%T',
+ b"www.python.org")
+
+ def test_a85_padding(self):
+ eq = self.assertEqual
+
+ eq(base64.a85encode(b"x", pad=True), b'GQ7^D')
+ eq(base64.a85encode(b"xx", pad=True), b"G^'2g")
+ eq(base64.a85encode(b"xxx", pad=True), b'G^+H5')
+ eq(base64.a85encode(b"xxxx", pad=True), b'G^+IX')
+ eq(base64.a85encode(b"xxxxx", pad=True), b'G^+IXGQ7^D')
+
+ eq(base64.a85decode(b'GQ7^D'), b"x\x00\x00\x00")
+ eq(base64.a85decode(b"G^'2g"), b"xx\x00\x00")
+ eq(base64.a85decode(b'G^+H5'), b"xxx\x00")
+ eq(base64.a85decode(b'G^+IX'), b"xxxx")
+ eq(base64.a85decode(b'G^+IXGQ7^D'), b"xxxxx\x00\x00\x00")
+
+ def test_b85_padding(self):
+ eq = self.assertEqual
+
+ eq(base64.b85encode(b"x", pad=True), b'cmMzZ')
+ eq(base64.b85encode(b"xx", pad=True), b'cz6H+')
+ eq(base64.b85encode(b"xxx", pad=True), b'czAdK')
+ eq(base64.b85encode(b"xxxx", pad=True), b'czAet')
+ eq(base64.b85encode(b"xxxxx", pad=True), b'czAetcmMzZ')
+
+ eq(base64.b85decode(b'cmMzZ'), b"x\x00\x00\x00")
+ eq(base64.b85decode(b'cz6H+'), b"xx\x00\x00")
+ eq(base64.b85decode(b'czAdK'), b"xxx\x00")
+ eq(base64.b85decode(b'czAet'), b"xxxx")
+ eq(base64.b85decode(b'czAetcmMzZ'), b"xxxxx\x00\x00\x00")
+
+ def test_a85decode_errors(self):
+ illegal = (set(range(32)) | set(range(118, 256))) - set(b' \t\n\r\v')
+ for c in illegal:
+ with self.assertRaises(ValueError, msg=bytes([c])):
+ base64.a85decode(b'!!!!' + bytes([c]))
+ with self.assertRaises(ValueError, msg=bytes([c])):
+ base64.a85decode(b'!!!!' + bytes([c]), adobe=False)
+ with self.assertRaises(ValueError, msg=bytes([c])):
+ base64.a85decode(b'<~!!!!' + bytes([c]) + b'~>', adobe=True)
+
+ self.assertRaises(ValueError, base64.a85decode,
+ b"malformed", adobe=True)
+ self.assertRaises(ValueError, base64.a85decode,
+ b"<~still malformed", adobe=True)
+ self.assertRaises(ValueError, base64.a85decode,
+ b"also malformed~>", adobe=True)
+
+ # With adobe=False (the default), Adobe framing markers are disallowed
+ self.assertRaises(ValueError, base64.a85decode,
+ b"<~~>")
+ self.assertRaises(ValueError, base64.a85decode,
+ b"<~~>", adobe=False)
+ base64.a85decode(b"<~~>", adobe=True) # sanity check
+
+ self.assertRaises(ValueError, base64.a85decode,
+ b"abcx", adobe=False)
+ self.assertRaises(ValueError, base64.a85decode,
+ b"abcdey", adobe=False)
+ self.assertRaises(ValueError, base64.a85decode,
+ b"a b\nc", adobe=False, ignorechars=b"")
+
+ self.assertRaises(ValueError, base64.a85decode, b's', adobe=False)
+ self.assertRaises(ValueError, base64.a85decode, b's8', adobe=False)
+ self.assertRaises(ValueError, base64.a85decode, b's8W', adobe=False)
+ self.assertRaises(ValueError, base64.a85decode, b's8W-', adobe=False)
+ self.assertRaises(ValueError, base64.a85decode, b's8W-"', adobe=False)
+
+ def test_b85decode_errors(self):
+ illegal = list(range(33)) + \
+ list(b'"\',./:[\\]') + \
+ list(range(128, 256))
+ for c in illegal:
+ with self.assertRaises(ValueError, msg=bytes([c])):
+ base64.b85decode(b'0000' + bytes([c]))
+
+ self.assertRaises(ValueError, base64.b85decode, b'|')
+ self.assertRaises(ValueError, base64.b85decode, b'|N')
+ self.assertRaises(ValueError, base64.b85decode, b'|Ns')
+ self.assertRaises(ValueError, base64.b85decode, b'|NsC')
+ self.assertRaises(ValueError, base64.b85decode, b'|NsC1')
def test_decode_nonascii_str(self):
decode_funcs = (base64.b64decode,
base64.standard_b64decode,
base64.urlsafe_b64decode,
base64.b32decode,
- base64.b16decode)
+ base64.b16decode,
+ base64.b85decode,
+ base64.a85decode)
for f in decode_funcs:
self.assertRaises(ValueError, f, 'with non-ascii \xcb')
diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py
index 7b9bd19..580a963 100644
--- a/Lib/test/test_bisect.py
+++ b/Lib/test/test_bisect.py
@@ -7,7 +7,7 @@ py_bisect = support.import_fresh_module('bisect', blocked=['_bisect'])
c_bisect = support.import_fresh_module('bisect', fresh=['_bisect'])
class Range(object):
- """A trivial range()-like object without any integer width limitations."""
+ """A trivial range()-like object that has an insert() method."""
def __init__(self, start, stop):
self.start = start
self.stop = stop
@@ -120,10 +120,10 @@ class TestBisect:
def test_negative_lo(self):
# Issue 3301
mod = self.module
- self.assertRaises(ValueError, mod.bisect_left, [1, 2, 3], 5, -1, 3),
- self.assertRaises(ValueError, mod.bisect_right, [1, 2, 3], 5, -1, 3),
- self.assertRaises(ValueError, mod.insort_left, [1, 2, 3], 5, -1, 3),
- self.assertRaises(ValueError, mod.insort_right, [1, 2, 3], 5, -1, 3),
+ self.assertRaises(ValueError, mod.bisect_left, [1, 2, 3], 5, -1, 3)
+ self.assertRaises(ValueError, mod.bisect_right, [1, 2, 3], 5, -1, 3)
+ self.assertRaises(ValueError, mod.insort_left, [1, 2, 3], 5, -1, 3)
+ self.assertRaises(ValueError, mod.insort_right, [1, 2, 3], 5, -1, 3)
def test_large_range(self):
# Issue 13496
diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py
index 04e3f61..1667847 100644
--- a/Lib/test/test_buffer.py
+++ b/Lib/test/test_buffer.py
@@ -4290,9 +4290,5 @@ class TestBufferProtocol(unittest.TestCase):
self.assertRaises(BufferError, memoryview, x)
-def test_main():
- support.run_unittest(TestBufferProtocol)
-
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index c342a43..b561a6f 100644
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -16,6 +16,7 @@ import unittest
import warnings
from operator import neg
from test.support import TESTFN, unlink, run_unittest, check_warnings
+from test.script_helper import assert_python_ok
try:
import pty, signal
except ImportError:
@@ -464,6 +465,11 @@ class BuiltinTest(unittest.TestCase):
self.assertRaises(TypeError, eval, ())
self.assertRaises(SyntaxError, eval, bom[:2] + b'a')
+ class X:
+ def __getitem__(self, key):
+ raise ValueError
+ self.assertRaises(ValueError, eval, "foo", {}, X())
+
def test_general_eval(self):
# Tests that general mappings can be used for the locals argument
@@ -579,7 +585,10 @@ class BuiltinTest(unittest.TestCase):
raise frozendict_error("frozendict is readonly")
# read-only builtins
- frozen_builtins = frozendict(__builtins__)
+ if isinstance(__builtins__, types.ModuleType):
+ frozen_builtins = frozendict(__builtins__.__dict__)
+ else:
+ frozen_builtins = frozendict(__builtins__)
code = compile("__builtins__['superglobal']=2; print(superglobal)", "test", "exec")
self.assertRaises(frozendict_error,
exec, code, {'__builtins__': frozen_builtins})
@@ -839,8 +848,19 @@ class BuiltinTest(unittest.TestCase):
self.assertEqual(max(1, 2.0, 3), 3)
self.assertEqual(max(1.0, 2, 3), 3)
+ self.assertRaises(TypeError, max)
+ self.assertRaises(TypeError, max, 42)
+ self.assertRaises(ValueError, max, ())
+ class BadSeq:
+ def __getitem__(self, index):
+ raise ValueError
+ self.assertRaises(ValueError, max, BadSeq())
+
for stmt in (
"max(key=int)", # no args
+ "max(default=None)",
+ "max(1, 2, default=None)", # require container for default
+ "max(default=None, key=int)",
"max(1, key=int)", # single arg not iterable
"max(1, 2, keystone=int)", # wrong keyword
"max(1, 2, key=int, abc=int)", # two many keywords
@@ -857,6 +877,13 @@ class BuiltinTest(unittest.TestCase):
self.assertEqual(max((1,2), key=neg), 1) # two elem iterable
self.assertEqual(max(1, 2, key=neg), 1) # two elems
+ self.assertEqual(max((), default=None), None) # zero elem iterable
+ self.assertEqual(max((1,), default=None), 1) # one elem iterable
+ self.assertEqual(max((1,2), default=None), 2) # two elem iterable
+
+ self.assertEqual(max((), default=1, key=neg), 1)
+ self.assertEqual(max((1, 2), default=3, key=neg), 1)
+
data = [random.randrange(200) for i in range(100)]
keys = dict((elem, random.randrange(50)) for elem in data)
f = keys.__getitem__
@@ -883,6 +910,9 @@ class BuiltinTest(unittest.TestCase):
for stmt in (
"min(key=int)", # no args
+ "min(default=None)",
+ "min(1, 2, default=None)", # require container for default
+ "min(default=None, key=int)",
"min(1, key=int)", # single arg not iterable
"min(1, 2, keystone=int)", # wrong keyword
"min(1, 2, key=int, abc=int)", # two many keywords
@@ -899,6 +929,13 @@ class BuiltinTest(unittest.TestCase):
self.assertEqual(min((1,2), key=neg), 2) # two elem iterable
self.assertEqual(min(1, 2, key=neg), 2) # two elems
+ self.assertEqual(min((), default=None), None) # zero elem iterable
+ self.assertEqual(min((1,), default=None), 1) # one elem iterable
+ self.assertEqual(min((1,2), default=None), 1) # two elem iterable
+
+ self.assertEqual(min((), default=1, key=neg), 1)
+ self.assertEqual(min((1, 2), default=1, key=neg), 2)
+
data = [random.randrange(200) for i in range(100)]
keys = dict((elem, random.randrange(50)) for elem in data)
f = keys.__getitem__
@@ -940,29 +977,25 @@ class BuiltinTest(unittest.TestCase):
def write_testfile(self):
# NB the first 4 lines are also used to test input, below
fp = open(TESTFN, 'w')
- try:
+ self.addCleanup(unlink, TESTFN)
+ with fp:
fp.write('1+1\n')
fp.write('The quick brown fox jumps over the lazy dog')
fp.write('.\n')
fp.write('Dear John\n')
fp.write('XXX'*100)
fp.write('YYY'*100)
- finally:
- fp.close()
def test_open(self):
self.write_testfile()
fp = open(TESTFN, 'r')
- try:
+ with fp:
self.assertEqual(fp.readline(4), '1+1\n')
self.assertEqual(fp.readline(), 'The quick brown fox jumps over the lazy dog.\n')
self.assertEqual(fp.readline(4), 'Dear')
self.assertEqual(fp.readline(100), ' John\n')
self.assertEqual(fp.read(300), 'XXX'*100)
self.assertEqual(fp.read(1000), 'YYY'*100)
- finally:
- fp.close()
- unlink(TESTFN)
def test_open_default_encoding(self):
old_environ = dict(os.environ)
@@ -977,15 +1010,17 @@ class BuiltinTest(unittest.TestCase):
self.write_testfile()
current_locale_encoding = locale.getpreferredencoding(False)
fp = open(TESTFN, 'w')
- try:
+ with fp:
self.assertEqual(fp.encoding, current_locale_encoding)
- finally:
- fp.close()
- unlink(TESTFN)
finally:
os.environ.clear()
os.environ.update(old_environ)
+ def test_open_non_inheritable(self):
+ fileobj = open(__file__)
+ with fileobj:
+ self.assertFalse(os.get_inheritable(fileobj.fileno()))
+
def test_ord(self):
self.assertEqual(ord(' '), 32)
self.assertEqual(ord('A'), 65)
@@ -1096,7 +1131,6 @@ class BuiltinTest(unittest.TestCase):
sys.stdin = savestdin
sys.stdout = savestdout
fp.close()
- unlink(TESTFN)
@unittest.skipUnless(pty, "the pty and signal modules must be available")
def check_input_tty(self, prompt, terminal_input, stdio_encoding=None):
@@ -1476,17 +1510,11 @@ class BuiltinTest(unittest.TestCase):
# --------------------------------------------------------------------
# Issue #7994: object.__format__ with a non-empty format string is
# deprecated
- def test_deprecated_format_string(obj, fmt_str, should_raise_warning):
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always", DeprecationWarning)
- format(obj, fmt_str)
- if should_raise_warning:
- self.assertEqual(len(w), 1)
- self.assertIsInstance(w[0].message, DeprecationWarning)
- self.assertIn('object.__format__ with a non-empty format '
- 'string', str(w[0].message))
+ def test_deprecated_format_string(obj, fmt_str, should_raise):
+ if should_raise:
+ self.assertRaises(TypeError, format, obj, fmt_str)
else:
- self.assertEqual(len(w), 0)
+ format(obj, fmt_str)
fmt_strs = ['', 's']
@@ -1565,21 +1593,45 @@ class TestSorted(unittest.TestCase):
data = 'The quick Brown fox Jumped over The lazy Dog'.split()
self.assertRaises(TypeError, sorted, data, None, lambda x,y: 0)
-def test_main(verbose=None):
- test_classes = (BuiltinTest, TestSorted)
-
- run_unittest(*test_classes)
-
- # verify reference counting
- if verbose and hasattr(sys, "gettotalrefcount"):
- import gc
- counts = [None] * 5
- for i in range(len(counts)):
- run_unittest(*test_classes)
- gc.collect()
- counts[i] = sys.gettotalrefcount()
- print(counts)
+class ShutdownTest(unittest.TestCase):
+
+ def test_cleanup(self):
+ # Issue #19255: builtins are still available at shutdown
+ code = """if 1:
+ import builtins
+ import sys
+
+ class C:
+ def __del__(self):
+ print("before")
+ # Check that builtins still exist
+ len(())
+ print("after")
+
+ c = C()
+ # Make this module survive until builtins and sys are cleaned
+ builtins.here = sys.modules[__name__]
+ sys.here = sys.modules[__name__]
+ # Create a reference loop so that this module needs to go
+ # through a GC phase.
+ here = sys.modules[__name__]
+ """
+ # Issue #20599: Force ASCII encoding to get a codec implemented in C,
+ # otherwise the codec may be unloaded before C.__del__() is called, and
+ # so print("before") fails because the codec cannot be used to encode
+ # "before" to sys.stdout.encoding. For example, on Windows,
+ # sys.stdout.encoding is the OEM code page and these code pages are
+ # implemented in Python
+ rc, out, err = assert_python_ok("-c", code,
+ PYTHONIOENCODING="ascii")
+ self.assertEqual(["before", "after"], out.decode().splitlines())
+
+
+def load_tests(loader, tests, pattern):
+ from doctest import DocTestSuite
+ tests.addTest(DocTestSuite(builtins))
+ return tests
if __name__ == "__main__":
- test_main(verbose=True)
+ unittest.main()
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index 3c09141..f350211 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -288,8 +288,22 @@ class BaseBytesTest:
self.assertEqual(self.type2test(b"").join(lst), b"abc")
self.assertEqual(self.type2test(b"").join(tuple(lst)), b"abc")
self.assertEqual(self.type2test(b"").join(iter(lst)), b"abc")
- self.assertEqual(self.type2test(b".").join([b"ab", b"cd"]), b"ab.cd")
- # XXX more...
+ dot_join = self.type2test(b".:").join
+ self.assertEqual(dot_join([b"ab", b"cd"]), b"ab.:cd")
+ self.assertEqual(dot_join([memoryview(b"ab"), b"cd"]), b"ab.:cd")
+ self.assertEqual(dot_join([b"ab", memoryview(b"cd")]), b"ab.:cd")
+ self.assertEqual(dot_join([bytearray(b"ab"), b"cd"]), b"ab.:cd")
+ self.assertEqual(dot_join([b"ab", bytearray(b"cd")]), b"ab.:cd")
+ # Stress it with many items
+ seq = [b"abc"] * 1000
+ expected = b"abc" + b".:abc" * 999
+ self.assertEqual(dot_join(seq), expected)
+ # Error handling and cleanup when some item in the middle of the
+ # sequence has the wrong type.
+ with self.assertRaises(TypeError):
+ dot_join([bytearray(b"ab"), "cd", b"ef"])
+ with self.assertRaises(TypeError):
+ dot_join([memoryview(b"ab"), "cd", b"ef"])
def test_count(self):
b = self.type2test(b'mississippi')
@@ -765,7 +779,7 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase):
finally:
try:
os.remove(tfn)
- except os.error:
+ except OSError:
pass
def test_reverse(self):
@@ -901,6 +915,15 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase):
with self.assertRaises(ValueError):
b[3:4] = elem
+ def test_setslice_extend(self):
+ # Exercise the resizing logic (see issue #19087)
+ b = bytearray(range(100))
+ self.assertEqual(list(b), list(range(100)))
+ del b[:10]
+ self.assertEqual(list(b), list(range(10, 100)))
+ b.extend(range(100, 110))
+ self.assertEqual(list(b), list(range(10, 110)))
+
def test_extended_set_del_slice(self):
indices = (0, None, 1, 3, 19, 300, 1<<333, -1, -2, -31, -300)
for start in indices:
@@ -1280,6 +1303,11 @@ class BytearrayPEP3137Test(unittest.TestCase,
self.assertEqual(val, newval)
self.assertTrue(val is not newval,
expr+' returned val on a mutable object')
+ sep = self.marshal(b'')
+ newval = sep.join([val])
+ self.assertEqual(val, newval)
+ self.assertIsNot(val, newval)
+
class FixedStringTest(test.string_tests.BaseTest):
diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py
index c8c9351..40d8425 100644
--- a/Lib/test/test_bz2.py
+++ b/Lib/test/test_bz2.py
@@ -1,5 +1,5 @@
from test import support
-from test.support import TESTFN, bigmemtest, _4G
+from test.support import bigmemtest, _4G
import unittest
from io import BytesIO
@@ -8,6 +8,7 @@ import pickle
import random
import subprocess
import sys
+from test.support import unlink
try:
import threading
@@ -18,10 +19,10 @@ except ImportError:
bz2 = support.import_module('bz2')
from bz2 import BZ2File, BZ2Compressor, BZ2Decompressor
-has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx")
class BaseTest(unittest.TestCase):
"Base for other testcases."
+
TEXT_LINES = [
b'root:x:0:0:root:/root:/bin/bash\n',
b'bin:x:1:1:bin:/bin:\n',
@@ -51,13 +52,17 @@ class BaseTest(unittest.TestCase):
BAD_DATA = b'this is not a valid bzip2 file'
def setUp(self):
- self.filename = TESTFN
+ self.filename = support.TESTFN
def tearDown(self):
if os.path.isfile(self.filename):
os.unlink(self.filename)
- if has_cmdline_bunzip2:
+ if sys.platform == "win32":
+ # bunzip2 isn't available to run on Windows.
+ def decompress(self, data):
+ return bz2.decompress(data)
+ else:
def decompress(self, data):
pop = subprocess.Popen("bunzip2", shell=True,
stdin=subprocess.PIPE,
@@ -71,13 +76,9 @@ class BaseTest(unittest.TestCase):
ret = bz2.decompress(data)
return ret
- else:
- # bunzip2 isn't available to run on Windows.
- def decompress(self, data):
- return bz2.decompress(data)
class BZ2FileTest(BaseTest):
- "Test BZ2File type miscellaneous methods."
+ "Test the BZ2File class."
def createTempFile(self, streams=1, suffix=b""):
with open(self.filename, "wb") as f:
@@ -85,18 +86,12 @@ class BZ2FileTest(BaseTest):
f.write(suffix)
def testBadArgs(self):
- with self.assertRaises(TypeError):
- BZ2File(123.456)
- with self.assertRaises(ValueError):
- BZ2File("/dev/null", "z")
- with self.assertRaises(ValueError):
- BZ2File("/dev/null", "rx")
- with self.assertRaises(ValueError):
- BZ2File("/dev/null", "rbt")
- with self.assertRaises(ValueError):
- BZ2File("/dev/null", compresslevel=0)
- with self.assertRaises(ValueError):
- BZ2File("/dev/null", compresslevel=10)
+ self.assertRaises(TypeError, BZ2File, 123.456)
+ self.assertRaises(ValueError, BZ2File, "/dev/null", "z")
+ self.assertRaises(ValueError, BZ2File, "/dev/null", "rx")
+ self.assertRaises(ValueError, BZ2File, "/dev/null", "rbt")
+ self.assertRaises(ValueError, BZ2File, "/dev/null", compresslevel=0)
+ self.assertRaises(ValueError, BZ2File, "/dev/null", compresslevel=10)
def testRead(self):
self.createTempFile()
@@ -232,9 +227,8 @@ class BZ2FileTest(BaseTest):
self.createTempFile()
bz2f = BZ2File(self.filename)
bz2f.close()
- self.assertRaises(ValueError, bz2f.__next__)
- # This call will deadlock if the above .__next__ call failed to
- # release the lock.
+ self.assertRaises(ValueError, next, bz2f)
+ # This call will deadlock if the above call failed to release the lock.
self.assertRaises(ValueError, bz2f.readlines)
def testWrite(self):
@@ -278,8 +272,8 @@ class BZ2FileTest(BaseTest):
bz2f.write(b"abc")
with BZ2File(self.filename, "r") as bz2f:
- self.assertRaises(IOError, bz2f.write, b"a")
- self.assertRaises(IOError, bz2f.writelines, [b"a"])
+ self.assertRaises(OSError, bz2f.write, b"a")
+ self.assertRaises(OSError, bz2f.writelines, [b"a"])
def testAppend(self):
with BZ2File(self.filename, "w") as bz2f:
@@ -397,7 +391,7 @@ class BZ2FileTest(BaseTest):
bz2f.close()
self.assertRaises(ValueError, bz2f.seekable)
- bz2f = BZ2File(BytesIO(), mode="w")
+ bz2f = BZ2File(BytesIO(), "w")
try:
self.assertFalse(bz2f.seekable())
finally:
@@ -423,7 +417,7 @@ class BZ2FileTest(BaseTest):
bz2f.close()
self.assertRaises(ValueError, bz2f.readable)
- bz2f = BZ2File(BytesIO(), mode="w")
+ bz2f = BZ2File(BytesIO(), "w")
try:
self.assertFalse(bz2f.readable())
finally:
@@ -440,7 +434,7 @@ class BZ2FileTest(BaseTest):
bz2f.close()
self.assertRaises(ValueError, bz2f.writable)
- bz2f = BZ2File(BytesIO(), mode="w")
+ bz2f = BZ2File(BytesIO(), "w")
try:
self.assertTrue(bz2f.writable())
finally:
@@ -454,7 +448,7 @@ class BZ2FileTest(BaseTest):
del o
def testOpenNonexistent(self):
- self.assertRaises(IOError, BZ2File, "/non/existent")
+ self.assertRaises(OSError, BZ2File, "/non/existent")
def testReadlinesNoNewline(self):
# Issue #1191043: readlines() fails on a file containing no newline.
@@ -494,7 +488,7 @@ class BZ2FileTest(BaseTest):
# Issue #7205: Using a BZ2File from several threads shouldn't deadlock.
data = b"1" * 2**20
nthreads = 10
- with bz2.BZ2File(self.filename, 'wb') as f:
+ with BZ2File(self.filename, 'wb') as f:
def comp():
for i in range(5):
f.write(data)
@@ -505,28 +499,27 @@ class BZ2FileTest(BaseTest):
t.join()
def testWithoutThreading(self):
- bz2 = support.import_fresh_module("bz2", blocked=("threading",))
- with bz2.BZ2File(self.filename, "wb") as f:
+ module = support.import_fresh_module("bz2", blocked=("threading",))
+ with module.BZ2File(self.filename, "wb") as f:
f.write(b"abc")
- with bz2.BZ2File(self.filename, "rb") as f:
+ with module.BZ2File(self.filename, "rb") as f:
self.assertEqual(f.read(), b"abc")
def testMixedIterationAndReads(self):
self.createTempFile()
linelen = len(self.TEXT_LINES[0])
halflen = linelen // 2
- with bz2.BZ2File(self.filename) as bz2f:
+ with BZ2File(self.filename) as bz2f:
bz2f.read(halflen)
self.assertEqual(next(bz2f), self.TEXT_LINES[0][halflen:])
self.assertEqual(bz2f.read(), self.TEXT[linelen:])
- with bz2.BZ2File(self.filename) as bz2f:
+ with BZ2File(self.filename) as bz2f:
bz2f.readline()
self.assertEqual(next(bz2f), self.TEXT_LINES[1])
self.assertEqual(bz2f.readline(), self.TEXT_LINES[2])
- with bz2.BZ2File(self.filename) as bz2f:
+ with BZ2File(self.filename) as bz2f:
bz2f.readlines()
- with self.assertRaises(StopIteration):
- next(bz2f)
+ self.assertRaises(StopIteration, next, bz2f)
self.assertEqual(bz2f.readlines(), [])
def testMultiStreamOrdering(self):
@@ -594,6 +587,20 @@ class BZ2FileTest(BaseTest):
bz2f.seek(-150, 1)
self.assertEqual(bz2f.read(), self.TEXT[500-150:])
+ def test_read_truncated(self):
+ # Drop the eos_magic field (6 bytes) and CRC (4 bytes).
+ truncated = self.DATA[:-10]
+ with BZ2File(BytesIO(truncated)) as f:
+ self.assertRaises(EOFError, f.read)
+ with BZ2File(BytesIO(truncated)) as f:
+ self.assertEqual(f.read(len(self.TEXT)), self.TEXT)
+ self.assertRaises(EOFError, f.read, 1)
+ # Incomplete 4-byte file header, and block header of at least 146 bits.
+ for i in range(22):
+ with BZ2File(BytesIO(truncated[:i])) as f:
+ self.assertRaises(EOFError, f.read, 1)
+
+
class BZ2CompressorTest(BaseTest):
def testCompress(self):
bz2c = BZ2Compressor()
@@ -740,97 +747,122 @@ class CompressDecompressTest(BaseTest):
class OpenTest(BaseTest):
+ "Test the open function."
+
+ def open(self, *args, **kwargs):
+ return bz2.open(*args, **kwargs)
+
def test_binary_modes(self):
- with bz2.open(self.filename, "wb") as f:
- f.write(self.TEXT)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read())
- self.assertEqual(file_data, self.TEXT)
- with bz2.open(self.filename, "rb") as f:
- self.assertEqual(f.read(), self.TEXT)
- with bz2.open(self.filename, "ab") as f:
- f.write(self.TEXT)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read())
- self.assertEqual(file_data, self.TEXT * 2)
+ for mode in ("wb", "xb"):
+ if mode == "xb":
+ unlink(self.filename)
+ with self.open(self.filename, mode) as f:
+ f.write(self.TEXT)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read())
+ self.assertEqual(file_data, self.TEXT)
+ with self.open(self.filename, "rb") as f:
+ self.assertEqual(f.read(), self.TEXT)
+ with self.open(self.filename, "ab") as f:
+ f.write(self.TEXT)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read())
+ self.assertEqual(file_data, self.TEXT * 2)
def test_implicit_binary_modes(self):
# Test implicit binary modes (no "b" or "t" in mode string).
- with bz2.open(self.filename, "w") as f:
- f.write(self.TEXT)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read())
- self.assertEqual(file_data, self.TEXT)
- with bz2.open(self.filename, "r") as f:
- self.assertEqual(f.read(), self.TEXT)
- with bz2.open(self.filename, "a") as f:
- f.write(self.TEXT)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read())
- self.assertEqual(file_data, self.TEXT * 2)
+ for mode in ("w", "x"):
+ if mode == "x":
+ unlink(self.filename)
+ with self.open(self.filename, mode) as f:
+ f.write(self.TEXT)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read())
+ self.assertEqual(file_data, self.TEXT)
+ with self.open(self.filename, "r") as f:
+ self.assertEqual(f.read(), self.TEXT)
+ with self.open(self.filename, "a") as f:
+ f.write(self.TEXT)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read())
+ self.assertEqual(file_data, self.TEXT * 2)
def test_text_modes(self):
text = self.TEXT.decode("ascii")
text_native_eol = text.replace("\n", os.linesep)
- with bz2.open(self.filename, "wt") as f:
- f.write(text)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read()).decode("ascii")
- self.assertEqual(file_data, text_native_eol)
- with bz2.open(self.filename, "rt") as f:
- self.assertEqual(f.read(), text)
- with bz2.open(self.filename, "at") as f:
- f.write(text)
- with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read()).decode("ascii")
- self.assertEqual(file_data, text_native_eol * 2)
+ for mode in ("wt", "xt"):
+ if mode == "xt":
+ unlink(self.filename)
+ with self.open(self.filename, mode) as f:
+ f.write(text)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read()).decode("ascii")
+ self.assertEqual(file_data, text_native_eol)
+ with self.open(self.filename, "rt") as f:
+ self.assertEqual(f.read(), text)
+ with self.open(self.filename, "at") as f:
+ f.write(text)
+ with open(self.filename, "rb") as f:
+ file_data = self.decompress(f.read()).decode("ascii")
+ self.assertEqual(file_data, text_native_eol * 2)
+
+ def test_x_mode(self):
+ for mode in ("x", "xb", "xt"):
+ unlink(self.filename)
+ with self.open(self.filename, mode) as f:
+ pass
+ with self.assertRaises(FileExistsError):
+ with self.open(self.filename, mode) as f:
+ pass
def test_fileobj(self):
- with bz2.open(BytesIO(self.DATA), "r") as f:
+ with self.open(BytesIO(self.DATA), "r") as f:
self.assertEqual(f.read(), self.TEXT)
- with bz2.open(BytesIO(self.DATA), "rb") as f:
+ with self.open(BytesIO(self.DATA), "rb") as f:
self.assertEqual(f.read(), self.TEXT)
text = self.TEXT.decode("ascii")
- with bz2.open(BytesIO(self.DATA), "rt") as f:
+ with self.open(BytesIO(self.DATA), "rt") as f:
self.assertEqual(f.read(), text)
def test_bad_params(self):
# Test invalid parameter combinations.
- with self.assertRaises(ValueError):
- bz2.open(self.filename, "wbt")
- with self.assertRaises(ValueError):
- bz2.open(self.filename, "rb", encoding="utf-8")
- with self.assertRaises(ValueError):
- bz2.open(self.filename, "rb", errors="ignore")
- with self.assertRaises(ValueError):
- bz2.open(self.filename, "rb", newline="\n")
+ self.assertRaises(ValueError,
+ self.open, self.filename, "wbt")
+ self.assertRaises(ValueError,
+ self.open, self.filename, "xbt")
+ self.assertRaises(ValueError,
+ self.open, self.filename, "rb", encoding="utf-8")
+ self.assertRaises(ValueError,
+ self.open, self.filename, "rb", errors="ignore")
+ self.assertRaises(ValueError,
+ self.open, self.filename, "rb", newline="\n")
def test_encoding(self):
# Test non-default encoding.
text = self.TEXT.decode("ascii")
text_native_eol = text.replace("\n", os.linesep)
- with bz2.open(self.filename, "wt", encoding="utf-16-le") as f:
+ with self.open(self.filename, "wt", encoding="utf-16-le") as f:
f.write(text)
with open(self.filename, "rb") as f:
- file_data = bz2.decompress(f.read()).decode("utf-16-le")
+ file_data = self.decompress(f.read()).decode("utf-16-le")
self.assertEqual(file_data, text_native_eol)
- with bz2.open(self.filename, "rt", encoding="utf-16-le") as f:
+ with self.open(self.filename, "rt", encoding="utf-16-le") as f:
self.assertEqual(f.read(), text)
def test_encoding_error_handler(self):
# Test with non-default encoding error handler.
- with bz2.open(self.filename, "wb") as f:
+ with self.open(self.filename, "wb") as f:
f.write(b"foo\xffbar")
- with bz2.open(self.filename, "rt", encoding="ascii", errors="ignore") \
+ with self.open(self.filename, "rt", encoding="ascii", errors="ignore") \
as f:
self.assertEqual(f.read(), "foobar")
def test_newline(self):
# Test with explicit newline (universal newline mode disabled).
text = self.TEXT.decode("ascii")
- with bz2.open(self.filename, "wt", newline="\n") as f:
+ with self.open(self.filename, "wt", newline="\n") as f:
f.write(text)
- with bz2.open(self.filename, "rt", newline="\r") as f:
+ with self.open(self.filename, "rt", newline="\r") as f:
self.assertEqual(f.readlines(), [text])
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 50c4bba..ba7c38d 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -9,6 +9,7 @@ import sys
import time
import unittest
from test import support
+from test.support import MISSING_C_DOCSTRINGS
try:
import _posixsubprocess
except ImportError:
@@ -44,7 +45,7 @@ class CAPITest(unittest.TestCase):
@unittest.skipUnless(threading, 'Threading required for this test.')
def test_no_FatalError_infinite_loop(self):
- with support.suppress_crash_popup():
+ with support.SuppressCrashReport():
p = subprocess.Popen([sys.executable, "-c",
'import _testcapi;'
'_testcapi.crash_no_current_thread()'],
@@ -110,6 +111,46 @@ class CAPITest(unittest.TestCase):
self.assertRaises(TypeError, _posixsubprocess.fork_exec,
Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_docstring_signature_parsing(self):
+
+ self.assertEqual(_testcapi.no_docstring.__doc__, None)
+ self.assertEqual(_testcapi.no_docstring.__text_signature__, None)
+
+ self.assertEqual(_testcapi.docstring_empty.__doc__, "")
+ self.assertEqual(_testcapi.docstring_empty.__text_signature__, None)
+
+ self.assertEqual(_testcapi.docstring_no_signature.__doc__,
+ "This docstring has no signature.")
+ self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None)
+
+ self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__,
+ "docstring_with_invalid_signature($module, /, boo)\n"
+ "\n"
+ "This docstring has an invalid signature."
+ )
+ self.assertEqual(_testcapi.docstring_with_invalid_signature.__text_signature__, None)
+
+ self.assertEqual(_testcapi.docstring_with_invalid_signature2.__doc__,
+ "docstring_with_invalid_signature2($module, /, boo)\n"
+ "\n"
+ "--\n"
+ "\n"
+ "This docstring also has an invalid signature."
+ )
+ self.assertEqual(_testcapi.docstring_with_invalid_signature2.__text_signature__, None)
+
+ self.assertEqual(_testcapi.docstring_with_signature.__doc__,
+ "This docstring has a valid signature.")
+ self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "($module, /, sig)")
+
+ self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__,
+ "\nThis docstring has a valid signature and some extra newlines.")
+ self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__,
+ "($module, /, parameter)")
+
+
@unittest.skipUnless(threading, 'Threading required for this test.')
class TestPendingCalls(unittest.TestCase):
@@ -193,6 +234,9 @@ class TestPendingCalls(unittest.TestCase):
self.pendingcalls_submit(l, n)
self.pendingcalls_wait(l, n)
+
+class SubinterpreterTest(unittest.TestCase):
+
def test_subinterps(self):
import builtins
r, w = os.pipe()
@@ -203,47 +247,109 @@ class TestPendingCalls(unittest.TestCase):
pickle.dump(id(builtins), f)
""".format(w)
with open(r, "rb") as f:
- ret = _testcapi.run_in_subinterp(code)
+ ret = support.run_in_subinterp(code)
self.assertEqual(ret, 0)
self.assertNotEqual(pickle.load(f), id(sys.modules))
self.assertNotEqual(pickle.load(f), id(builtins))
+
# Bug #6012
class Test6012(unittest.TestCase):
def test(self):
self.assertEqual(_testcapi.argparsing("Hello", "World"), 1)
-class EmbeddingTest(unittest.TestCase):
-
- @unittest.skipIf(
- sys.platform.startswith('win'),
- "test doesn't work under Windows")
- def test_subinterps(self):
- # XXX only tested under Unix checkouts
+class EmbeddingTests(unittest.TestCase):
+ def setUp(self):
basepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
- oldcwd = os.getcwd()
+ exename = "_testembed"
+ if sys.platform.startswith("win"):
+ ext = ("_d" if "_d" in sys.executable else "") + ".exe"
+ exename += ext
+ exepath = os.path.dirname(sys.executable)
+ else:
+ exepath = os.path.join(basepath, "Modules")
+ self.test_exe = exe = os.path.join(exepath, exename)
+ if not os.path.exists(exe):
+ self.skipTest("%r doesn't exist" % exe)
# This is needed otherwise we get a fatal error:
# "Py_Initialize: Unable to get the locale encoding
# LookupError: no codec search functions registered: can't find encoding"
+ self.oldcwd = os.getcwd()
os.chdir(basepath)
+
+ def tearDown(self):
+ os.chdir(self.oldcwd)
+
+ def run_embedded_interpreter(self, *args):
+ """Runs a test in the embedded interpreter"""
+ cmd = [self.test_exe]
+ cmd.extend(args)
+ p = subprocess.Popen(cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ (out, err) = p.communicate()
+ self.assertEqual(p.returncode, 0,
+ "bad returncode %d, stderr is %r" %
+ (p.returncode, err))
+ return out.decode("latin1"), err.decode("latin1")
+
+ def test_subinterps(self):
+ # This is just a "don't crash" test
+ out, err = self.run_embedded_interpreter()
+ if support.verbose:
+ print()
+ print(out)
+ print(err)
+
+ @staticmethod
+ def _get_default_pipe_encoding():
+ rp, wp = os.pipe()
try:
- exe = os.path.join(basepath, "Modules", "_testembed")
- if not os.path.exists(exe):
- self.skipTest("%r doesn't exist" % exe)
- p = subprocess.Popen([exe],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- (out, err) = p.communicate()
- self.assertEqual(p.returncode, 0,
- "bad returncode %d, stderr is %r" %
- (p.returncode, err))
- if support.verbose:
- print()
- print(out.decode('latin1'))
- print(err.decode('latin1'))
+ with os.fdopen(wp, 'w') as w:
+ default_pipe_encoding = w.encoding
finally:
- os.chdir(oldcwd)
+ os.close(rp)
+ return default_pipe_encoding
+
+ def test_forced_io_encoding(self):
+ # Checks forced configuration of embedded interpreter IO streams
+ out, err = self.run_embedded_interpreter("forced_io_encoding")
+ if support.verbose:
+ print()
+ print(out)
+ print(err)
+ expected_stdin_encoding = sys.__stdin__.encoding
+ expected_pipe_encoding = self._get_default_pipe_encoding()
+ expected_output = os.linesep.join([
+ "--- Use defaults ---",
+ "Expected encoding: default",
+ "Expected errors: default",
+ "stdin: {0}:strict",
+ "stdout: {1}:strict",
+ "stderr: {1}:backslashreplace",
+ "--- Set errors only ---",
+ "Expected encoding: default",
+ "Expected errors: surrogateescape",
+ "stdin: {0}:surrogateescape",
+ "stdout: {1}:surrogateescape",
+ "stderr: {1}:backslashreplace",
+ "--- Set encoding only ---",
+ "Expected encoding: latin-1",
+ "Expected errors: default",
+ "stdin: latin-1:strict",
+ "stdout: latin-1:strict",
+ "stderr: latin-1:backslashreplace",
+ "--- Set encoding and errors ---",
+ "Expected encoding: latin-1",
+ "Expected errors: surrogateescape",
+ "stdin: latin-1:surrogateescape",
+ "stdout: latin-1:surrogateescape",
+ "stderr: latin-1:backslashreplace"]).format(expected_stdin_encoding,
+ expected_pipe_encoding)
+ # This is useful if we ever trip over odd platform behaviour
+ self.maxDiff = None
+ self.assertEqual(out.strip(), expected_output)
class SkipitemTest(unittest.TestCase):
@@ -355,8 +461,9 @@ class Test_testcapi(unittest.TestCase):
def test__testcapi(self):
for name in dir(_testcapi):
if name.startswith('test_'):
- test = getattr(_testcapi, name)
- test()
+ with self.subTest("internal", name=name):
+ test = getattr(_testcapi, name)
+ test()
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index a89d7e4..327c145 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -4,6 +4,7 @@
import test.support, unittest
import os
+import shutil
import sys
import subprocess
import tempfile
@@ -41,8 +42,10 @@ class CmdLineTest(unittest.TestCase):
def test_version(self):
version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii")
- rc, out, err = assert_python_ok('-V')
- self.assertTrue(err.startswith(version))
+ for switch in '-V', '--version':
+ rc, out, err = assert_python_ok(switch)
+ self.assertFalse(err.startswith(version))
+ self.assertTrue(out.startswith(version))
def test_verbose(self):
# -v causes imports to write to stderr. If the write to
@@ -54,14 +57,49 @@ class CmdLineTest(unittest.TestCase):
self.assertNotIn(b'stack overflow', err)
def test_xoptions(self):
- rc, out, err = assert_python_ok('-c', 'import sys; print(sys._xoptions)')
- opts = eval(out.splitlines()[0])
+ def get_xoptions(*args):
+ # use subprocess module directly because test.script_helper adds
+ # "-X faulthandler" to the command line
+ args = (sys.executable, '-E') + args
+ args += ('-c', 'import sys; print(sys._xoptions)')
+ out = subprocess.check_output(args)
+ opts = eval(out.splitlines()[0])
+ return opts
+
+ opts = get_xoptions()
self.assertEqual(opts, {})
- rc, out, err = assert_python_ok(
- '-Xa', '-Xb=c,d=e', '-c', 'import sys; print(sys._xoptions)')
- opts = eval(out.splitlines()[0])
+
+ opts = get_xoptions('-Xa', '-Xb=c,d=e')
self.assertEqual(opts, {'a': True, 'b': 'c,d=e'})
+ def test_showrefcount(self):
+ def run_python(*args):
+ # this is similar to assert_python_ok but doesn't strip
+ # the refcount from stderr. It can be replaced once
+ # assert_python_ok stops doing that.
+ cmd = [sys.executable]
+ cmd.extend(args)
+ PIPE = subprocess.PIPE
+ p = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE)
+ out, err = p.communicate()
+ p.stdout.close()
+ p.stderr.close()
+ rc = p.returncode
+ self.assertEqual(rc, 0)
+ return rc, out, err
+ code = 'import sys; print(sys._xoptions)'
+ # normally the refcount is hidden
+ rc, out, err = run_python('-c', code)
+ self.assertEqual(out.rstrip(), b'{}')
+ self.assertEqual(err, b'')
+ # "-X showrefcount" shows the refcount, but only in debug builds
+ rc, out, err = run_python('-X', 'showrefcount', '-c', code)
+ self.assertEqual(out.rstrip(), b"{'showrefcount': True}")
+ if hasattr(sys, 'gettotalrefcount'): # debug build
+ self.assertRegex(err, br'^\[\d+ refs, \d+ blocks\]')
+ else:
+ self.assertEqual(err, b'')
+
def test_run_module(self):
# Test expected operation of the '-m' switch
# Switch needs an argument
@@ -70,9 +108,9 @@ class CmdLineTest(unittest.TestCase):
assert_python_failure('-m', 'fnord43520xyz')
# Check the runpy module also gives an error for
# a nonexistent module
- assert_python_failure('-m', 'runpy', 'fnord43520xyz'),
+ assert_python_failure('-m', 'runpy', 'fnord43520xyz')
# All good if module is located and run successfully
- assert_python_ok('-m', 'timeit', '-n', '1'),
+ assert_python_ok('-m', 'timeit', '-n', '1')
def test_run_module_bug1764407(self):
# -m and -i need to play well together
@@ -213,6 +251,23 @@ class CmdLineTest(unittest.TestCase):
self.assertIn(path1.encode('ascii'), out)
self.assertIn(path2.encode('ascii'), out)
+ def test_empty_PYTHONPATH_issue16309(self):
+ # On Posix, it is documented that setting PATH to the
+ # empty string is equivalent to not setting PATH at all,
+ # which is an exception to the rule that in a string like
+ # "/bin::/usr/bin" the empty string in the middle gets
+ # interpreted as '.'
+ code = """if 1:
+ import sys
+ path = ":".join(sys.path)
+ path = path.encode("ascii", "backslashreplace")
+ sys.stdout.buffer.write(path)"""
+ rc1, out1, err1 = assert_python_ok('-c', code, PYTHONPATH="")
+ rc2, out2, err2 = assert_python_ok('-c', code, __isolated=False)
+ # regarding to Posix specification, outputs should be equal
+ # for empty and unset PYTHONPATH
+ self.assertEqual(out1, out2)
+
def test_displayhook_unencodable(self):
for encoding in ('ascii', 'latin-1', 'utf-8'):
env = os.environ.copy()
@@ -290,7 +345,7 @@ class CmdLineTest(unittest.TestCase):
rc, out, err = assert_python_ok('-c', code)
self.assertEqual(b'', out)
self.assertRegex(err.decode('ascii', 'ignore'),
- 'Exception OSError: .* ignored')
+ 'Exception ignored in.*\nOSError: .*')
def test_closed_stdout(self):
# Issue #13444: if stdout has been explicitly closed, we should
@@ -385,6 +440,31 @@ class CmdLineTest(unittest.TestCase):
self.assertEqual(b'', out)
+ def test_isolatedmode(self):
+ self.verify_valid_flag('-I')
+ self.verify_valid_flag('-IEs')
+ rc, out, err = assert_python_ok('-I', '-c',
+ 'from sys import flags as f; '
+ 'print(f.no_user_site, f.ignore_environment, f.isolated)',
+ # dummyvar to prevent extranous -E
+ dummyvar="")
+ self.assertEqual(out.strip(), b'1 1 1')
+ with test.support.temp_cwd() as tmpdir:
+ fake = os.path.join(tmpdir, "uuid.py")
+ main = os.path.join(tmpdir, "main.py")
+ with open(fake, "w") as f:
+ f.write("raise RuntimeError('isolated mode test')\n")
+ with open(main, "w") as f:
+ f.write("import uuid\n")
+ f.write("print('ok')\n")
+ self.assertRaises(subprocess.CalledProcessError,
+ subprocess.check_output,
+ [sys.executable, main], cwd=tmpdir,
+ stderr=subprocess.DEVNULL)
+ out = subprocess.check_output([sys.executable, "-I", main],
+ cwd=tmpdir)
+ self.assertEqual(out.strip(), b"ok")
+
def test_main():
test.support.run_unittest(CmdLineTest)
test.support.reap_children()
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index b4269b9..1e6746d 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -41,11 +41,28 @@ from importlib.machinery import BuiltinImporter
_loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
print('__loader__==%a' % _loader)
print('__file__==%a' % __file__)
-assertEqual(__cached__, None)
+print('__cached__==%a' % __cached__)
print('__package__==%r' % __package__)
+# Check PEP 451 details
+import os.path
+if __package__ is not None:
+ print('__main__ was located through the import system')
+ assertIdentical(__spec__.loader, __loader__)
+ expected_spec_name = os.path.splitext(os.path.basename(__file__))[0]
+ if __package__:
+ expected_spec_name = __package__ + "." + expected_spec_name
+ assertEqual(__spec__.name, expected_spec_name)
+ assertEqual(__spec__.parent, __package__)
+ assertIdentical(__spec__.submodule_search_locations, None)
+ assertEqual(__spec__.origin, __file__)
+ if __spec__.cached is not None:
+ assertEqual(__spec__.cached, __cached__)
# Check the sys module
import sys
assertIdentical(globals(), sys.modules[__name__].__dict__)
+if __spec__ is not None:
+ # XXX: We're not currently making __main__ available under its real name
+ pass # assertIdentical(globals(), sys.modules[__spec__.name].__dict__)
from test import test_cmd_line_script
example_args_list = test_cmd_line_script.example_args
assertEqual(sys.argv[1:], example_args_list)
@@ -123,7 +140,7 @@ class CmdLineTest(unittest.TestCase):
if not __debug__:
cmd_line_switches += ('-' + 'O' * sys.flags.optimize,)
run_args = cmd_line_switches + (script_name,) + tuple(example_args)
- rc, out, err = assert_python_ok(*run_args)
+ rc, out, err = assert_python_ok(*run_args, __isolated=False)
self._check_output(script_name, rc, out + err, expected_file,
expected_argv0, expected_path0,
expected_package, expected_loader)
@@ -294,7 +311,7 @@ class CmdLineTest(unittest.TestCase):
pkg_dir = os.path.join(script_dir, 'test_pkg')
make_pkg(pkg_dir, "import sys; print('init_argv0==%r' % sys.argv[0])")
script_name = _make_test_script(pkg_dir, 'script')
- rc, out, err = assert_python_ok('-m', 'test_pkg.script', *example_args)
+ rc, out, err = assert_python_ok('-m', 'test_pkg.script', *example_args, __isolated=False)
if verbose > 1:
print(out)
expected = "init_argv0==%r" % '-m'
@@ -311,7 +328,8 @@ class CmdLineTest(unittest.TestCase):
with open("-c", "w") as f:
f.write("data")
rc, out, err = assert_python_ok('-c',
- 'import sys; print("sys.path[0]==%r" % sys.path[0])')
+ 'import sys; print("sys.path[0]==%r" % sys.path[0])',
+ __isolated=False)
if verbose > 1:
print(out)
expected = "sys.path[0]==%r" % ''
@@ -325,7 +343,8 @@ class CmdLineTest(unittest.TestCase):
with support.change_cwd(path=script_dir):
with open("-m", "w") as f:
f.write("data")
- rc, out, err = assert_python_ok('-m', 'other', *example_args)
+ rc, out, err = assert_python_ok('-m', 'other', *example_args,
+ __isolated=False)
self._check_output(script_name, rc, out,
script_name, script_name, '', '',
importlib.machinery.SourceFileLoader)
@@ -386,6 +405,24 @@ class CmdLineTest(unittest.TestCase):
'stdout=%r stderr=%r' % (stdout, stderr))
self.assertEqual(0, rc)
+ def test_issue20500_exit_with_exception_value(self):
+ script = textwrap.dedent("""\
+ import sys
+ error = None
+ try:
+ raise ValueError('some text')
+ except ValueError as err:
+ error = err
+
+ if error:
+ sys.exit(error)
+ """)
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, 'script', script)
+ exitcode, stdout, stderr = assert_python_failure(script_name)
+ text = stderr.decode('ascii')
+ self.assertEqual(text, "some text")
+
def test_main():
support.run_unittest(CmdLineTest)
diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py
index adef170..5fd21dc 100644
--- a/Lib/test/test_code_module.py
+++ b/Lib/test/test_code_module.py
@@ -64,6 +64,20 @@ class TestInteractiveConsole(unittest.TestCase):
self.console.interact()
self.assertTrue(hook.called)
+ def test_banner(self):
+ # with banner
+ self.infunc.side_effect = EOFError('Finished')
+ self.console.interact(banner='Foo')
+ self.assertEqual(len(self.stderr.method_calls), 2)
+ banner_call = self.stderr.method_calls[0]
+ self.assertEqual(banner_call, ['write', ('Foo\n',), {}])
+
+ # no banner
+ self.stderr.reset_mock()
+ self.infunc.side_effect = EOFError('Finished')
+ self.console.interact(banner='')
+ self.assertEqual(len(self.stderr.method_calls), 1)
+
def test_main():
support.run_unittest(TestInteractiveConsole)
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
index fd88505..84804bb 100644
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -875,8 +875,6 @@ class CodecCallbackTest(unittest.TestCase):
with self.assertRaises(TypeError):
data.decode(encoding, "test.replacing")
-def test_main():
- test.support.run_unittest(CodecCallbackTest)
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index a8b3da0..9b62d5b 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -1,4 +1,5 @@
import codecs
+import contextlib
import io
import locale
import sys
@@ -342,8 +343,46 @@ class ReadTest(MixInCheckStateHandling):
self.assertEqual(reader.readline(), s5)
self.assertEqual(reader.readline(), "")
+ ill_formed_sequence_replace = "\ufffd"
+
+ def test_lone_surrogates(self):
+ self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding)
+ self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"),
+ "[\\udc80]".encode(self.encoding))
+ self.assertEqual("[\uDC80]".encode(self.encoding, "xmlcharrefreplace"),
+ "[&#56448;]".encode(self.encoding))
+ self.assertEqual("[\uDC80]".encode(self.encoding, "ignore"),
+ "[]".encode(self.encoding))
+ self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
+ "[?]".encode(self.encoding))
+
+ bom = "".encode(self.encoding)
+ for before, after in [("\U00010fff", "A"), ("[", "]"),
+ ("A", "\U00010fff")]:
+ before_sequence = before.encode(self.encoding)[len(bom):]
+ after_sequence = after.encode(self.encoding)[len(bom):]
+ test_string = before + "\uDC80" + after
+ test_sequence = (bom + before_sequence +
+ self.ill_formed_sequence + after_sequence)
+ self.assertRaises(UnicodeDecodeError, test_sequence.decode,
+ self.encoding)
+ self.assertEqual(test_string.encode(self.encoding,
+ "surrogatepass"),
+ test_sequence)
+ self.assertEqual(test_sequence.decode(self.encoding,
+ "surrogatepass"),
+ test_string)
+ self.assertEqual(test_sequence.decode(self.encoding, "ignore"),
+ before + after)
+ self.assertEqual(test_sequence.decode(self.encoding, "replace"),
+ before + self.ill_formed_sequence_replace + after)
+
class UTF32Test(ReadTest, unittest.TestCase):
encoding = "utf-32"
+ if sys.byteorder == 'little':
+ ill_formed_sequence = b"\x80\xdc\x00\x00"
+ else:
+ ill_formed_sequence = b"\x00\x00\xdc\x80"
spamle = (b'\xff\xfe\x00\x00'
b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00'
@@ -435,6 +474,7 @@ class UTF32Test(ReadTest, unittest.TestCase):
class UTF32LETest(ReadTest, unittest.TestCase):
encoding = "utf-32-le"
+ ill_formed_sequence = b"\x80\xdc\x00\x00"
def test_partial(self):
self.check_partial(
@@ -479,6 +519,7 @@ class UTF32LETest(ReadTest, unittest.TestCase):
class UTF32BETest(ReadTest, unittest.TestCase):
encoding = "utf-32-be"
+ ill_formed_sequence = b"\x00\x00\xdc\x80"
def test_partial(self):
self.check_partial(
@@ -524,6 +565,10 @@ class UTF32BETest(ReadTest, unittest.TestCase):
class UTF16Test(ReadTest, unittest.TestCase):
encoding = "utf-16"
+ if sys.byteorder == 'little':
+ ill_formed_sequence = b"\x80\xdc"
+ else:
+ ill_formed_sequence = b"\xdc\x80"
spamle = b'\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
spambe = b'\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m'
@@ -599,11 +644,14 @@ class UTF16Test(ReadTest, unittest.TestCase):
self.addCleanup(support.unlink, support.TESTFN)
with open(support.TESTFN, 'wb') as fp:
fp.write(s)
- with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader:
+ with support.check_warnings(('', DeprecationWarning)):
+ reader = codecs.open(support.TESTFN, 'U', encoding=self.encoding)
+ with reader:
self.assertEqual(reader.read(), s1)
class UTF16LETest(ReadTest, unittest.TestCase):
encoding = "utf-16-le"
+ ill_formed_sequence = b"\x80\xdc"
def test_partial(self):
self.check_partial(
@@ -647,6 +695,7 @@ class UTF16LETest(ReadTest, unittest.TestCase):
class UTF16BETest(ReadTest, unittest.TestCase):
encoding = "utf-16-be"
+ ill_formed_sequence = b"\xdc\x80"
def test_partial(self):
self.check_partial(
@@ -690,6 +739,8 @@ class UTF16BETest(ReadTest, unittest.TestCase):
class UTF8Test(ReadTest, unittest.TestCase):
encoding = "utf-8"
+ ill_formed_sequence = b"\xed\xb2\x80"
+ ill_formed_sequence_replace = "\ufffd" * 3
def test_partial(self):
self.check_partial(
@@ -719,18 +770,11 @@ class UTF8Test(ReadTest, unittest.TestCase):
u, u.encode(self.encoding))
def test_lone_surrogates(self):
- self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8")
- self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8")
- self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"),
- b'[\\udc80]')
- self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"),
- b'[&#56448;]')
- self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"),
+ super().test_lone_surrogates()
+ # not sure if this is making sense for
+ # UTF-16 and UTF-32
+ self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"),
b'[\x80]')
- self.assertEqual("[\uDC80]".encode("utf-8", "ignore"),
- b'[]')
- self.assertEqual("[\uDC80]".encode("utf-8", "replace"),
- b'[?]')
def test_surrogatepass_handler(self):
self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"),
@@ -913,15 +957,19 @@ class UTF7Test(ReadTest, unittest.TestCase):
(b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
]
for raw, expected in tests:
- self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
- raw, 'strict', True)
- self.assertEqual(raw.decode('utf-7', 'replace'), expected)
+ with self.subTest(raw=raw):
+ self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
+ raw, 'strict', True)
+ self.assertEqual(raw.decode('utf-7', 'replace'), expected)
def test_nonbmp(self):
self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
+ test_lone_surrogates = None
+
+
class UTF16ExTest(unittest.TestCase):
def test_errors(self):
@@ -946,7 +994,7 @@ class ReadBufferTest(unittest.TestCase):
self.assertRaises(TypeError, codecs.readbuffer_encode)
self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
-class UTF8SigTest(ReadTest, unittest.TestCase):
+class UTF8SigTest(UTF8Test, unittest.TestCase):
encoding = "utf-8-sig"
def test_partial(self):
@@ -1628,6 +1676,7 @@ all_unicode_encodings = [
"cp037",
"cp1006",
"cp1026",
+ "cp1125",
"cp1140",
"cp1250",
"cp1251",
@@ -2370,60 +2419,93 @@ bytes_transform_encodings = [
"quopri_codec",
"hex_codec",
]
+
+transform_aliases = {
+ "base64_codec": ["base64", "base_64"],
+ "uu_codec": ["uu"],
+ "quopri_codec": ["quopri", "quoted_printable", "quotedprintable"],
+ "hex_codec": ["hex"],
+ "rot_13": ["rot13"],
+}
+
try:
import zlib
except ImportError:
- pass
+ zlib = None
else:
bytes_transform_encodings.append("zlib_codec")
+ transform_aliases["zlib_codec"] = ["zip", "zlib"]
try:
import bz2
except ImportError:
pass
else:
bytes_transform_encodings.append("bz2_codec")
+ transform_aliases["bz2_codec"] = ["bz2"]
class TransformCodecTest(unittest.TestCase):
def test_basics(self):
binput = bytes(range(256))
for encoding in bytes_transform_encodings:
- # generic codecs interface
- (o, size) = codecs.getencoder(encoding)(binput)
- self.assertEqual(size, len(binput))
- (i, size) = codecs.getdecoder(encoding)(o)
- self.assertEqual(size, len(o))
- self.assertEqual(i, binput)
+ with self.subTest(encoding=encoding):
+ # generic codecs interface
+ (o, size) = codecs.getencoder(encoding)(binput)
+ self.assertEqual(size, len(binput))
+ (i, size) = codecs.getdecoder(encoding)(o)
+ self.assertEqual(size, len(o))
+ self.assertEqual(i, binput)
def test_read(self):
for encoding in bytes_transform_encodings:
- sin = codecs.encode(b"\x80", encoding)
- reader = codecs.getreader(encoding)(io.BytesIO(sin))
- sout = reader.read()
- self.assertEqual(sout, b"\x80")
+ with self.subTest(encoding=encoding):
+ sin = codecs.encode(b"\x80", encoding)
+ reader = codecs.getreader(encoding)(io.BytesIO(sin))
+ sout = reader.read()
+ self.assertEqual(sout, b"\x80")
def test_readline(self):
for encoding in bytes_transform_encodings:
- sin = codecs.encode(b"\x80", encoding)
- reader = codecs.getreader(encoding)(io.BytesIO(sin))
- sout = reader.readline()
- self.assertEqual(sout, b"\x80")
+ with self.subTest(encoding=encoding):
+ sin = codecs.encode(b"\x80", encoding)
+ reader = codecs.getreader(encoding)(io.BytesIO(sin))
+ sout = reader.readline()
+ self.assertEqual(sout, b"\x80")
+
+ def test_buffer_api_usage(self):
+ # We check all the transform codecs accept memoryview input
+ # for encoding and decoding
+ # and also that they roundtrip correctly
+ original = b"12345\x80"
+ for encoding in bytes_transform_encodings:
+ with self.subTest(encoding=encoding):
+ data = original
+ view = memoryview(data)
+ data = codecs.encode(data, encoding)
+ view_encoded = codecs.encode(view, encoding)
+ self.assertEqual(view_encoded, data)
+ view = memoryview(data)
+ data = codecs.decode(data, encoding)
+ self.assertEqual(data, original)
+ view_decoded = codecs.decode(view, encoding)
+ self.assertEqual(view_decoded, data)
def test_text_to_binary_blacklists_binary_transforms(self):
# Check binary -> binary codecs give a good error for str input
bad_input = "bad input type"
for encoding in bytes_transform_encodings:
- fmt = (r"{!r} is not a text encoding; "
- r"use codecs.encode\(\) to handle arbitrary codecs")
- msg = fmt.format(encoding)
- with self.assertRaisesRegex(LookupError, msg) as failure:
- bad_input.encode(encoding)
- self.assertIsNone(failure.exception.__cause__)
+ with self.subTest(encoding=encoding):
+ fmt = ( "{!r} is not a text encoding; "
+ "use codecs.encode\(\) to handle arbitrary codecs")
+ msg = fmt.format(encoding)
+ with self.assertRaisesRegex(LookupError, msg) as failure:
+ bad_input.encode(encoding)
+ self.assertIsNone(failure.exception.__cause__)
def test_text_to_binary_blacklists_text_transforms(self):
# Check str.encode gives a good error message for str -> str codecs
msg = (r"^'rot_13' is not a text encoding; "
- r"use codecs.encode\(\) to handle arbitrary codecs")
+ "use codecs.encode\(\) to handle arbitrary codecs")
with self.assertRaisesRegex(LookupError, msg):
"just an example message".encode("rot_13")
@@ -2432,23 +2514,224 @@ class TransformCodecTest(unittest.TestCase):
# message for binary -> binary codecs
data = b"encode first to ensure we meet any format restrictions"
for encoding in bytes_transform_encodings:
- encoded_data = codecs.encode(data, encoding)
- fmt = (r"{!r} is not a text encoding; "
- r"use codecs.decode\(\) to handle arbitrary codecs")
- msg = fmt.format(encoding)
- with self.assertRaisesRegex(LookupError, msg):
- encoded_data.decode(encoding)
- with self.assertRaisesRegex(LookupError, msg):
- bytearray(encoded_data).decode(encoding)
+ with self.subTest(encoding=encoding):
+ encoded_data = codecs.encode(data, encoding)
+ fmt = (r"{!r} is not a text encoding; "
+ "use codecs.decode\(\) to handle arbitrary codecs")
+ msg = fmt.format(encoding)
+ with self.assertRaisesRegex(LookupError, msg):
+ encoded_data.decode(encoding)
+ with self.assertRaisesRegex(LookupError, msg):
+ bytearray(encoded_data).decode(encoding)
def test_binary_to_text_blacklists_text_transforms(self):
# Check str -> str codec gives a good error for binary input
for bad_input in (b"immutable", bytearray(b"mutable")):
- msg = (r"^'rot_13' is not a text encoding; "
- r"use codecs.decode\(\) to handle arbitrary codecs")
- with self.assertRaisesRegex(LookupError, msg) as failure:
- bad_input.decode("rot_13")
- self.assertIsNone(failure.exception.__cause__)
+ with self.subTest(bad_input=bad_input):
+ msg = (r"^'rot_13' is not a text encoding; "
+ "use codecs.decode\(\) to handle arbitrary codecs")
+ with self.assertRaisesRegex(LookupError, msg) as failure:
+ bad_input.decode("rot_13")
+ self.assertIsNone(failure.exception.__cause__)
+
+ @unittest.skipUnless(zlib, "Requires zlib support")
+ def test_custom_zlib_error_is_wrapped(self):
+ # Check zlib codec gives a good error for malformed input
+ msg = "^decoding with 'zlib_codec' codec failed"
+ with self.assertRaisesRegex(Exception, msg) as failure:
+ codecs.decode(b"hello", "zlib_codec")
+ self.assertIsInstance(failure.exception.__cause__,
+ type(failure.exception))
+
+ def test_custom_hex_error_is_wrapped(self):
+ # Check hex codec gives a good error for malformed input
+ msg = "^decoding with 'hex_codec' codec failed"
+ with self.assertRaisesRegex(Exception, msg) as failure:
+ codecs.decode(b"hello", "hex_codec")
+ self.assertIsInstance(failure.exception.__cause__,
+ type(failure.exception))
+
+ # Unfortunately, the bz2 module throws OSError, which the codec
+ # machinery currently can't wrap :(
+
+ # Ensure codec aliases from http://bugs.python.org/issue7475 work
+ def test_aliases(self):
+ for codec_name, aliases in transform_aliases.items():
+ expected_name = codecs.lookup(codec_name).name
+ for alias in aliases:
+ with self.subTest(alias=alias):
+ info = codecs.lookup(alias)
+ self.assertEqual(info.name, expected_name)
+
+
+# The codec system tries to wrap exceptions in order to ensure the error
+# mentions the operation being performed and the codec involved. We
+# currently *only* want this to happen for relatively stateless
+# exceptions, where the only significant information they contain is their
+# type and a single str argument.
+
+# Use a local codec registry to avoid appearing to leak objects when
+# registering multiple seach functions
+_TEST_CODECS = {}
+
+def _get_test_codec(codec_name):
+ return _TEST_CODECS.get(codec_name)
+codecs.register(_get_test_codec) # Returns None, not usable as a decorator
+
+class ExceptionChainingTest(unittest.TestCase):
+
+ def setUp(self):
+ # There's no way to unregister a codec search function, so we just
+ # ensure we render this one fairly harmless after the test
+ # case finishes by using the test case repr as the codec name
+ # The codecs module normalizes codec names, although this doesn't
+ # appear to be formally documented...
+ # We also make sure we use a truly unique id for the custom codec
+ # to avoid issues with the codec cache when running these tests
+ # multiple times (e.g. when hunting for refleaks)
+ unique_id = repr(self) + str(id(self))
+ self.codec_name = encodings.normalize_encoding(unique_id).lower()
+
+ # We store the object to raise on the instance because of a bad
+ # interaction between the codec caching (which means we can't
+ # recreate the codec entry) and regrtest refleak hunting (which
+ # runs the same test instance multiple times). This means we
+ # need to ensure the codecs call back in to the instance to find
+ # out which exception to raise rather than binding them in a
+ # closure to an object that may change on the next run
+ self.obj_to_raise = RuntimeError
+
+ def tearDown(self):
+ _TEST_CODECS.pop(self.codec_name, None)
+
+ def set_codec(self, encode, decode):
+ codec_info = codecs.CodecInfo(encode, decode,
+ name=self.codec_name)
+ _TEST_CODECS[self.codec_name] = codec_info
+
+ @contextlib.contextmanager
+ def assertWrapped(self, operation, exc_type, msg):
+ full_msg = r"{} with {!r} codec failed \({}: {}\)".format(
+ operation, self.codec_name, exc_type.__name__, msg)
+ with self.assertRaisesRegex(exc_type, full_msg) as caught:
+ yield caught
+ self.assertIsInstance(caught.exception.__cause__, exc_type)
+ self.assertIsNotNone(caught.exception.__cause__.__traceback__)
+
+ def raise_obj(self, *args, **kwds):
+ # Helper to dynamically change the object raised by a test codec
+ raise self.obj_to_raise
+
+ def check_wrapped(self, obj_to_raise, msg, exc_type=RuntimeError):
+ self.obj_to_raise = obj_to_raise
+ self.set_codec(self.raise_obj, self.raise_obj)
+ with self.assertWrapped("encoding", exc_type, msg):
+ "str_input".encode(self.codec_name)
+ with self.assertWrapped("encoding", exc_type, msg):
+ codecs.encode("str_input", self.codec_name)
+ with self.assertWrapped("decoding", exc_type, msg):
+ b"bytes input".decode(self.codec_name)
+ with self.assertWrapped("decoding", exc_type, msg):
+ codecs.decode(b"bytes input", self.codec_name)
+
+ def test_raise_by_type(self):
+ self.check_wrapped(RuntimeError, "")
+
+ def test_raise_by_value(self):
+ msg = "This should be wrapped"
+ self.check_wrapped(RuntimeError(msg), msg)
+
+ def test_raise_grandchild_subclass_exact_size(self):
+ msg = "This should be wrapped"
+ class MyRuntimeError(RuntimeError):
+ __slots__ = ()
+ self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError)
+
+ def test_raise_subclass_with_weakref_support(self):
+ msg = "This should be wrapped"
+ class MyRuntimeError(RuntimeError):
+ pass
+ self.check_wrapped(MyRuntimeError(msg), msg, MyRuntimeError)
+
+ def check_not_wrapped(self, obj_to_raise, msg):
+ def raise_obj(*args, **kwds):
+ raise obj_to_raise
+ self.set_codec(raise_obj, raise_obj)
+ with self.assertRaisesRegex(RuntimeError, msg):
+ "str input".encode(self.codec_name)
+ with self.assertRaisesRegex(RuntimeError, msg):
+ codecs.encode("str input", self.codec_name)
+ with self.assertRaisesRegex(RuntimeError, msg):
+ b"bytes input".decode(self.codec_name)
+ with self.assertRaisesRegex(RuntimeError, msg):
+ codecs.decode(b"bytes input", self.codec_name)
+
+ def test_init_override_is_not_wrapped(self):
+ class CustomInit(RuntimeError):
+ def __init__(self):
+ pass
+ self.check_not_wrapped(CustomInit, "")
+
+ def test_new_override_is_not_wrapped(self):
+ class CustomNew(RuntimeError):
+ def __new__(cls):
+ return super().__new__(cls)
+ self.check_not_wrapped(CustomNew, "")
+
+ def test_instance_attribute_is_not_wrapped(self):
+ msg = "This should NOT be wrapped"
+ exc = RuntimeError(msg)
+ exc.attr = 1
+ self.check_not_wrapped(exc, "^{}$".format(msg))
+
+ def test_non_str_arg_is_not_wrapped(self):
+ self.check_not_wrapped(RuntimeError(1), "1")
+
+ def test_multiple_args_is_not_wrapped(self):
+ msg_re = r"^\('a', 'b', 'c'\)$"
+ self.check_not_wrapped(RuntimeError('a', 'b', 'c'), msg_re)
+
+ # http://bugs.python.org/issue19609
+ def test_codec_lookup_failure_not_wrapped(self):
+ msg = "^unknown encoding: {}$".format(self.codec_name)
+ # The initial codec lookup should not be wrapped
+ with self.assertRaisesRegex(LookupError, msg):
+ "str input".encode(self.codec_name)
+ with self.assertRaisesRegex(LookupError, msg):
+ codecs.encode("str input", self.codec_name)
+ with self.assertRaisesRegex(LookupError, msg):
+ b"bytes input".decode(self.codec_name)
+ with self.assertRaisesRegex(LookupError, msg):
+ codecs.decode(b"bytes input", self.codec_name)
+
+ def test_unflagged_non_text_codec_handling(self):
+ # The stdlib non-text codecs are now marked so they're
+ # pre-emptively skipped by the text model related methods
+ # However, third party codecs won't be flagged, so we still make
+ # sure the case where an inappropriate output type is produced is
+ # handled appropriately
+ def encode_to_str(*args, **kwds):
+ return "not bytes!", 0
+ def decode_to_bytes(*args, **kwds):
+ return b"not str!", 0
+ self.set_codec(encode_to_str, decode_to_bytes)
+ # No input or output type checks on the codecs module functions
+ encoded = codecs.encode(None, self.codec_name)
+ self.assertEqual(encoded, "not bytes!")
+ decoded = codecs.decode(None, self.codec_name)
+ self.assertEqual(decoded, b"not str!")
+ # Text model methods should complain
+ fmt = (r"^{!r} encoder returned 'str' instead of 'bytes'; "
+ "use codecs.encode\(\) to encode to arbitrary types$")
+ msg = fmt.format(self.codec_name)
+ with self.assertRaisesRegex(TypeError, msg):
+ "str_input".encode(self.codec_name)
+ fmt = (r"^{!r} decoder returned 'bytes' instead of 'str'; "
+ "use codecs.decode\(\) to decode to arbitrary types$")
+ msg = fmt.format(self.codec_name)
+ with self.assertRaisesRegex(TypeError, msg):
+ b"bytes input".decode(self.codec_name)
+
@unittest.skipUnless(sys.platform == 'win32',
@@ -2460,8 +2743,8 @@ class CodePageTest(unittest.TestCase):
def test_invalid_code_page(self):
self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
- self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a')
- self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a')
+ self.assertRaises(OSError, codecs.code_page_encode, 123, 'a')
+ self.assertRaises(OSError, codecs.code_page_decode, 123, b'a')
def test_code_page_name(self):
self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
diff --git a/Lib/test/test_coding.py b/Lib/test/test_coding.py
deleted file mode 100644
index bdbb51f..0000000
--- a/Lib/test/test_coding.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import unittest
-from test.support import TESTFN, unlink, unload
-import importlib, os, sys, subprocess
-
-class CodingTest(unittest.TestCase):
- def test_bad_coding(self):
- module_name = 'bad_coding'
- self.verify_bad_module(module_name)
-
- def test_bad_coding2(self):
- module_name = 'bad_coding2'
- self.verify_bad_module(module_name)
-
- def verify_bad_module(self, module_name):
- self.assertRaises(SyntaxError, __import__, 'test.' + module_name)
-
- path = os.path.dirname(__file__)
- filename = os.path.join(path, module_name + '.py')
- with open(filename, "rb") as fp:
- bytes = fp.read()
- self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')
-
- def test_exec_valid_coding(self):
- d = {}
- exec(b'# coding: cp949\na = "\xaa\xa7"\n', d)
- self.assertEqual(d['a'], '\u3047')
-
- def test_file_parse(self):
- # issue1134: all encodings outside latin-1 and utf-8 fail on
- # multiline strings and long lines (>512 columns)
- unload(TESTFN)
- filename = TESTFN + ".py"
- f = open(filename, "w", encoding="cp1252")
- sys.path.insert(0, os.curdir)
- try:
- with f:
- f.write("# -*- coding: cp1252 -*-\n")
- f.write("'''A short string\n")
- f.write("'''\n")
- f.write("'A very long string %s'\n" % ("X" * 1000))
-
- importlib.invalidate_caches()
- __import__(TESTFN)
- finally:
- del sys.path[0]
- unlink(filename)
- unlink(filename + "c")
- unlink(filename + "o")
- unload(TESTFN)
-
- def test_error_from_string(self):
- # See http://bugs.python.org/issue6289
- input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8')
- with self.assertRaises(SyntaxError) as c:
- compile(input, "<string>", "exec")
- expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \
- "ordinal not in range(128)"
- self.assertTrue(c.exception.args[0].startswith(expected),
- msg=c.exception.args[0])
-
- def test_20731(self):
- sub = subprocess.Popen([sys.executable,
- os.path.join(os.path.dirname(__file__),
- 'coding20731.py')],
- stderr=subprocess.PIPE)
- err = sub.communicate()[1]
- self.assertEqual(sub.returncode, 0)
- self.assertNotIn(b'SyntaxError', err)
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py
index ff52755..ee28a6c 100644
--- a/Lib/test/test_collections.py
+++ b/Lib/test/test_collections.py
@@ -112,6 +112,38 @@ class TestChainMap(unittest.TestCase):
self.assertEqual(dict(d), dict(a=1, b=2, c=30))
self.assertEqual(dict(d.items()), dict(a=1, b=2, c=30))
+ def test_new_child(self):
+ 'Tests for changes for issue #16613.'
+ c = ChainMap()
+ c['a'] = 1
+ c['b'] = 2
+ m = {'b':20, 'c': 30}
+ d = c.new_child(m)
+ self.assertEqual(d.maps, [{'b':20, 'c':30}, {'a':1, 'b':2}]) # check internal state
+ self.assertIs(m, d.maps[0])
+
+ # Use a different map than a dict
+ class lowerdict(dict):
+ def __getitem__(self, key):
+ if isinstance(key, str):
+ key = key.lower()
+ return dict.__getitem__(self, key)
+ def __contains__(self, key):
+ if isinstance(key, str):
+ key = key.lower()
+ return dict.__contains__(self, key)
+
+ c = ChainMap()
+ c['a'] = 1
+ c['b'] = 2
+ m = lowerdict(b=20, c=30)
+ d = c.new_child(m)
+ self.assertIs(m, d.maps[0])
+ for key in 'abc': # check contains
+ self.assertIn(key, d)
+ for k, v in dict(a=1, B=20, C=30, z=100).items(): # check get
+ self.assertEqual(d.get(k, 100), v)
+
################################################################################
### Named Tuples
@@ -269,7 +301,7 @@ class TestNamedTuple(unittest.TestCase):
for module in (pickle,):
loads = getattr(module, 'loads')
dumps = getattr(module, 'dumps')
- for protocol in -1, 0, 1, 2:
+ for protocol in range(-1, module.HIGHEST_PROTOCOL + 1):
q = loads(dumps(p, protocol))
self.assertEqual(p, q)
self.assertEqual(p._fields, q._fields)
@@ -750,6 +782,8 @@ class TestCollectionABCs(ABCTestCase):
self.assertTrue(issubclass(sample, Sequence))
self.assertIsInstance(range(10), Sequence)
self.assertTrue(issubclass(range, Sequence))
+ self.assertIsInstance(memoryview(b""), Sequence)
+ self.assertTrue(issubclass(memoryview, Sequence))
self.assertTrue(issubclass(str, Sequence))
self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__',
'__getitem__')
@@ -904,23 +938,26 @@ class TestCounter(unittest.TestCase):
words = Counter('which witch had which witches wrist watch'.split())
update_test = Counter()
update_test.update(words)
- for i, dup in enumerate([
- words.copy(),
- copy.copy(words),
- copy.deepcopy(words),
- pickle.loads(pickle.dumps(words, 0)),
- pickle.loads(pickle.dumps(words, 1)),
- pickle.loads(pickle.dumps(words, 2)),
- pickle.loads(pickle.dumps(words, -1)),
- eval(repr(words)),
- update_test,
- Counter(words),
- ]):
- msg = (i, dup, words)
- self.assertTrue(dup is not words)
- self.assertEqual(dup, words)
- self.assertEqual(len(dup), len(words))
- self.assertEqual(type(dup), type(words))
+ for label, dup in [
+ ('words.copy()', words.copy()),
+ ('copy.copy(words)', copy.copy(words)),
+ ('copy.deepcopy(words)', copy.deepcopy(words)),
+ ('pickle.loads(pickle.dumps(words, 0))',
+ pickle.loads(pickle.dumps(words, 0))),
+ ('pickle.loads(pickle.dumps(words, 1))',
+ pickle.loads(pickle.dumps(words, 1))),
+ ('pickle.loads(pickle.dumps(words, 2))',
+ pickle.loads(pickle.dumps(words, 2))),
+ ('pickle.loads(pickle.dumps(words, -1))',
+ pickle.loads(pickle.dumps(words, -1))),
+ ('eval(repr(words))', eval(repr(words))),
+ ('update_test', update_test),
+ ('Counter(words)', Counter(words)),
+ ]:
+ with self.subTest(label=label):
+ msg = "\ncopy: %s\nwords: %s" % (dup, words)
+ self.assertIsNot(dup, words, msg)
+ self.assertEqual(dup, words)
def test_copy_subclass(self):
class MyCounter(Counter):
@@ -1043,8 +1080,10 @@ class TestCounter(unittest.TestCase):
# test fidelity to the pure python version
c = CounterSubclassWithSetItem('abracadabra')
self.assertTrue(c.called)
+ self.assertEqual(dict(c), {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r':2 })
c = CounterSubclassWithGet('abracadabra')
self.assertTrue(c.called)
+ self.assertEqual(dict(c), {'a': 5, 'b': 2, 'c': 1, 'd': 1, 'r':2 })
################################################################################
@@ -1205,24 +1244,28 @@ class TestOrderedDict(unittest.TestCase):
od = OrderedDict(pairs)
update_test = OrderedDict()
update_test.update(od)
- for i, dup in enumerate([
- od.copy(),
- copy.copy(od),
- copy.deepcopy(od),
- pickle.loads(pickle.dumps(od, 0)),
- pickle.loads(pickle.dumps(od, 1)),
- pickle.loads(pickle.dumps(od, 2)),
- pickle.loads(pickle.dumps(od, 3)),
- pickle.loads(pickle.dumps(od, -1)),
- eval(repr(od)),
- update_test,
- OrderedDict(od),
- ]):
- self.assertTrue(dup is not od)
- self.assertEqual(dup, od)
- self.assertEqual(list(dup.items()), list(od.items()))
- self.assertEqual(len(dup), len(od))
- self.assertEqual(type(dup), type(od))
+ for label, dup in [
+ ('od.copy()', od.copy()),
+ ('copy.copy(od)', copy.copy(od)),
+ ('copy.deepcopy(od)', copy.deepcopy(od)),
+ ('pickle.loads(pickle.dumps(od, 0))',
+ pickle.loads(pickle.dumps(od, 0))),
+ ('pickle.loads(pickle.dumps(od, 1))',
+ pickle.loads(pickle.dumps(od, 1))),
+ ('pickle.loads(pickle.dumps(od, 2))',
+ pickle.loads(pickle.dumps(od, 2))),
+ ('pickle.loads(pickle.dumps(od, 3))',
+ pickle.loads(pickle.dumps(od, 3))),
+ ('pickle.loads(pickle.dumps(od, -1))',
+ pickle.loads(pickle.dumps(od, -1))),
+ ('eval(repr(od))', eval(repr(od))),
+ ('update_test', update_test),
+ ('OrderedDict(od)', OrderedDict(od)),
+ ]:
+ with self.subTest(label=label):
+ msg = "\ncopy: %s\nod: %s" % (dup, od)
+ self.assertIsNot(dup, od, msg)
+ self.assertEqual(dup, od)
def test_yaml_linkage(self):
# Verify that __reduce__ is setup in a way that supports PyYAML's dump() feature.
@@ -1237,9 +1280,18 @@ class TestOrderedDict(unittest.TestCase):
# do not save instance dictionary if not needed
pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
od = OrderedDict(pairs)
- self.assertEqual(len(od.__reduce__()), 2)
+ self.assertIsNone(od.__reduce__()[2])
od.x = 10
- self.assertEqual(len(od.__reduce__()), 3)
+ self.assertIsNotNone(od.__reduce__()[2])
+
+ def test_pickle_recursive(self):
+ od = OrderedDict()
+ od[1] = od
+ for proto in range(-1, pickle.HIGHEST_PROTOCOL + 1):
+ dup = pickle.loads(pickle.dumps(od, proto))
+ self.assertIsNot(dup, od)
+ self.assertEqual(list(dup.keys()), [1])
+ self.assertIs(dup[1], dup)
def test_repr(self):
od = OrderedDict([('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)])
diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py
index e405b8a..a24e3ad 100644
--- a/Lib/test/test_colorsys.py
+++ b/Lib/test/test_colorsys.py
@@ -1,4 +1,4 @@
-import unittest, test.support
+import unittest
import colorsys
def frange(start, stop, step):
@@ -69,8 +69,32 @@ class ColorsysTest(unittest.TestCase):
self.assertTripleEqual(hls, colorsys.rgb_to_hls(*rgb))
self.assertTripleEqual(rgb, colorsys.hls_to_rgb(*hls))
-def test_main():
- test.support.run_unittest(ColorsysTest)
+ def test_yiq_roundtrip(self):
+ for r in frange(0.0, 1.0, 0.2):
+ for g in frange(0.0, 1.0, 0.2):
+ for b in frange(0.0, 1.0, 0.2):
+ rgb = (r, g, b)
+ self.assertTripleEqual(
+ rgb,
+ colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb))
+ )
+
+ def test_yiq_values(self):
+ values = [
+ # rgb, yiq
+ ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)), # black
+ ((0.0, 0.0, 1.0), (0.11, -0.3217, 0.3121)), # blue
+ ((0.0, 1.0, 0.0), (0.59, -0.2773, -0.5251)), # green
+ ((0.0, 1.0, 1.0), (0.7, -0.599, -0.213)), # cyan
+ ((1.0, 0.0, 0.0), (0.3, 0.599, 0.213)), # red
+ ((1.0, 0.0, 1.0), (0.41, 0.2773, 0.5251)), # purple
+ ((1.0, 1.0, 0.0), (0.89, 0.3217, -0.3121)), # yellow
+ ((1.0, 1.0, 1.0), (1.0, 0.0, 0.0)), # white
+ ((0.5, 0.5, 0.5), (0.5, 0.0, 0.0)), # grey
+ ]
+ for (rgb, yiq) in values:
+ self.assertTripleEqual(yiq, colorsys.rgb_to_yiq(*rgb))
+ self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq))
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 0505c52..2a42238 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -1,11 +1,10 @@
import sys
import compileall
-import imp
+import importlib.util
import os
import py_compile
import shutil
import struct
-import subprocess
import tempfile
import time
import unittest
@@ -18,11 +17,11 @@ class CompileallTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.mkdtemp()
self.source_path = os.path.join(self.directory, '_test.py')
- self.bc_path = imp.cache_from_source(self.source_path)
+ self.bc_path = importlib.util.cache_from_source(self.source_path)
with open(self.source_path, 'w') as file:
file.write('x = 123\n')
self.source_path2 = os.path.join(self.directory, '_test2.py')
- self.bc_path2 = imp.cache_from_source(self.source_path2)
+ self.bc_path2 = importlib.util.cache_from_source(self.source_path2)
shutil.copyfile(self.source_path, self.source_path2)
self.subdirectory = os.path.join(self.directory, '_subdir')
os.mkdir(self.subdirectory)
@@ -36,7 +35,7 @@ class CompileallTests(unittest.TestCase):
with open(self.bc_path, 'rb') as file:
data = file.read(8)
mtime = int(os.stat(self.source_path).st_mtime)
- compare = struct.pack('<4sl', imp.get_magic(), mtime)
+ compare = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, mtime)
return data, compare
@unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()')
@@ -56,7 +55,8 @@ class CompileallTests(unittest.TestCase):
def test_mtime(self):
# Test a change in mtime leads to a new .pyc.
- self.recreation_check(struct.pack('<4sl', imp.get_magic(), 1))
+ self.recreation_check(struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
+ 1))
def test_magic_number(self):
# Test a change in mtime leads to a new .pyc.
@@ -96,14 +96,14 @@ class CompileallTests(unittest.TestCase):
# interpreter's creates the correct file names
optimize = 1 if __debug__ else 0
compileall.compile_dir(self.directory, quiet=True, optimize=optimize)
- cached = imp.cache_from_source(self.source_path,
- debug_override=not optimize)
+ cached = importlib.util.cache_from_source(self.source_path,
+ debug_override=not optimize)
self.assertTrue(os.path.isfile(cached))
- cached2 = imp.cache_from_source(self.source_path2,
- debug_override=not optimize)
+ cached2 = importlib.util.cache_from_source(self.source_path2,
+ debug_override=not optimize)
self.assertTrue(os.path.isfile(cached2))
- cached3 = imp.cache_from_source(self.source_path3,
- debug_override=not optimize)
+ cached3 = importlib.util.cache_from_source(self.source_path3,
+ debug_override=not optimize)
self.assertTrue(os.path.isfile(cached3))
@@ -151,10 +151,12 @@ class CommandLineTests(unittest.TestCase):
return rc, out, err
def assertCompiled(self, fn):
- self.assertTrue(os.path.exists(imp.cache_from_source(fn)))
+ path = importlib.util.cache_from_source(fn)
+ self.assertTrue(os.path.exists(path))
def assertNotCompiled(self, fn):
- self.assertFalse(os.path.exists(imp.cache_from_source(fn)))
+ path = importlib.util.cache_from_source(fn)
+ self.assertFalse(os.path.exists(path))
def setUp(self):
self.addCleanup(self._cleanup)
@@ -180,7 +182,7 @@ class CommandLineTests(unittest.TestCase):
def test_no_args_respects_force_flag(self):
bazfn = script_helper.make_script(self.directory, 'baz', '')
self.assertRunOK(PYTHONPATH=self.directory)
- pycpath = imp.cache_from_source(bazfn)
+ pycpath = importlib.util.cache_from_source(bazfn)
# Set atime/mtime backward to avoid file timestamp resolution issues
os.utime(pycpath, (time.time()-60,)*2)
mtime = os.stat(pycpath).st_mtime
@@ -212,8 +214,8 @@ class CommandLineTests(unittest.TestCase):
['-m', 'compileall', '-q', self.pkgdir]))
# Verify the __pycache__ directory contents.
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
- expected = sorted(base.format(imp.get_tag(), ext) for base in
- ('__init__.{}.{}', 'bar.{}.{}'))
+ expected = sorted(base.format(sys.implementation.cache_tag, ext)
+ for base in ('__init__.{}.{}', 'bar.{}.{}'))
self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected)
# Make sure there are no .pyc files in the source directory.
self.assertFalse([fn for fn in os.listdir(self.pkgdir)
@@ -246,7 +248,7 @@ class CommandLineTests(unittest.TestCase):
def test_force(self):
self.assertRunOK('-q', self.pkgdir)
- pycpath = imp.cache_from_source(self.barfn)
+ pycpath = importlib.util.cache_from_source(self.barfn)
# set atime/mtime backward to avoid file timestamp resolution issues
os.utime(pycpath, (time.time()-60,)*2)
mtime = os.stat(pycpath).st_mtime
@@ -310,10 +312,10 @@ class CommandLineTests(unittest.TestCase):
bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception')
self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir)
fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz')
- pyc = imp.cache_from_source(bazfn)
+ pyc = importlib.util.cache_from_source(bazfn)
os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc'))
os.remove(bazfn)
- rc, out, err = script_helper.assert_python_failure(fn)
+ rc, out, err = script_helper.assert_python_failure(fn, __isolated=False)
self.assertRegex(err, b'File "dinsdale')
def test_include_bad_file(self):
@@ -321,7 +323,7 @@ class CommandLineTests(unittest.TestCase):
'-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir)
self.assertRegex(out, b'rror.*nosuchfile')
self.assertNotRegex(err, b'Traceback')
- self.assertFalse(os.path.exists(imp.cache_from_source(
+ self.assertFalse(os.path.exists(importlib.util.cache_from_source(
self.pkgdir_cachedir)))
def test_include_file_with_arg(self):
@@ -378,13 +380,5 @@ class CommandLineTests(unittest.TestCase):
self.assertRegex(out, b"Can't list 'badfilename'")
-def test_main():
- support.run_unittest(
- CommandLineTests,
- CompileallTests,
- EncodingTest,
- )
-
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py
index 1bf4097..cd55375 100644
--- a/Lib/test/test_complex.py
+++ b/Lib/test/test_complex.py
@@ -220,6 +220,8 @@ class ComplexTest(unittest.TestCase):
self.assertRaises(TypeError, complex, OS(None))
self.assertRaises(TypeError, complex, NS(None))
self.assertRaises(TypeError, complex, {})
+ self.assertRaises(TypeError, complex, NS(1.5))
+ self.assertRaises(TypeError, complex, NS(1))
self.assertAlmostEqual(complex("1+10j"), 1+10j)
self.assertAlmostEqual(complex(10), 10+0j)
@@ -301,6 +303,7 @@ class ComplexTest(unittest.TestCase):
self.assertRaises(TypeError, float, 5+3j)
self.assertRaises(ValueError, complex, "")
self.assertRaises(TypeError, complex, None)
+ self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None)
self.assertRaises(ValueError, complex, "\0")
self.assertRaises(ValueError, complex, "3\09")
self.assertRaises(TypeError, complex, "1", "2")
diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py
index 04c4c37..c74b2ca 100644
--- a/Lib/test/test_concurrent_futures.py
+++ b/Lib/test/test_concurrent_futures.py
@@ -15,6 +15,7 @@ import sys
import threading
import time
import unittest
+import weakref
from concurrent import futures
from concurrent.futures._base import (
@@ -52,6 +53,11 @@ def sleep_and_print(t, msg):
sys.stdout.flush()
+class MyObject(object):
+ def my_method(self):
+ pass
+
+
class ExecutorMixin:
worker_count = 5
@@ -403,6 +409,22 @@ class ExecutorTest:
self.executor.map(str, [2] * (self.worker_count + 1))
self.executor.shutdown()
+ @test.support.cpython_only
+ def test_no_stale_references(self):
+ # Issue #16284: check that the executors don't unnecessarily hang onto
+ # references.
+ my_object = MyObject()
+ my_object_collected = threading.Event()
+ my_object_callback = weakref.ref(
+ my_object, lambda obj: my_object_collected.set())
+ # Deliberately discarding the future.
+ self.executor.submit(my_object.my_method)
+ del my_object
+
+ collected = my_object_collected.wait(timeout=5.0)
+ self.assertTrue(collected,
+ "Stale reference not collected within timeout.")
+
class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, unittest.TestCase):
def test_map_submits_without_iteration(self):
diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py
index e38d043..39cc776 100644
--- a/Lib/test/test_contextlib.py
+++ b/Lib/test/test_contextlib.py
@@ -1,5 +1,6 @@
"""Unit tests for contextlib.py, and other context managers."""
+import io
import sys
import tempfile
import unittest
@@ -100,15 +101,25 @@ class ContextManagerTestCase(unittest.TestCase):
self.assertEqual(baz.__name__,'baz')
self.assertEqual(baz.foo, 'bar')
- @unittest.skipIf(sys.flags.optimize >= 2,
- "Docstrings are omitted with -O2 and above")
+ @support.requires_docstrings
def test_contextmanager_doc_attrib(self):
baz = self._create_contextmanager_attribs()
self.assertEqual(baz.__doc__, "Whee!")
+ @support.requires_docstrings
+ def test_instance_docstring_given_cm_docstring(self):
+ baz = self._create_contextmanager_attribs()(None)
+ self.assertEqual(baz.__doc__, "Whee!")
+
+
class ClosingTestCase(unittest.TestCase):
- # XXX This needs more work
+ @support.requires_docstrings
+ def test_instance_docs(self):
+ # Issue 19330: ensure context manager instances have good docstrings
+ cm_docstring = closing.__doc__
+ obj = closing(None)
+ self.assertEqual(obj.__doc__, cm_docstring)
def test_closing(self):
state = []
@@ -204,6 +215,7 @@ class LockContextTestCase(unittest.TestCase):
class mycontext(ContextDecorator):
+ """Example decoration-compatible context manager for testing"""
started = False
exc = None
catch = False
@@ -219,6 +231,13 @@ class mycontext(ContextDecorator):
class TestContextDecorator(unittest.TestCase):
+ @support.requires_docstrings
+ def test_instance_docs(self):
+ # Issue 19330: ensure context manager instances have good docstrings
+ cm_docstring = mycontext.__doc__
+ obj = mycontext()
+ self.assertEqual(obj.__doc__, cm_docstring)
+
def test_contextdecorator(self):
context = mycontext()
with context as result:
@@ -372,6 +391,13 @@ class TestContextDecorator(unittest.TestCase):
class TestExitStack(unittest.TestCase):
+ @support.requires_docstrings
+ def test_instance_docs(self):
+ # Issue 19330: ensure context manager instances have good docstrings
+ cm_docstring = ExitStack.__doc__
+ obj = ExitStack()
+ self.assertEqual(obj.__doc__, cm_docstring)
+
def test_no_resources(self):
with ExitStack():
pass
@@ -692,10 +718,111 @@ class TestExitStack(unittest.TestCase):
stack.push(cm)
self.assertIs(stack._exit_callbacks[-1], cm)
+class TestRedirectStdout(unittest.TestCase):
+
+ @support.requires_docstrings
+ def test_instance_docs(self):
+ # Issue 19330: ensure context manager instances have good docstrings
+ cm_docstring = redirect_stdout.__doc__
+ obj = redirect_stdout(None)
+ self.assertEqual(obj.__doc__, cm_docstring)
+
+ def test_no_redirect_in_init(self):
+ orig_stdout = sys.stdout
+ redirect_stdout(None)
+ self.assertIs(sys.stdout, orig_stdout)
+
+ def test_redirect_to_string_io(self):
+ f = io.StringIO()
+ msg = "Consider an API like help(), which prints directly to stdout"
+ orig_stdout = sys.stdout
+ with redirect_stdout(f):
+ print(msg)
+ self.assertIs(sys.stdout, orig_stdout)
+ s = f.getvalue().strip()
+ self.assertEqual(s, msg)
+
+ def test_enter_result_is_target(self):
+ f = io.StringIO()
+ with redirect_stdout(f) as enter_result:
+ self.assertIs(enter_result, f)
+
+ def test_cm_is_reusable(self):
+ f = io.StringIO()
+ write_to_f = redirect_stdout(f)
+ orig_stdout = sys.stdout
+ with write_to_f:
+ print("Hello", end=" ")
+ with write_to_f:
+ print("World!")
+ self.assertIs(sys.stdout, orig_stdout)
+ s = f.getvalue()
+ self.assertEqual(s, "Hello World!\n")
+
+ def test_cm_is_reentrant(self):
+ f = io.StringIO()
+ write_to_f = redirect_stdout(f)
+ orig_stdout = sys.stdout
+ with write_to_f:
+ print("Hello", end=" ")
+ with write_to_f:
+ print("World!")
+ self.assertIs(sys.stdout, orig_stdout)
+ s = f.getvalue()
+ self.assertEqual(s, "Hello World!\n")
+
+
+class TestSuppress(unittest.TestCase):
+
+ @support.requires_docstrings
+ def test_instance_docs(self):
+ # Issue 19330: ensure context manager instances have good docstrings
+ cm_docstring = suppress.__doc__
+ obj = suppress()
+ self.assertEqual(obj.__doc__, cm_docstring)
+
+ def test_no_result_from_enter(self):
+ with suppress(ValueError) as enter_result:
+ self.assertIsNone(enter_result)
+
+ def test_no_exception(self):
+ with suppress(ValueError):
+ self.assertEqual(pow(2, 5), 32)
+
+ def test_exact_exception(self):
+ with suppress(TypeError):
+ len(5)
+
+ def test_exception_hierarchy(self):
+ with suppress(LookupError):
+ 'Hello'[50]
+
+ def test_other_exception(self):
+ with self.assertRaises(ZeroDivisionError):
+ with suppress(TypeError):
+ 1/0
+
+ def test_no_args(self):
+ with self.assertRaises(ZeroDivisionError):
+ with suppress():
+ 1/0
-# This is needed to make the test actually run under regrtest.py!
-def test_main():
- support.run_unittest(__name__)
+ def test_multiple_exception_args(self):
+ with suppress(ZeroDivisionError, TypeError):
+ 1/0
+ with suppress(ZeroDivisionError, TypeError):
+ len(5)
+
+ def test_cm_is_reentrant(self):
+ ignore_exceptions = suppress(Exception)
+ with ignore_exceptions:
+ pass
+ with ignore_exceptions:
+ len(5)
+ with ignore_exceptions:
+ 1/0
+ with ignore_exceptions: # Check nested usage
+ len(5)
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py
index 5676668..ce5d27e 100644
--- a/Lib/test/test_cprofile.py
+++ b/Lib/test/test_cprofile.py
@@ -6,9 +6,11 @@ from test.support import run_unittest, TESTFN, unlink
# rip off all interesting stuff from test_profile
import cProfile
from test.test_profile import ProfileTest, regenerate_expected_output
+from test.profilee import testfunc
class CProfileTest(ProfileTest):
profilerclass = cProfile.Profile
+ profilermodule = cProfile
expected_max_output = "{built-in method max}"
def get_expected_output(self):
@@ -27,6 +29,7 @@ class CProfileTest(ProfileTest):
obj.enable()
obj = _lsprof.Profiler(1)
obj.disable()
+ obj.clear()
finally:
sys.stderr = orig_stderr
finally:
diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py
index cfb7341..624d702 100644
--- a/Lib/test/test_crypt.py
+++ b/Lib/test/test_crypt.py
@@ -1,11 +1,7 @@
from test import support
import unittest
-def setUpModule():
- # this import will raise unittest.SkipTest if _crypt doesn't exist,
- # so it has to be done in setUpModule for test discovery to work
- global crypt
- crypt = support.import_module('crypt')
+crypt = support.import_module('crypt')
class CryptTestCase(unittest.TestCase):
diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index 83f8cb3..7e2485f 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -136,12 +136,12 @@ class Test_Csv(unittest.TestCase):
return 10;
def __getitem__(self, i):
if i > 2:
- raise IOError
- self.assertRaises(IOError, self._write_test, BadList(), '')
+ raise OSError
+ self.assertRaises(OSError, self._write_test, BadList(), '')
class BadItem:
def __str__(self):
- raise IOError
- self.assertRaises(IOError, self._write_test, [BadItem()], '')
+ raise OSError
+ self.assertRaises(OSError, self._write_test, [BadItem()], '')
def test_write_bigfield(self):
# This exercises the buffer realloc functionality
@@ -186,9 +186,9 @@ class Test_Csv(unittest.TestCase):
def test_writerows(self):
class BrokenFile:
def write(self, buf):
- raise IOError
+ raise OSError
writer = csv.writer(BrokenFile())
- self.assertRaises(IOError, writer.writerows, [['a']])
+ self.assertRaises(OSError, writer.writerows, [['a']])
with TemporaryFile("w+", newline='') as fileobj:
writer = csv.writer(fileobj)
@@ -308,6 +308,15 @@ class Test_Csv(unittest.TestCase):
for i, row in enumerate(csv.reader(fileobj)):
self.assertEqual(row, rows[i])
+ def test_roundtrip_escaped_unquoted_newlines(self):
+ with TemporaryFile("w+", newline='') as fileobj:
+ writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")
+ rows = [['a\nb','b'],['c','x\r\nd']]
+ writer.writerows(rows)
+ fileobj.seek(0)
+ for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")):
+ self.assertEqual(row,rows[i])
+
class TestDialectRegistry(unittest.TestCase):
def test_registry_badargs(self):
self.assertRaises(TypeError, csv.list_dialects, None)
@@ -836,10 +845,11 @@ class TestDialectValidity(unittest.TestCase):
d = mydialect()
for field_name in ("delimiter", "escapechar", "quotechar"):
- self.assertRaises(csv.Error, create_invalid, field_name, "")
- self.assertRaises(csv.Error, create_invalid, field_name, "abc")
- self.assertRaises(csv.Error, create_invalid, field_name, b'x')
- self.assertRaises(csv.Error, create_invalid, field_name, 5)
+ with self.subTest(field_name=field_name):
+ self.assertRaises(csv.Error, create_invalid, field_name, "")
+ self.assertRaises(csv.Error, create_invalid, field_name, "abc")
+ self.assertRaises(csv.Error, create_invalid, field_name, b'x')
+ self.assertRaises(csv.Error, create_invalid, field_name, 5)
class TestSniffer(unittest.TestCase):
diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py
index 7310afc..ce8f254 100644
--- a/Lib/test/test_curses.py
+++ b/Lib/test/test_curses.py
@@ -115,8 +115,8 @@ def window_funcs(stdscr):
stdscr.notimeout(1)
win2.overlay(win)
win2.overwrite(win)
- win2.overlay(win, 1, 2, 3, 3, 2, 1)
- win2.overwrite(win, 1, 2, 3, 3, 2, 1)
+ win2.overlay(win, 1, 2, 2, 1, 3, 3)
+ win2.overwrite(win, 1, 2, 2, 1, 3, 3)
stdscr.redrawln(1,2)
stdscr.scrollok(1)
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
index d11d924..623d992 100644
--- a/Lib/test/test_dbm.py
+++ b/Lib/test/test_dbm.py
@@ -61,7 +61,7 @@ class AnyDBMTestCase:
return keys
def test_error(self):
- self.assertTrue(issubclass(self.module.error, IOError))
+ self.assertTrue(issubclass(self.module.error, OSError))
def test_anydbm_not_existing(self):
self.assertRaises(dbm.error, dbm.open, _fname)
diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py
index 4f03ae7..7601e64 100644
--- a/Lib/test/test_dbm_dumb.py
+++ b/Lib/test/test_dbm_dumb.py
@@ -183,6 +183,19 @@ class DumbDBMTestCase(unittest.TestCase):
self.assertEqual(expected, got)
f.close()
+ def test_context_manager(self):
+ with dumbdbm.open(_fname, 'c') as db:
+ db["dumbdbm context manager"] = "context manager"
+
+ with dumbdbm.open(_fname, 'r') as db:
+ self.assertEqual(list(db.keys()), [b"dumbdbm context manager"])
+
+ # This currently just raises AttributeError rather than a specific
+ # exception like the GNU or NDBM based implementations. See
+ # http://bugs.python.org/issue19385 for details.
+ with self.assertRaises(Exception):
+ db.keys()
+
def tearDown(self):
_delete_files()
diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py
index 4fb66c5..a7808f5 100644
--- a/Lib/test/test_dbm_gnu.py
+++ b/Lib/test/test_dbm_gnu.py
@@ -81,6 +81,17 @@ class TestGdbm(unittest.TestCase):
size2 = os.path.getsize(filename)
self.assertTrue(size1 > size2 >= size0)
+ def test_context_manager(self):
+ with gdbm.open(filename, 'c') as db:
+ db["gdbm context manager"] = "context manager"
+
+ with gdbm.open(filename, 'r') as db:
+ self.assertEqual(list(db.keys()), [b"gdbm context manager"])
+
+ with self.assertRaises(gdbm.error) as cm:
+ db.keys()
+ self.assertEqual(str(cm.exception),
+ "GDBM object has already been closed")
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py
index b57e1f0..2291561 100644
--- a/Lib/test/test_dbm_ndbm.py
+++ b/Lib/test/test_dbm_ndbm.py
@@ -37,5 +37,18 @@ class DbmTestCase(unittest.TestCase):
except error:
self.fail()
+ def test_context_manager(self):
+ with dbm.ndbm.open(self.filename, 'c') as db:
+ db["ndbm context manager"] = "context manager"
+
+ with dbm.ndbm.open(self.filename, 'r') as db:
+ self.assertEqual(list(db.keys()), [b"ndbm context manager"])
+
+ with self.assertRaises(dbm.ndbm.error) as cm:
+ db.keys()
+ self.assertEqual(str(cm.exception),
+ "DBM object has already been closed")
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index a8487d2..7bff1d2 100644
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -543,8 +543,8 @@ class TestBasic(unittest.TestCase):
check = self.check_sizeof
check(deque(), basesize + blocksize)
check(deque('a'), basesize + blocksize)
- check(deque('a' * (BLOCKLEN // 2)), basesize + blocksize)
- check(deque('a' * (BLOCKLEN // 2 + 1)), basesize + 2 * blocksize)
+ check(deque('a' * (BLOCKLEN - 1)), basesize + blocksize)
+ check(deque('a' * BLOCKLEN), basesize + 2 * blocksize)
check(deque('a' * (42 * BLOCKLEN)), basesize + 43 * blocksize)
class TestVariousIteratorArgs(unittest.TestCase):
diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py
index 1a891a8..8bb7d6a 100644
--- a/Lib/test/test_descr.py
+++ b/Lib/test/test_descr.py
@@ -1,8 +1,11 @@
import builtins
+import copyreg
import gc
+import itertools
+import math
+import pickle
import sys
import types
-import math
import unittest
import weakref
@@ -1797,6 +1800,7 @@ order (MRO) for bases """
("__trunc__", int, zero, set(), {}),
("__ceil__", math.ceil, zero, set(), {}),
("__dir__", dir, empty_seq, set(), {}),
+ ("__round__", round, zero, set(), {}),
]
class Checker(object):
@@ -2258,7 +2262,9 @@ order (MRO) for bases """
minstance = M("m")
minstance.b = 2
minstance.a = 1
- names = [x for x in dir(minstance) if x not in ["__name__", "__doc__"]]
+ default_attributes = ['__name__', '__doc__', '__package__',
+ '__loader__', '__spec__']
+ names = [x for x in dir(minstance) if x not in default_attributes]
self.assertEqual(names, ['a', 'b'])
class M2(M):
@@ -3151,176 +3157,6 @@ order (MRO) for bases """
self.assertEqual(e.a, 1)
self.assertEqual(can_delete_dict(e), can_delete_dict(ValueError()))
- def test_pickles(self):
- # Testing pickling and copying new-style classes and objects...
- import pickle
-
- def sorteditems(d):
- L = list(d.items())
- L.sort()
- return L
-
- global C
- class C(object):
- def __init__(self, a, b):
- super(C, self).__init__()
- self.a = a
- self.b = b
- def __repr__(self):
- return "C(%r, %r)" % (self.a, self.b)
-
- global C1
- class C1(list):
- def __new__(cls, a, b):
- return super(C1, cls).__new__(cls)
- def __getnewargs__(self):
- return (self.a, self.b)
- def __init__(self, a, b):
- self.a = a
- self.b = b
- def __repr__(self):
- return "C1(%r, %r)<%r>" % (self.a, self.b, list(self))
-
- global C2
- class C2(int):
- def __new__(cls, a, b, val=0):
- return super(C2, cls).__new__(cls, val)
- def __getnewargs__(self):
- return (self.a, self.b, int(self))
- def __init__(self, a, b, val=0):
- self.a = a
- self.b = b
- def __repr__(self):
- return "C2(%r, %r)<%r>" % (self.a, self.b, int(self))
-
- global C3
- class C3(object):
- def __init__(self, foo):
- self.foo = foo
- def __getstate__(self):
- return self.foo
- def __setstate__(self, foo):
- self.foo = foo
-
- global C4classic, C4
- class C4classic: # classic
- pass
- class C4(C4classic, object): # mixed inheritance
- pass
-
- for bin in 0, 1:
- for cls in C, C1, C2:
- s = pickle.dumps(cls, bin)
- cls2 = pickle.loads(s)
- self.assertIs(cls2, cls)
-
- a = C1(1, 2); a.append(42); a.append(24)
- b = C2("hello", "world", 42)
- s = pickle.dumps((a, b), bin)
- x, y = pickle.loads(s)
- self.assertEqual(x.__class__, a.__class__)
- self.assertEqual(sorteditems(x.__dict__), sorteditems(a.__dict__))
- self.assertEqual(y.__class__, b.__class__)
- self.assertEqual(sorteditems(y.__dict__), sorteditems(b.__dict__))
- self.assertEqual(repr(x), repr(a))
- self.assertEqual(repr(y), repr(b))
- # Test for __getstate__ and __setstate__ on new style class
- u = C3(42)
- s = pickle.dumps(u, bin)
- v = pickle.loads(s)
- self.assertEqual(u.__class__, v.__class__)
- self.assertEqual(u.foo, v.foo)
- # Test for picklability of hybrid class
- u = C4()
- u.foo = 42
- s = pickle.dumps(u, bin)
- v = pickle.loads(s)
- self.assertEqual(u.__class__, v.__class__)
- self.assertEqual(u.foo, v.foo)
-
- # Testing copy.deepcopy()
- import copy
- for cls in C, C1, C2:
- cls2 = copy.deepcopy(cls)
- self.assertIs(cls2, cls)
-
- a = C1(1, 2); a.append(42); a.append(24)
- b = C2("hello", "world", 42)
- x, y = copy.deepcopy((a, b))
- self.assertEqual(x.__class__, a.__class__)
- self.assertEqual(sorteditems(x.__dict__), sorteditems(a.__dict__))
- self.assertEqual(y.__class__, b.__class__)
- self.assertEqual(sorteditems(y.__dict__), sorteditems(b.__dict__))
- self.assertEqual(repr(x), repr(a))
- self.assertEqual(repr(y), repr(b))
-
- def test_pickle_slots(self):
- # Testing pickling of classes with __slots__ ...
- import pickle
- # Pickling of classes with __slots__ but without __getstate__ should fail
- # (if using protocol 0 or 1)
- global B, C, D, E
- class B(object):
- pass
- for base in [object, B]:
- class C(base):
- __slots__ = ['a']
- class D(C):
- pass
- try:
- pickle.dumps(C(), 0)
- except TypeError:
- pass
- else:
- self.fail("should fail: pickle C instance - %s" % base)
- try:
- pickle.dumps(C(), 0)
- except TypeError:
- pass
- else:
- self.fail("should fail: pickle D instance - %s" % base)
- # Give C a nice generic __getstate__ and __setstate__
- class C(base):
- __slots__ = ['a']
- def __getstate__(self):
- try:
- d = self.__dict__.copy()
- except AttributeError:
- d = {}
- for cls in self.__class__.__mro__:
- for sn in cls.__dict__.get('__slots__', ()):
- try:
- d[sn] = getattr(self, sn)
- except AttributeError:
- pass
- return d
- def __setstate__(self, d):
- for k, v in list(d.items()):
- setattr(self, k, v)
- class D(C):
- pass
- # Now it should work
- x = C()
- y = pickle.loads(pickle.dumps(x))
- self.assertNotHasAttr(y, 'a')
- x.a = 42
- y = pickle.loads(pickle.dumps(x))
- self.assertEqual(y.a, 42)
- x = D()
- x.a = 42
- x.b = 100
- y = pickle.loads(pickle.dumps(x))
- self.assertEqual(y.a + y.b, 142)
- # A subclass that adds a slot should also work
- class E(C):
- __slots__ = ['b']
- x = E()
- x.a = 42
- x.b = "foo"
- y = pickle.loads(pickle.dumps(x))
- self.assertEqual(y.a, x.a)
- self.assertEqual(y.b, x.b)
-
def test_binary_operator_override(self):
# Testing overrides of binary operations...
class I(int):
@@ -3745,18 +3581,8 @@ order (MRO) for bases """
# bug).
del c
- # If that didn't blow up, it's also interesting to see whether clearing
- # the last container slot works: that will attempt to delete c again,
- # which will cause c to get appended back to the container again
- # "during" the del. (On non-CPython implementations, however, __del__
- # is typically not called again.)
support.gc_collect()
self.assertEqual(len(C.container), 1)
- del C.container[-1]
- if support.check_impl_detail():
- support.gc_collect()
- self.assertEqual(len(C.container), 1)
- self.assertEqual(C.container[-1].attr, 42)
# Make c mortal again, so that the test framework with -l doesn't report
# it as a leak.
@@ -4536,6 +4362,13 @@ order (MRO) for bases """
self.assertRaises(TypeError, type.__dict__['__qualname__'].__set__,
str, 'Oink')
+ global Y
+ class Y:
+ class Inside:
+ pass
+ self.assertEqual(Y.__qualname__, 'Y')
+ self.assertEqual(Y.Inside.__qualname__, 'Y.Inside')
+
def test_qualname_dict(self):
ns = {'__qualname__': 'some.name'}
tp = type('Foo', (), ns)
@@ -4581,6 +4414,14 @@ order (MRO) for bases """
self.assertRaises(TypeError, case, 1, 2, 3)
self.assertRaises(TypeError, case, 1, 2, foo=3)
+ def test_subclassing_does_not_duplicate_dict_descriptors(self):
+ class Base:
+ pass
+ class Sub(Base):
+ pass
+ self.assertIn("__dict__", Base.__dict__)
+ self.assertNotIn("__dict__", Sub.__dict__)
+
class DictProxyTests(unittest.TestCase):
def setUp(self):
@@ -4691,11 +4532,475 @@ class MiscTests(unittest.TestCase):
self.assertEqual(X.mykey2, 'from Base2')
+class PicklingTests(unittest.TestCase):
+
+ def _check_reduce(self, proto, obj, args=(), kwargs={}, state=None,
+ listitems=None, dictitems=None):
+ if proto >= 4:
+ reduce_value = obj.__reduce_ex__(proto)
+ self.assertEqual(reduce_value[:3],
+ (copyreg.__newobj_ex__,
+ (type(obj), args, kwargs),
+ state))
+ if listitems is not None:
+ self.assertListEqual(list(reduce_value[3]), listitems)
+ else:
+ self.assertIsNone(reduce_value[3])
+ if dictitems is not None:
+ self.assertDictEqual(dict(reduce_value[4]), dictitems)
+ else:
+ self.assertIsNone(reduce_value[4])
+ elif proto >= 2:
+ reduce_value = obj.__reduce_ex__(proto)
+ self.assertEqual(reduce_value[:3],
+ (copyreg.__newobj__,
+ (type(obj),) + args,
+ state))
+ if listitems is not None:
+ self.assertListEqual(list(reduce_value[3]), listitems)
+ else:
+ self.assertIsNone(reduce_value[3])
+ if dictitems is not None:
+ self.assertDictEqual(dict(reduce_value[4]), dictitems)
+ else:
+ self.assertIsNone(reduce_value[4])
+ else:
+ base_type = type(obj).__base__
+ reduce_value = (copyreg._reconstructor,
+ (type(obj),
+ base_type,
+ None if base_type is object else base_type(obj)))
+ if state is not None:
+ reduce_value += (state,)
+ self.assertEqual(obj.__reduce_ex__(proto), reduce_value)
+ self.assertEqual(obj.__reduce__(), reduce_value)
+
+ def test_reduce(self):
+ protocols = range(pickle.HIGHEST_PROTOCOL + 1)
+ args = (-101, "spam")
+ kwargs = {'bacon': -201, 'fish': -301}
+ state = {'cheese': -401}
+
+ class C1:
+ def __getnewargs__(self):
+ return args
+ obj = C1()
+ for proto in protocols:
+ self._check_reduce(proto, obj, args)
+
+ for name, value in state.items():
+ setattr(obj, name, value)
+ for proto in protocols:
+ self._check_reduce(proto, obj, args, state=state)
+
+ class C2:
+ def __getnewargs__(self):
+ return "bad args"
+ obj = C2()
+ for proto in protocols:
+ if proto >= 2:
+ with self.assertRaises(TypeError):
+ obj.__reduce_ex__(proto)
+
+ class C3:
+ def __getnewargs_ex__(self):
+ return (args, kwargs)
+ obj = C3()
+ for proto in protocols:
+ if proto >= 4:
+ self._check_reduce(proto, obj, args, kwargs)
+ elif proto >= 2:
+ with self.assertRaises(ValueError):
+ obj.__reduce_ex__(proto)
+
+ class C4:
+ def __getnewargs_ex__(self):
+ return (args, "bad dict")
+ class C5:
+ def __getnewargs_ex__(self):
+ return ("bad tuple", kwargs)
+ class C6:
+ def __getnewargs_ex__(self):
+ return ()
+ class C7:
+ def __getnewargs_ex__(self):
+ return "bad args"
+ for proto in protocols:
+ for cls in C4, C5, C6, C7:
+ obj = cls()
+ if proto >= 2:
+ with self.assertRaises((TypeError, ValueError)):
+ obj.__reduce_ex__(proto)
+
+ class C8:
+ def __getnewargs_ex__(self):
+ return (args, kwargs)
+ obj = C8()
+ for proto in protocols:
+ if 2 <= proto < 4:
+ with self.assertRaises(ValueError):
+ obj.__reduce_ex__(proto)
+ class C9:
+ def __getnewargs_ex__(self):
+ return (args, {})
+ obj = C9()
+ for proto in protocols:
+ self._check_reduce(proto, obj, args)
+
+ class C10:
+ def __getnewargs_ex__(self):
+ raise IndexError
+ obj = C10()
+ for proto in protocols:
+ if proto >= 2:
+ with self.assertRaises(IndexError):
+ obj.__reduce_ex__(proto)
+
+ class C11:
+ def __getstate__(self):
+ return state
+ obj = C11()
+ for proto in protocols:
+ self._check_reduce(proto, obj, state=state)
+
+ class C12:
+ def __getstate__(self):
+ return "not dict"
+ obj = C12()
+ for proto in protocols:
+ self._check_reduce(proto, obj, state="not dict")
+
+ class C13:
+ def __getstate__(self):
+ raise IndexError
+ obj = C13()
+ for proto in protocols:
+ with self.assertRaises(IndexError):
+ obj.__reduce_ex__(proto)
+ if proto < 2:
+ with self.assertRaises(IndexError):
+ obj.__reduce__()
+
+ class C14:
+ __slots__ = tuple(state)
+ def __init__(self):
+ for name, value in state.items():
+ setattr(self, name, value)
+
+ obj = C14()
+ for proto in protocols:
+ if proto >= 2:
+ self._check_reduce(proto, obj, state=(None, state))
+ else:
+ with self.assertRaises(TypeError):
+ obj.__reduce_ex__(proto)
+ with self.assertRaises(TypeError):
+ obj.__reduce__()
+
+ class C15(dict):
+ pass
+ obj = C15({"quebec": -601})
+ for proto in protocols:
+ self._check_reduce(proto, obj, dictitems=dict(obj))
+
+ class C16(list):
+ pass
+ obj = C16(["yukon"])
+ for proto in protocols:
+ self._check_reduce(proto, obj, listitems=list(obj))
+
+ def test_special_method_lookup(self):
+ protocols = range(pickle.HIGHEST_PROTOCOL + 1)
+ class Picky:
+ def __getstate__(self):
+ return {}
+
+ def __getattr__(self, attr):
+ if attr in ("__getnewargs__", "__getnewargs_ex__"):
+ raise AssertionError(attr)
+ return None
+ for protocol in protocols:
+ state = {} if protocol >= 2 else None
+ self._check_reduce(protocol, Picky(), state=state)
+
+ def _assert_is_copy(self, obj, objcopy, msg=None):
+ """Utility method to verify if two objects are copies of each others.
+ """
+ if msg is None:
+ msg = "{!r} is not a copy of {!r}".format(obj, objcopy)
+ if type(obj).__repr__ is object.__repr__:
+ # We have this limitation for now because we use the object's repr
+ # to help us verify that the two objects are copies. This allows
+ # us to delegate the non-generic verification logic to the objects
+ # themselves.
+ raise ValueError("object passed to _assert_is_copy must " +
+ "override the __repr__ method.")
+ self.assertIsNot(obj, objcopy, msg=msg)
+ self.assertIs(type(obj), type(objcopy), msg=msg)
+ if hasattr(obj, '__dict__'):
+ self.assertDictEqual(obj.__dict__, objcopy.__dict__, msg=msg)
+ self.assertIsNot(obj.__dict__, objcopy.__dict__, msg=msg)
+ if hasattr(obj, '__slots__'):
+ self.assertListEqual(obj.__slots__, objcopy.__slots__, msg=msg)
+ for slot in obj.__slots__:
+ self.assertEqual(
+ hasattr(obj, slot), hasattr(objcopy, slot), msg=msg)
+ self.assertEqual(getattr(obj, slot, None),
+ getattr(objcopy, slot, None), msg=msg)
+ self.assertEqual(repr(obj), repr(objcopy), msg=msg)
+
+ @staticmethod
+ def _generate_pickle_copiers():
+ """Utility method to generate the many possible pickle configurations.
+ """
+ class PickleCopier:
+ "This class copies object using pickle."
+ def __init__(self, proto, dumps, loads):
+ self.proto = proto
+ self.dumps = dumps
+ self.loads = loads
+ def copy(self, obj):
+ return self.loads(self.dumps(obj, self.proto))
+ def __repr__(self):
+ # We try to be as descriptive as possible here since this is
+ # the string which we will allow us to tell the pickle
+ # configuration we are using during debugging.
+ return ("PickleCopier(proto={}, dumps={}.{}, loads={}.{})"
+ .format(self.proto,
+ self.dumps.__module__, self.dumps.__qualname__,
+ self.loads.__module__, self.loads.__qualname__))
+ return (PickleCopier(*args) for args in
+ itertools.product(range(pickle.HIGHEST_PROTOCOL + 1),
+ {pickle.dumps, pickle._dumps},
+ {pickle.loads, pickle._loads}))
+
+ def test_pickle_slots(self):
+ # Tests pickling of classes with __slots__.
+
+ # Pickling of classes with __slots__ but without __getstate__ should
+ # fail (if using protocol 0 or 1)
+ global C
+ class C:
+ __slots__ = ['a']
+ with self.assertRaises(TypeError):
+ pickle.dumps(C(), 0)
+
+ global D
+ class D(C):
+ pass
+ with self.assertRaises(TypeError):
+ pickle.dumps(D(), 0)
+
+ class C:
+ "A class with __getstate__ and __setstate__ implemented."
+ __slots__ = ['a']
+ def __getstate__(self):
+ state = getattr(self, '__dict__', {}).copy()
+ for cls in type(self).__mro__:
+ for slot in cls.__dict__.get('__slots__', ()):
+ try:
+ state[slot] = getattr(self, slot)
+ except AttributeError:
+ pass
+ return state
+ def __setstate__(self, state):
+ for k, v in state.items():
+ setattr(self, k, v)
+ def __repr__(self):
+ return "%s()<%r>" % (type(self).__name__, self.__getstate__())
+
+ class D(C):
+ "A subclass of a class with slots."
+ pass
+
+ global E
+ class E(C):
+ "A subclass with an extra slot."
+ __slots__ = ['b']
+
+ # Now it should work
+ for pickle_copier in self._generate_pickle_copiers():
+ with self.subTest(pickle_copier=pickle_copier):
+ x = C()
+ y = pickle_copier.copy(x)
+ self._assert_is_copy(x, y)
+
+ x.a = 42
+ y = pickle_copier.copy(x)
+ self._assert_is_copy(x, y)
+
+ x = D()
+ x.a = 42
+ x.b = 100
+ y = pickle_copier.copy(x)
+ self._assert_is_copy(x, y)
+
+ x = E()
+ x.a = 42
+ x.b = "foo"
+ y = pickle_copier.copy(x)
+ self._assert_is_copy(x, y)
+
+ def test_reduce_copying(self):
+ # Tests pickling and copying new-style classes and objects.
+ global C1
+ class C1:
+ "The state of this class is copyable via its instance dict."
+ ARGS = (1, 2)
+ NEED_DICT_COPYING = True
+ def __init__(self, a, b):
+ super().__init__()
+ self.a = a
+ self.b = b
+ def __repr__(self):
+ return "C1(%r, %r)" % (self.a, self.b)
+
+ global C2
+ class C2(list):
+ "A list subclass copyable via __getnewargs__."
+ ARGS = (1, 2)
+ NEED_DICT_COPYING = False
+ def __new__(cls, a, b):
+ self = super().__new__(cls)
+ self.a = a
+ self.b = b
+ return self
+ def __init__(self, *args):
+ super().__init__()
+ # This helps testing that __init__ is not called during the
+ # unpickling process, which would cause extra appends.
+ self.append("cheese")
+ @classmethod
+ def __getnewargs__(cls):
+ return cls.ARGS
+ def __repr__(self):
+ return "C2(%r, %r)<%r>" % (self.a, self.b, list(self))
+
+ global C3
+ class C3(list):
+ "A list subclass copyable via __getstate__."
+ ARGS = (1, 2)
+ NEED_DICT_COPYING = False
+ def __init__(self, a, b):
+ self.a = a
+ self.b = b
+ # This helps testing that __init__ is not called during the
+ # unpickling process, which would cause extra appends.
+ self.append("cheese")
+ @classmethod
+ def __getstate__(cls):
+ return cls.ARGS
+ def __setstate__(self, state):
+ a, b = state
+ self.a = a
+ self.b = b
+ def __repr__(self):
+ return "C3(%r, %r)<%r>" % (self.a, self.b, list(self))
+
+ global C4
+ class C4(int):
+ "An int subclass copyable via __getnewargs__."
+ ARGS = ("hello", "world", 1)
+ NEED_DICT_COPYING = False
+ def __new__(cls, a, b, value):
+ self = super().__new__(cls, value)
+ self.a = a
+ self.b = b
+ return self
+ @classmethod
+ def __getnewargs__(cls):
+ return cls.ARGS
+ def __repr__(self):
+ return "C4(%r, %r)<%r>" % (self.a, self.b, int(self))
+
+ global C5
+ class C5(int):
+ "An int subclass copyable via __getnewargs_ex__."
+ ARGS = (1, 2)
+ KWARGS = {'value': 3}
+ NEED_DICT_COPYING = False
+ def __new__(cls, a, b, *, value=0):
+ self = super().__new__(cls, value)
+ self.a = a
+ self.b = b
+ return self
+ @classmethod
+ def __getnewargs_ex__(cls):
+ return (cls.ARGS, cls.KWARGS)
+ def __repr__(self):
+ return "C5(%r, %r)<%r>" % (self.a, self.b, int(self))
+
+ test_classes = (C1, C2, C3, C4, C5)
+ # Testing copying through pickle
+ pickle_copiers = self._generate_pickle_copiers()
+ for cls, pickle_copier in itertools.product(test_classes, pickle_copiers):
+ with self.subTest(cls=cls, pickle_copier=pickle_copier):
+ kwargs = getattr(cls, 'KWARGS', {})
+ obj = cls(*cls.ARGS, **kwargs)
+ proto = pickle_copier.proto
+ if 2 <= proto < 4 and hasattr(cls, '__getnewargs_ex__'):
+ with self.assertRaises(ValueError):
+ pickle_copier.dumps(obj, proto)
+ continue
+ objcopy = pickle_copier.copy(obj)
+ self._assert_is_copy(obj, objcopy)
+ # For test classes that supports this, make sure we didn't go
+ # around the reduce protocol by simply copying the attribute
+ # dictionary. We clear attributes using the previous copy to
+ # not mutate the original argument.
+ if proto >= 2 and not cls.NEED_DICT_COPYING:
+ objcopy.__dict__.clear()
+ objcopy2 = pickle_copier.copy(objcopy)
+ self._assert_is_copy(obj, objcopy2)
+
+ # Testing copying through copy.deepcopy()
+ for cls in test_classes:
+ with self.subTest(cls=cls):
+ kwargs = getattr(cls, 'KWARGS', {})
+ obj = cls(*cls.ARGS, **kwargs)
+ # XXX: We need to modify the copy module to support PEP 3154's
+ # reduce protocol 4.
+ if hasattr(cls, '__getnewargs_ex__'):
+ continue
+ objcopy = deepcopy(obj)
+ self._assert_is_copy(obj, objcopy)
+ # For test classes that supports this, make sure we didn't go
+ # around the reduce protocol by simply copying the attribute
+ # dictionary. We clear attributes using the previous copy to
+ # not mutate the original argument.
+ if not cls.NEED_DICT_COPYING:
+ objcopy.__dict__.clear()
+ objcopy2 = deepcopy(objcopy)
+ self._assert_is_copy(obj, objcopy2)
+
+
+class SharedKeyTests(unittest.TestCase):
+
+ @support.cpython_only
+ def test_subclasses(self):
+ # Verify that subclasses can share keys (per PEP 412)
+ class A:
+ pass
+ class B(A):
+ pass
+
+ a, b = A(), B()
+ self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b)))
+ self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({}))
+ a.x, a.y, a.z, a.w = range(4)
+ self.assertNotEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(b)))
+ a2 = A()
+ self.assertEqual(sys.getsizeof(vars(a)), sys.getsizeof(vars(a2)))
+ self.assertLess(sys.getsizeof(vars(a)), sys.getsizeof({}))
+ b.u, b.v, b.w, b.t = range(4)
+ self.assertLess(sys.getsizeof(vars(b)), sys.getsizeof({}))
+
+
def test_main():
# Run all local test cases, with PTypesLongInitTest first.
support.run_unittest(PTypesLongInitTest, OperatorsTest,
ClassPropertiesAndMethods, DictProxyTests,
- MiscTests)
+ MiscTests, PicklingTests, SharedKeyTests)
if __name__ == "__main__":
test_main()
diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py
index 9129ac0..e634d3a 100644
--- a/Lib/test/test_devpoll.py
+++ b/Lib/test/test_devpoll.py
@@ -2,13 +2,17 @@
# Initial tests are copied as is from "test_poll.py"
-import os, select, random, unittest, sys
+import os
+import random
+import select
+import sys
+import unittest
from test.support import TESTFN, run_unittest, cpython_only
try:
select.devpoll
except AttributeError:
- raise unittest.SkipTest("select.devpoll not defined -- skipping test_devpoll")
+ raise unittest.SkipTest("select.devpoll not defined")
def find_ready_matching(ready, flag):
@@ -87,6 +91,35 @@ class DevPollTests(unittest.TestCase):
self.assertRaises(OverflowError, pollster.poll, 1 << 63)
self.assertRaises(OverflowError, pollster.poll, 1 << 64)
+ def test_close(self):
+ open_file = open(__file__, "rb")
+ self.addCleanup(open_file.close)
+ fd = open_file.fileno()
+ devpoll = select.devpoll()
+
+ # test fileno() method and closed attribute
+ self.assertIsInstance(devpoll.fileno(), int)
+ self.assertFalse(devpoll.closed)
+
+ # test close()
+ devpoll.close()
+ self.assertTrue(devpoll.closed)
+ self.assertRaises(ValueError, devpoll.fileno)
+
+ # close() can be called more than once
+ devpoll.close()
+
+ # operations must fail with ValueError("I/O operation on closed ...")
+ self.assertRaises(ValueError, devpoll.modify, fd, select.POLLIN)
+ self.assertRaises(ValueError, devpoll.poll)
+ self.assertRaises(ValueError, devpoll.register, fd, fd, select.POLLIN)
+ self.assertRaises(ValueError, devpoll.unregister, fd)
+
+ def test_fd_non_inheritable(self):
+ devpoll = select.devpoll()
+ self.addCleanup(devpoll.close)
+ self.assertEqual(os.get_inheritable(devpoll.fileno()), False)
+
def test_events_mask_overflow(self):
pollster = select.devpoll()
w, r = os.pipe()
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index b86cc86..d1229fb 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -1,12 +1,30 @@
# Minimal tests for dis module
from test.support import run_unittest, captured_stdout
+from test.bytecode_helper import BytecodeTestCase
import difflib
import unittest
import sys
import dis
import io
import re
+import types
+import contextlib
+
+def get_tb():
+ def _error():
+ try:
+ 1 / 0
+ except Exception as e:
+ tb = e.__traceback__
+ return tb
+
+ tb = _error()
+ while tb.tb_next:
+ tb = tb.tb_next
+ return tb
+
+TRACEBACK_CODE = get_tb().tb_frame.f_code
class _C:
def __init__(self, x):
@@ -23,12 +41,12 @@ dis_c_instance_method = """\
""" % (_C.__init__.__code__.co_firstlineno + 1,)
dis_c_instance_method_bytes = """\
- 0 LOAD_FAST 1 (1)
- 3 LOAD_CONST 1 (1)
- 6 COMPARE_OP 2 (==)
- 9 LOAD_FAST 0 (0)
- 12 STORE_ATTR 0 (0)
- 15 LOAD_CONST 0 (0)
+ 0 LOAD_FAST 1 (1)
+ 3 LOAD_CONST 1 (1)
+ 6 COMPARE_OP 2 (==)
+ 9 LOAD_FAST 0 (0)
+ 12 STORE_ATTR 0 (0)
+ 15 LOAD_CONST 0 (0)
18 RETURN_VALUE
"""
@@ -49,11 +67,11 @@ dis_f = """\
dis_f_co_code = """\
- 0 LOAD_GLOBAL 0 (0)
- 3 LOAD_FAST 0 (0)
- 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair)
+ 0 LOAD_GLOBAL 0 (0)
+ 3 LOAD_FAST 0 (0)
+ 6 CALL_FUNCTION 1 (1 positional, 0 keyword pair)
9 POP_TOP
- 10 LOAD_CONST 1 (1)
+ 10 LOAD_CONST 1 (1)
13 RETURN_VALUE
"""
@@ -89,7 +107,7 @@ def bug1333982(x=[]):
pass
dis_bug1333982 = """\
- %-4d 0 LOAD_CONST 1 (0)
+%3d 0 LOAD_CONST 1 (0)
3 POP_JUMP_IF_TRUE 35
6 LOAD_GLOBAL 0 (AssertionError)
9 LOAD_CONST 2 (<code object <listcomp> at 0x..., file "%s", line %d>)
@@ -99,12 +117,12 @@ dis_bug1333982 = """\
21 GET_ITER
22 CALL_FUNCTION 1 (1 positional, 0 keyword pair)
- %-4d 25 LOAD_CONST 4 (1)
+%3d 25 LOAD_CONST 4 (1)
28 BINARY_ADD
29 CALL_FUNCTION 1 (1 positional, 0 keyword pair)
32 RAISE_VARARGS 1
- %-4d >> 35 LOAD_CONST 0 (None)
+%3d >> 35 LOAD_CONST 0 (None)
38 RETURN_VALUE
""" % (bug1333982.__code__.co_firstlineno + 1,
__file__,
@@ -171,38 +189,69 @@ dis_compound_stmt_str = """\
25 RETURN_VALUE
"""
+dis_traceback = """\
+ %-4d 0 SETUP_EXCEPT 12 (to 15)
+
+ %-4d 3 LOAD_CONST 1 (1)
+ 6 LOAD_CONST 2 (0)
+ --> 9 BINARY_TRUE_DIVIDE
+ 10 POP_TOP
+ 11 POP_BLOCK
+ 12 JUMP_FORWARD 46 (to 61)
+
+ %-4d >> 15 DUP_TOP
+ 16 LOAD_GLOBAL 0 (Exception)
+ 19 COMPARE_OP 10 (exception match)
+ 22 POP_JUMP_IF_FALSE 60
+ 25 POP_TOP
+ 26 STORE_FAST 0 (e)
+ 29 POP_TOP
+ 30 SETUP_FINALLY 14 (to 47)
+
+ %-4d 33 LOAD_FAST 0 (e)
+ 36 LOAD_ATTR 1 (__traceback__)
+ 39 STORE_FAST 1 (tb)
+ 42 POP_BLOCK
+ 43 POP_EXCEPT
+ 44 LOAD_CONST 0 (None)
+ >> 47 LOAD_CONST 0 (None)
+ 50 STORE_FAST 0 (e)
+ 53 DELETE_FAST 0 (e)
+ 56 END_FINALLY
+ 57 JUMP_FORWARD 1 (to 61)
+ >> 60 END_FINALLY
+
+ %-4d >> 61 LOAD_FAST 1 (tb)
+ 64 RETURN_VALUE
+""" % (TRACEBACK_CODE.co_firstlineno + 1,
+ TRACEBACK_CODE.co_firstlineno + 2,
+ TRACEBACK_CODE.co_firstlineno + 3,
+ TRACEBACK_CODE.co_firstlineno + 4,
+ TRACEBACK_CODE.co_firstlineno + 5)
+
class DisTests(unittest.TestCase):
def get_disassembly(self, func, lasti=-1, wrapper=True):
- s = io.StringIO()
- save_stdout = sys.stdout
- sys.stdout = s
- try:
+ # We want to test the default printing behaviour, not the file arg
+ output = io.StringIO()
+ with contextlib.redirect_stdout(output):
if wrapper:
dis.dis(func)
else:
dis.disassemble(func, lasti)
- finally:
- sys.stdout = save_stdout
- # Trim trailing blanks (if any).
- return [line.rstrip() for line in s.getvalue().splitlines()]
+ return output.getvalue()
def get_disassemble_as_string(self, func, lasti=-1):
- return '\n'.join(self.get_disassembly(func, lasti, False))
+ return self.get_disassembly(func, lasti, False)
+
+ def strip_addresses(self, text):
+ return re.sub(r'\b0x[0-9A-Fa-f]+\b', '0x...', text)
def do_disassembly_test(self, func, expected):
- lines = self.get_disassembly(func)
- expected = expected.splitlines()
- if expected == lines:
- return
- else:
- lines = [re.sub('0x[0-9A-Fa-f]+', '0x...', l) for l in lines]
- if expected == lines:
- return
- self.fail(
- "events did not match expectation:\n" +
- "\n".join(difflib.ndiff(expected,
- lines)))
+ got = self.get_disassembly(func)
+ if got != expected:
+ got = self.strip_addresses(got)
+ self.assertEqual(got, expected)
def test_opmap(self):
self.assertEqual(dis.opmap["NOP"], 9)
@@ -290,35 +339,35 @@ class DisTests(unittest.TestCase):
def test_dis_object(self):
self.assertRaises(TypeError, dis.dis, object())
+class DisWithFileTests(DisTests):
+
+ # Run the tests again, using the file arg instead of print
+ def get_disassembly(self, func, lasti=-1, wrapper=True):
+ output = io.StringIO()
+ if wrapper:
+ dis.dis(func, file=output)
+ else:
+ dis.disassemble(func, lasti, file=output)
+ return output.getvalue()
+
+
+
code_info_code_info = """\
Name: code_info
Filename: (.*)
Argument count: 1
Kw-only arguments: 0
Number of locals: 1
-Stack size: 4
+Stack size: 3
Flags: OPTIMIZED, NEWLOCALS, NOFREE
Constants:
0: %r
- 1: '__func__'
- 2: '__code__'
- 3: '<code_info>'
- 4: 'co_code'
- 5: "don't know how to disassemble %%s objects"
-%sNames:
- 0: hasattr
- 1: __func__
- 2: __code__
- 3: isinstance
- 4: str
- 5: _try_compile
- 6: _format_code_info
- 7: TypeError
- 8: type
- 9: __name__
+Names:
+ 0: _format_code_info
+ 1: _get_code_object
Variable names:
- 0: x""" % (('Formatted details of methods, functions, or code.', ' 6: None\n')
- if sys.flags.optimize < 2 else (None, ''))
+ 0: x""" % (('Formatted details of methods, functions, or code.',)
+ if sys.flags.optimize < 2 else (None,))
@staticmethod
def tricky(x, y, z=True, *args, c, d, e=[], **kwds):
@@ -382,7 +431,7 @@ Free variables:
code_info_expr_str = """\
Name: <module>
-Filename: <code_info>
+Filename: <disassembly>
Argument count: 0
Kw-only arguments: 0
Number of locals: 0
@@ -395,7 +444,7 @@ Names:
code_info_simple_stmt_str = """\
Name: <module>
-Filename: <code_info>
+Filename: <disassembly>
Argument count: 0
Kw-only arguments: 0
Number of locals: 0
@@ -409,7 +458,7 @@ Names:
code_info_compound_stmt_str = """\
Name: <module>
-Filename: <code_info>
+Filename: <disassembly>
Argument count: 0
Kw-only arguments: 0
Number of locals: 0
@@ -443,6 +492,9 @@ class CodeInfoTests(unittest.TestCase):
with captured_stdout() as output:
dis.show_code(x)
self.assertRegex(output.getvalue(), expected+"\n")
+ output = io.StringIO()
+ dis.show_code(x, file=output)
+ self.assertRegex(output.getvalue(), expected)
def test_code_info_object(self):
self.assertRaises(TypeError, dis.code_info, object())
@@ -451,5 +503,330 @@ class CodeInfoTests(unittest.TestCase):
self.assertEqual(dis.pretty_flags(0), '0x0')
+# Fodder for instruction introspection tests
+# Editing any of these may require recalculating the expected output
+def outer(a=1, b=2):
+ def f(c=3, d=4):
+ def inner(e=5, f=6):
+ print(a, b, c, d, e, f)
+ print(a, b, c, d)
+ return inner
+ print(a, b, '', 1, [], {}, "Hello world!")
+ return f
+
+def jumpy():
+ # This won't actually run (but that's OK, we only disassemble it)
+ for i in range(10):
+ print(i)
+ if i < 4:
+ continue
+ if i > 6:
+ break
+ else:
+ print("I can haz else clause?")
+ while i:
+ print(i)
+ i -= 1
+ if i > 6:
+ continue
+ if i < 4:
+ break
+ else:
+ print("Who let lolcatz into this test suite?")
+ try:
+ 1 / 0
+ except ZeroDivisionError:
+ print("Here we go, here we go, here we go...")
+ else:
+ with i as dodgy:
+ print("Never reach this")
+ finally:
+ print("OK, now we're done")
+
+# End fodder for opinfo generation tests
+expected_outer_line = 1
+_line_offset = outer.__code__.co_firstlineno - 1
+code_object_f = outer.__code__.co_consts[3]
+expected_f_line = code_object_f.co_firstlineno - _line_offset
+code_object_inner = code_object_f.co_consts[3]
+expected_inner_line = code_object_inner.co_firstlineno - _line_offset
+expected_jumpy_line = 1
+
+# The following lines are useful to regenerate the expected results after
+# either the fodder is modified or the bytecode generation changes
+# After regeneration, update the references to code_object_f and
+# code_object_inner before rerunning the tests
+
+#_instructions = dis.get_instructions(outer, first_line=expected_outer_line)
+#print('expected_opinfo_outer = [\n ',
+ #',\n '.join(map(str, _instructions)), ',\n]', sep='')
+#_instructions = dis.get_instructions(outer(), first_line=expected_outer_line)
+#print('expected_opinfo_f = [\n ',
+ #',\n '.join(map(str, _instructions)), ',\n]', sep='')
+#_instructions = dis.get_instructions(outer()(), first_line=expected_outer_line)
+#print('expected_opinfo_inner = [\n ',
+ #',\n '.join(map(str, _instructions)), ',\n]', sep='')
+#_instructions = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
+#print('expected_opinfo_jumpy = [\n ',
+ #',\n '.join(map(str, _instructions)), ',\n]', sep='')
+
+
+Instruction = dis.Instruction
+expected_opinfo_outer = [
+ Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=3, argrepr='3', offset=0, starts_line=2, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=3, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False),
+ Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=12, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_f, argrepr=repr(code_object_f), offset=15, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f', argrepr="'outer.<locals>.f'", offset=18, starts_line=None, is_jump_target=False),
+ Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=21, starts_line=None, is_jump_target=False),
+ Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=27, starts_line=7, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=30, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=33, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval='', argrepr="''", offset=36, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval=1, argrepr='1', offset=39, starts_line=None, is_jump_target=False),
+ Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=42, starts_line=None, is_jump_target=False),
+ Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=45, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval='Hello world!', argrepr="'Hello world!'", offset=48, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=7, argval=7, argrepr='7 positional, 0 keyword pair', offset=51, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=55, starts_line=8, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False),
+]
+
+expected_opinfo_f = [
+ Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=5, argrepr='5', offset=0, starts_line=3, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=6, argrepr='6', offset=3, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=2, argval='a', argrepr='a', offset=6, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=3, argval='b', argrepr='b', offset=9, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=0, argval='c', argrepr='c', offset=12, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CLOSURE', opcode=135, arg=1, argval='d', argrepr='d', offset=15, starts_line=None, is_jump_target=False),
+ Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=18, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=code_object_inner, argrepr=repr(code_object_inner), offset=21, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='outer.<locals>.f.<locals>.inner', argrepr="'outer.<locals>.f.<locals>.inner'", offset=24, starts_line=None, is_jump_target=False),
+ Instruction(opname='MAKE_CLOSURE', opcode=134, arg=2, argval=2, argrepr='', offset=27, starts_line=None, is_jump_target=False),
+ Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=30, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=33, starts_line=5, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='a', argrepr='a', offset=36, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='b', argrepr='b', offset=39, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='c', argrepr='c', offset=42, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='d', argrepr='d', offset=45, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=4, argval=4, argrepr='4 positional, 0 keyword pair', offset=48, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=51, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=52, starts_line=6, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=55, starts_line=None, is_jump_target=False),
+]
+
+expected_opinfo_inner = [
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='print', argrepr='print', offset=0, starts_line=4, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=0, argval='a', argrepr='a', offset=3, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=1, argval='b', argrepr='b', offset=6, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=2, argval='c', argrepr='c', offset=9, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_DEREF', opcode=136, arg=3, argval='d', argrepr='d', offset=12, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=15, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=18, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=6, argval=6, argrepr='6 positional, 0 keyword pair', offset=21, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=25, starts_line=None, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False),
+]
+
+expected_opinfo_jumpy = [
+ Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=77, argrepr='to 77', offset=0, starts_line=3, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=0, argval='range', argrepr='range', offset=3, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=6, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=9, starts_line=None, is_jump_target=False),
+ Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=12, starts_line=None, is_jump_target=False),
+ Instruction(opname='FOR_ITER', opcode=93, arg=50, argval=66, argrepr='to 66', offset=13, starts_line=None, is_jump_target=True),
+ Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=16, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=19, starts_line=4, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=22, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=25, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=28, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=29, starts_line=5, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=32, starts_line=None, is_jump_target=False),
+ Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=35, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=47, argval=47, argrepr='', offset=38, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=41, starts_line=6, is_jump_target=False),
+ Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=47, argrepr='to 47', offset=44, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=47, starts_line=7, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=50, starts_line=None, is_jump_target=False),
+ Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=53, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=13, argval=13, argrepr='', offset=56, starts_line=None, is_jump_target=False),
+ Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=59, starts_line=8, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=60, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=13, argval=13, argrepr='', offset=63, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=66, starts_line=None, is_jump_target=True),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=67, starts_line=10, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=70, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=73, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=76, starts_line=None, is_jump_target=False),
+ Instruction(opname='SETUP_LOOP', opcode=120, arg=74, argval=154, argrepr='to 154', offset=77, starts_line=11, is_jump_target=True),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=80, starts_line=None, is_jump_target=True),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=143, argval=143, argrepr='', offset=83, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=86, starts_line=12, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=89, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=92, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=95, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=96, starts_line=13, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=99, starts_line=None, is_jump_target=False),
+ Instruction(opname='INPLACE_SUBTRACT', opcode=56, arg=None, argval=None, argrepr='', offset=102, starts_line=None, is_jump_target=False),
+ Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=103, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=106, starts_line=14, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=109, starts_line=None, is_jump_target=False),
+ Instruction(opname='COMPARE_OP', opcode=107, arg=4, argval='>', argrepr='>', offset=112, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=124, argval=124, argrepr='', offset=115, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=118, starts_line=15, is_jump_target=False),
+ Instruction(opname='JUMP_FORWARD', opcode=110, arg=0, argval=124, argrepr='to 124', offset=121, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=16, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=127, starts_line=None, is_jump_target=False),
+ Instruction(opname='COMPARE_OP', opcode=107, arg=0, argval='<', argrepr='<', offset=130, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=80, argval=80, argrepr='', offset=133, starts_line=None, is_jump_target=False),
+ Instruction(opname='BREAK_LOOP', opcode=80, arg=None, argval=None, argrepr='', offset=136, starts_line=17, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=137, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_ABSOLUTE', opcode=113, arg=80, argval=80, argrepr='', offset=140, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=143, starts_line=None, is_jump_target=True),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=144, starts_line=19, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=147, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=150, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=153, starts_line=None, is_jump_target=False),
+ Instruction(opname='SETUP_FINALLY', opcode=122, arg=72, argval=229, argrepr='to 229', offset=154, starts_line=20, is_jump_target=True),
+ Instruction(opname='SETUP_EXCEPT', opcode=121, arg=12, argval=172, argrepr='to 172', offset=157, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=160, starts_line=21, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=163, starts_line=None, is_jump_target=False),
+ Instruction(opname='BINARY_TRUE_DIVIDE', opcode=27, arg=None, argval=None, argrepr='', offset=166, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=167, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=168, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_FORWARD', opcode=110, arg=28, argval=200, argrepr='to 200', offset=169, starts_line=None, is_jump_target=False),
+ Instruction(opname='DUP_TOP', opcode=4, arg=None, argval=None, argrepr='', offset=172, starts_line=22, is_jump_target=True),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=2, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=173, starts_line=None, is_jump_target=False),
+ Instruction(opname='COMPARE_OP', opcode=107, arg=10, argval='exception match', argrepr='exception match', offset=176, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=199, argval=199, argrepr='', offset=179, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=182, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=183, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=184, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=185, starts_line=23, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=188, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=191, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=194, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=195, starts_line=None, is_jump_target=False),
+ Instruction(opname='JUMP_FORWARD', opcode=110, arg=26, argval=225, argrepr='to 225', offset=196, starts_line=None, is_jump_target=False),
+ Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=199, starts_line=None, is_jump_target=True),
+ Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=200, starts_line=25, is_jump_target=True),
+ Instruction(opname='SETUP_WITH', opcode=143, arg=17, argval=223, argrepr='to 223', offset=203, starts_line=None, is_jump_target=False),
+ Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=206, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=209, starts_line=26, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Never reach this', argrepr="'Never reach this'", offset=212, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=215, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=218, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=219, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=220, starts_line=None, is_jump_target=False),
+ Instruction(opname='WITH_CLEANUP', opcode=81, arg=None, argval=None, argrepr='', offset=223, starts_line=None, is_jump_target=True),
+ Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=224, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_BLOCK', opcode=87, arg=None, argval=None, argrepr='', offset=225, starts_line=None, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=226, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print', offset=229, starts_line=28, is_jump_target=True),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=232, starts_line=None, is_jump_target=False),
+ Instruction(opname='CALL_FUNCTION', opcode=131, arg=1, argval=1, argrepr='1 positional, 0 keyword pair', offset=235, starts_line=None, is_jump_target=False),
+ Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=238, starts_line=None, is_jump_target=False),
+ Instruction(opname='END_FINALLY', opcode=88, arg=None, argval=None, argrepr='', offset=239, starts_line=None, is_jump_target=False),
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=240, starts_line=None, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=243, starts_line=None, is_jump_target=False),
+]
+
+# One last piece of inspect fodder to check the default line number handling
+def simple(): pass
+expected_opinfo_simple = [
+ Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False),
+ Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=3, starts_line=None, is_jump_target=False)
+]
+
+
+class InstructionTests(BytecodeTestCase):
+
+ def test_default_first_line(self):
+ actual = dis.get_instructions(simple)
+ self.assertEqual(list(actual), expected_opinfo_simple)
+
+ def test_first_line_set_to_None(self):
+ actual = dis.get_instructions(simple, first_line=None)
+ self.assertEqual(list(actual), expected_opinfo_simple)
+
+ def test_outer(self):
+ actual = dis.get_instructions(outer, first_line=expected_outer_line)
+ self.assertEqual(list(actual), expected_opinfo_outer)
+
+ def test_nested(self):
+ with captured_stdout():
+ f = outer()
+ actual = dis.get_instructions(f, first_line=expected_f_line)
+ self.assertEqual(list(actual), expected_opinfo_f)
+
+ def test_doubly_nested(self):
+ with captured_stdout():
+ inner = outer()()
+ actual = dis.get_instructions(inner, first_line=expected_inner_line)
+ self.assertEqual(list(actual), expected_opinfo_inner)
+
+ def test_jumpy(self):
+ actual = dis.get_instructions(jumpy, first_line=expected_jumpy_line)
+ self.assertEqual(list(actual), expected_opinfo_jumpy)
+
+# get_instructions has its own tests above, so can rely on it to validate
+# the object oriented API
+class BytecodeTests(unittest.TestCase):
+ def test_instantiation(self):
+ # Test with function, method, code string and code object
+ for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:
+ with self.subTest(obj=obj):
+ b = dis.Bytecode(obj)
+ self.assertIsInstance(b.codeobj, types.CodeType)
+
+ self.assertRaises(TypeError, dis.Bytecode, object())
+
+ def test_iteration(self):
+ for obj in [_f, _C(1).__init__, "a=1", _f.__code__]:
+ with self.subTest(obj=obj):
+ via_object = list(dis.Bytecode(obj))
+ via_generator = list(dis.get_instructions(obj))
+ self.assertEqual(via_object, via_generator)
+
+ def test_explicit_first_line(self):
+ actual = dis.Bytecode(outer, first_line=expected_outer_line)
+ self.assertEqual(list(actual), expected_opinfo_outer)
+
+ def test_source_line_in_disassembly(self):
+ # Use the line in the source code
+ actual = dis.Bytecode(simple).dis()[:3]
+ expected = "{:>3}".format(simple.__code__.co_firstlineno)
+ self.assertEqual(actual, expected)
+ # Use an explicit first line number
+ actual = dis.Bytecode(simple, first_line=350).dis()[:3]
+ self.assertEqual(actual, "350")
+
+ def test_info(self):
+ self.maxDiff = 1000
+ for x, expected in CodeInfoTests.test_pairs:
+ b = dis.Bytecode(x)
+ self.assertRegex(b.info(), expected)
+
+ def test_disassembled(self):
+ actual = dis.Bytecode(_f).dis()
+ self.assertEqual(actual, dis_f)
+
+ def test_from_traceback(self):
+ tb = get_tb()
+ b = dis.Bytecode.from_traceback(tb)
+ while tb.tb_next: tb = tb.tb_next
+
+ self.assertEqual(b.current_offset, tb.tb_lasti)
+
+ def test_from_traceback_dis(self):
+ tb = get_tb()
+ b = dis.Bytecode.from_traceback(tb)
+ self.assertEqual(b.dis(), dis_traceback)
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index 86259c3..56193e8 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -409,7 +409,8 @@ Compare `DocTestCase`:
"""
-def test_DocTestFinder(): r"""
+class test_DocTestFinder:
+ def basics(): r"""
Unit tests for the `DocTestFinder` class.
DocTestFinder is used to extract DocTests from an object's docstring
@@ -646,6 +647,39 @@ DocTestFinder finds the line number of each example:
[1, 9, 12]
"""
+ if int.__doc__: # simple check for --without-doc-strings, skip if lacking
+ def non_Python_modules(): r"""
+
+Finding Doctests in Modules Not Written in Python
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+DocTestFinder can also find doctests in most modules not written in Python.
+We'll use builtins as an example, since it almost certainly isn't written in
+plain ol' Python and is guaranteed to be available.
+
+ >>> import builtins
+ >>> tests = doctest.DocTestFinder().find(builtins)
+ >>> 790 < len(tests) < 800 # approximate number of objects with docstrings
+ True
+ >>> real_tests = [t for t in tests if len(t.examples) > 0]
+ >>> len(real_tests) # objects that actually have doctests
+ 8
+ >>> for t in real_tests:
+ ... print('{} {}'.format(len(t.examples), t.name))
+ ...
+ 1 builtins.bin
+ 3 builtins.float.as_integer_ratio
+ 2 builtins.float.fromhex
+ 2 builtins.float.hex
+ 1 builtins.hex
+ 1 builtins.int
+ 2 builtins.int.bit_length
+ 1 builtins.oct
+
+Note here that 'bin', 'oct', and 'hex' are functions; 'float.as_integer_ratio',
+'float.hex', and 'int.bit_length' are methods; 'float.fromhex' is a classmethod,
+and 'int' is a type.
+"""
+
def test_DocTestParser(): r"""
Unit tests for the `DocTestParser` class.
@@ -1436,8 +1470,40 @@ However, output from `report_start` is not suppressed:
2
TestResults(failed=3, attempted=5)
-For the purposes of REPORT_ONLY_FIRST_FAILURE, unexpected exceptions
-count as failures:
+The FAIL_FAST flag causes the runner to exit after the first failing example,
+so subsequent examples are not even attempted:
+
+ >>> flags = doctest.FAIL_FAST
+ >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+ ... # doctest: +ELLIPSIS
+ **********************************************************************
+ File ..., line 5, in f
+ Failed example:
+ print(2) # first failure
+ Expected:
+ 200
+ Got:
+ 2
+ TestResults(failed=1, attempted=2)
+
+Specifying both FAIL_FAST and REPORT_ONLY_FIRST_FAILURE is equivalent to
+FAIL_FAST only:
+
+ >>> flags = doctest.FAIL_FAST | doctest.REPORT_ONLY_FIRST_FAILURE
+ >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+ ... # doctest: +ELLIPSIS
+ **********************************************************************
+ File ..., line 5, in f
+ Failed example:
+ print(2) # first failure
+ Expected:
+ 200
+ Got:
+ 2
+ TestResults(failed=1, attempted=2)
+
+For the purposes of both REPORT_ONLY_FIRST_FAILURE and FAIL_FAST, unexpected
+exceptions count as failures:
>>> def f(x):
... r'''
@@ -1464,6 +1530,17 @@ count as failures:
...
ValueError: 2
TestResults(failed=3, attempted=5)
+ >>> flags = doctest.FAIL_FAST
+ >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test)
+ ... # doctest: +ELLIPSIS
+ **********************************************************************
+ File ..., line 5, in f
+ Failed example:
+ raise ValueError(2) # first failure
+ Exception raised:
+ ...
+ ValueError: 2
+ TestResults(failed=1, attempted=2)
New option flags can also be registered, via register_optionflag(). Here
we reach into doctest's internals a bit.
@@ -2580,6 +2657,240 @@ Check doctest with a non-ascii filename:
TestResults(failed=1, attempted=1)
"""
+def test_CLI(): r"""
+The doctest module can be used to run doctests against an arbitrary file.
+These tests test this CLI functionality.
+
+We'll use the support module's script_helpers for this, and write a test files
+to a temp dir to run the command against. Due to a current limitation in
+script_helpers, though, we need a little utility function to turn the returned
+output into something we can doctest against:
+
+ >>> def normalize(s):
+ ... return '\n'.join(s.decode().splitlines())
+
+Note: we also pass TERM='' to all the assert_python calls to avoid a bug
+in the readline library that is triggered in these tests because we are
+running them in a new python process. See:
+
+ http://lists.gnu.org/archive/html/bug-readline/2013-06/msg00000.html
+
+With those preliminaries out of the way, we'll start with a file with two
+simple tests and no errors. We'll run both the unadorned doctest command, and
+the verbose version, and then check the output:
+
+ >>> from test import script_helper
+ >>> with script_helper.temp_dir() as tmpdir:
+ ... fn = os.path.join(tmpdir, 'myfile.doc')
+ ... with open(fn, 'w') as f:
+ ... _ = f.write('This is a very simple test file.\n')
+ ... _ = f.write(' >>> 1 + 1\n')
+ ... _ = f.write(' 2\n')
+ ... _ = f.write(' >>> "a"\n')
+ ... _ = f.write(" 'a'\n")
+ ... _ = f.write('\n')
+ ... _ = f.write('And that is it.\n')
+ ... rc1, out1, err1 = script_helper.assert_python_ok(
+ ... '-m', 'doctest', fn, TERM='')
+ ... rc2, out2, err2 = script_helper.assert_python_ok(
+ ... '-m', 'doctest', '-v', fn, TERM='')
+
+With no arguments and passing tests, we should get no output:
+
+ >>> rc1, out1, err1
+ (0, b'', b'')
+
+With the verbose flag, we should see the test output, but no error output:
+
+ >>> rc2, err2
+ (0, b'')
+ >>> print(normalize(out2))
+ Trying:
+ 1 + 1
+ Expecting:
+ 2
+ ok
+ Trying:
+ "a"
+ Expecting:
+ 'a'
+ ok
+ 1 items passed all tests:
+ 2 tests in myfile.doc
+ 2 tests in 1 items.
+ 2 passed and 0 failed.
+ Test passed.
+
+Now we'll write a couple files, one with three tests, the other a python module
+with two tests, both of the files having "errors" in the tests that can be made
+non-errors by applying the appropriate doctest options to the run (ELLIPSIS in
+the first file, NORMALIZE_WHITESPACE in the second). This combination will
+allow to thoroughly test the -f and -o flags, as well as the doctest command's
+ability to process more than one file on the command line and, since the second
+file ends in '.py', its handling of python module files (as opposed to straight
+text files).
+
+ >>> from test import script_helper
+ >>> with script_helper.temp_dir() as tmpdir:
+ ... fn = os.path.join(tmpdir, 'myfile.doc')
+ ... with open(fn, 'w') as f:
+ ... _ = f.write('This is another simple test file.\n')
+ ... _ = f.write(' >>> 1 + 1\n')
+ ... _ = f.write(' 2\n')
+ ... _ = f.write(' >>> "abcdef"\n')
+ ... _ = f.write(" 'a...f'\n")
+ ... _ = f.write(' >>> "ajkml"\n')
+ ... _ = f.write(" 'a...l'\n")
+ ... _ = f.write('\n')
+ ... _ = f.write('And that is it.\n')
+ ... fn2 = os.path.join(tmpdir, 'myfile2.py')
+ ... with open(fn2, 'w') as f:
+ ... _ = f.write('def test_func():\n')
+ ... _ = f.write(' \"\"\"\n')
+ ... _ = f.write(' This is simple python test function.\n')
+ ... _ = f.write(' >>> 1 + 1\n')
+ ... _ = f.write(' 2\n')
+ ... _ = f.write(' >>> "abc def"\n')
+ ... _ = f.write(" 'abc def'\n")
+ ... _ = f.write("\n")
+ ... _ = f.write(' \"\"\"\n')
+ ... import shutil
+ ... rc1, out1, err1 = script_helper.assert_python_failure(
+ ... '-m', 'doctest', fn, fn2, TERM='')
+ ... rc2, out2, err2 = script_helper.assert_python_ok(
+ ... '-m', 'doctest', '-o', 'ELLIPSIS', fn, TERM='')
+ ... rc3, out3, err3 = script_helper.assert_python_ok(
+ ... '-m', 'doctest', '-o', 'ELLIPSIS',
+ ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2, TERM='')
+ ... rc4, out4, err4 = script_helper.assert_python_failure(
+ ... '-m', 'doctest', '-f', fn, fn2, TERM='')
+ ... rc5, out5, err5 = script_helper.assert_python_ok(
+ ... '-m', 'doctest', '-v', '-o', 'ELLIPSIS',
+ ... '-o', 'NORMALIZE_WHITESPACE', fn, fn2, TERM='')
+
+Our first test run will show the errors from the first file (doctest stops if a
+file has errors). Note that doctest test-run error output appears on stdout,
+not stderr:
+
+ >>> rc1, err1
+ (1, b'')
+ >>> print(normalize(out1)) # doctest: +ELLIPSIS
+ **********************************************************************
+ File "...myfile.doc", line 4, in myfile.doc
+ Failed example:
+ "abcdef"
+ Expected:
+ 'a...f'
+ Got:
+ 'abcdef'
+ **********************************************************************
+ File "...myfile.doc", line 6, in myfile.doc
+ Failed example:
+ "ajkml"
+ Expected:
+ 'a...l'
+ Got:
+ 'ajkml'
+ **********************************************************************
+ 1 items had failures:
+ 2 of 3 in myfile.doc
+ ***Test Failed*** 2 failures.
+
+With -o ELLIPSIS specified, the second run, against just the first file, should
+produce no errors, and with -o NORMALIZE_WHITESPACE also specified, neither
+should the third, which ran against both files:
+
+ >>> rc2, out2, err2
+ (0, b'', b'')
+ >>> rc3, out3, err3
+ (0, b'', b'')
+
+The fourth run uses FAIL_FAST, so we should see only one error:
+
+ >>> rc4, err4
+ (1, b'')
+ >>> print(normalize(out4)) # doctest: +ELLIPSIS
+ **********************************************************************
+ File "...myfile.doc", line 4, in myfile.doc
+ Failed example:
+ "abcdef"
+ Expected:
+ 'a...f'
+ Got:
+ 'abcdef'
+ **********************************************************************
+ 1 items had failures:
+ 1 of 2 in myfile.doc
+ ***Test Failed*** 1 failures.
+
+The fifth test uses verbose with the two options, so we should get verbose
+success output for the tests in both files:
+
+ >>> rc5, err5
+ (0, b'')
+ >>> print(normalize(out5))
+ Trying:
+ 1 + 1
+ Expecting:
+ 2
+ ok
+ Trying:
+ "abcdef"
+ Expecting:
+ 'a...f'
+ ok
+ Trying:
+ "ajkml"
+ Expecting:
+ 'a...l'
+ ok
+ 1 items passed all tests:
+ 3 tests in myfile.doc
+ 3 tests in 1 items.
+ 3 passed and 0 failed.
+ Test passed.
+ Trying:
+ 1 + 1
+ Expecting:
+ 2
+ ok
+ Trying:
+ "abc def"
+ Expecting:
+ 'abc def'
+ ok
+ 1 items had no tests:
+ myfile2
+ 1 items passed all tests:
+ 2 tests in myfile2.test_func
+ 2 tests in 2 items.
+ 2 passed and 0 failed.
+ Test passed.
+
+We should also check some typical error cases.
+
+Invalid file name:
+
+ >>> rc, out, err = script_helper.assert_python_failure(
+ ... '-m', 'doctest', 'nosuchfile', TERM='')
+ >>> rc, out
+ (1, b'')
+ >>> print(normalize(err)) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ FileNotFoundError: [Errno ...] No such file or directory: 'nosuchfile'
+
+Invalid doctest option:
+
+ >>> rc, out, err = script_helper.assert_python_failure(
+ ... '-m', 'doctest', '-o', 'nosuchoption', TERM='')
+ >>> rc, out
+ (2, b'')
+ >>> print(normalize(err)) # doctest: +ELLIPSIS
+ usage...invalid...nosuchoption...
+
+"""
+
######################################################################
## Main
######################################################################
diff --git a/Lib/test/test_dynamicclassattribute.py b/Lib/test/test_dynamicclassattribute.py
new file mode 100644
index 0000000..079d6c3
--- /dev/null
+++ b/Lib/test/test_dynamicclassattribute.py
@@ -0,0 +1,304 @@
+# Test case for DynamicClassAttribute
+# more tests are in test_descr
+
+import abc
+import sys
+import unittest
+from test.support import run_unittest
+from types import DynamicClassAttribute
+
+class PropertyBase(Exception):
+ pass
+
+class PropertyGet(PropertyBase):
+ pass
+
+class PropertySet(PropertyBase):
+ pass
+
+class PropertyDel(PropertyBase):
+ pass
+
+class BaseClass(object):
+ def __init__(self):
+ self._spam = 5
+
+ @DynamicClassAttribute
+ def spam(self):
+ """BaseClass.getter"""
+ return self._spam
+
+ @spam.setter
+ def spam(self, value):
+ self._spam = value
+
+ @spam.deleter
+ def spam(self):
+ del self._spam
+
+class SubClass(BaseClass):
+
+ spam = BaseClass.__dict__['spam']
+
+ @spam.getter
+ def spam(self):
+ """SubClass.getter"""
+ raise PropertyGet(self._spam)
+
+ @spam.setter
+ def spam(self, value):
+ raise PropertySet(self._spam)
+
+ @spam.deleter
+ def spam(self):
+ raise PropertyDel(self._spam)
+
+class PropertyDocBase(object):
+ _spam = 1
+ def _get_spam(self):
+ return self._spam
+ spam = DynamicClassAttribute(_get_spam, doc="spam spam spam")
+
+class PropertyDocSub(PropertyDocBase):
+ spam = PropertyDocBase.__dict__['spam']
+ @spam.getter
+ def spam(self):
+ """The decorator does not use this doc string"""
+ return self._spam
+
+class PropertySubNewGetter(BaseClass):
+ spam = BaseClass.__dict__['spam']
+ @spam.getter
+ def spam(self):
+ """new docstring"""
+ return 5
+
+class PropertyNewGetter(object):
+ @DynamicClassAttribute
+ def spam(self):
+ """original docstring"""
+ return 1
+ @spam.getter
+ def spam(self):
+ """new docstring"""
+ return 8
+
+class ClassWithAbstractVirtualProperty(metaclass=abc.ABCMeta):
+ @DynamicClassAttribute
+ @abc.abstractmethod
+ def color():
+ pass
+
+class ClassWithPropertyAbstractVirtual(metaclass=abc.ABCMeta):
+ @abc.abstractmethod
+ @DynamicClassAttribute
+ def color():
+ pass
+
+class PropertyTests(unittest.TestCase):
+ def test_property_decorator_baseclass(self):
+ # see #1620
+ base = BaseClass()
+ self.assertEqual(base.spam, 5)
+ self.assertEqual(base._spam, 5)
+ base.spam = 10
+ self.assertEqual(base.spam, 10)
+ self.assertEqual(base._spam, 10)
+ delattr(base, "spam")
+ self.assertTrue(not hasattr(base, "spam"))
+ self.assertTrue(not hasattr(base, "_spam"))
+ base.spam = 20
+ self.assertEqual(base.spam, 20)
+ self.assertEqual(base._spam, 20)
+
+ def test_property_decorator_subclass(self):
+ # see #1620
+ sub = SubClass()
+ self.assertRaises(PropertyGet, getattr, sub, "spam")
+ self.assertRaises(PropertySet, setattr, sub, "spam", None)
+ self.assertRaises(PropertyDel, delattr, sub, "spam")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_property_decorator_subclass_doc(self):
+ sub = SubClass()
+ self.assertEqual(sub.__class__.__dict__['spam'].__doc__, "SubClass.getter")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_property_decorator_baseclass_doc(self):
+ base = BaseClass()
+ self.assertEqual(base.__class__.__dict__['spam'].__doc__, "BaseClass.getter")
+
+ def test_property_decorator_doc(self):
+ base = PropertyDocBase()
+ sub = PropertyDocSub()
+ self.assertEqual(base.__class__.__dict__['spam'].__doc__, "spam spam spam")
+ self.assertEqual(sub.__class__.__dict__['spam'].__doc__, "spam spam spam")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_property_getter_doc_override(self):
+ newgettersub = PropertySubNewGetter()
+ self.assertEqual(newgettersub.spam, 5)
+ self.assertEqual(newgettersub.__class__.__dict__['spam'].__doc__, "new docstring")
+ newgetter = PropertyNewGetter()
+ self.assertEqual(newgetter.spam, 8)
+ self.assertEqual(newgetter.__class__.__dict__['spam'].__doc__, "new docstring")
+
+ def test_property___isabstractmethod__descriptor(self):
+ for val in (True, False, [], [1], '', '1'):
+ class C(object):
+ def foo(self):
+ pass
+ foo.__isabstractmethod__ = val
+ foo = DynamicClassAttribute(foo)
+ self.assertIs(C.__dict__['foo'].__isabstractmethod__, bool(val))
+
+ # check that the DynamicClassAttribute's __isabstractmethod__ descriptor does the
+ # right thing when presented with a value that fails truth testing:
+ class NotBool(object):
+ def __nonzero__(self):
+ raise ValueError()
+ __len__ = __nonzero__
+ with self.assertRaises(ValueError):
+ class C(object):
+ def foo(self):
+ pass
+ foo.__isabstractmethod__ = NotBool()
+ foo = DynamicClassAttribute(foo)
+
+ def test_abstract_virtual(self):
+ self.assertRaises(TypeError, ClassWithAbstractVirtualProperty)
+ self.assertRaises(TypeError, ClassWithPropertyAbstractVirtual)
+ class APV(ClassWithPropertyAbstractVirtual):
+ pass
+ self.assertRaises(TypeError, APV)
+ class AVP(ClassWithAbstractVirtualProperty):
+ pass
+ self.assertRaises(TypeError, AVP)
+ class Okay1(ClassWithAbstractVirtualProperty):
+ @DynamicClassAttribute
+ def color(self):
+ return self._color
+ def __init__(self):
+ self._color = 'cyan'
+ with self.assertRaises(AttributeError):
+ Okay1.color
+ self.assertEqual(Okay1().color, 'cyan')
+ class Okay2(ClassWithAbstractVirtualProperty):
+ @DynamicClassAttribute
+ def color(self):
+ return self._color
+ def __init__(self):
+ self._color = 'magenta'
+ with self.assertRaises(AttributeError):
+ Okay2.color
+ self.assertEqual(Okay2().color, 'magenta')
+
+
+# Issue 5890: subclasses of DynamicClassAttribute do not preserve method __doc__ strings
+class PropertySub(DynamicClassAttribute):
+ """This is a subclass of DynamicClassAttribute"""
+
+class PropertySubSlots(DynamicClassAttribute):
+ """This is a subclass of DynamicClassAttribute that defines __slots__"""
+ __slots__ = ()
+
+class PropertySubclassTests(unittest.TestCase):
+
+ @unittest.skipIf(hasattr(PropertySubSlots, '__doc__'),
+ "__doc__ is already present, __slots__ will have no effect")
+ def test_slots_docstring_copy_exception(self):
+ try:
+ class Foo(object):
+ @PropertySubSlots
+ def spam(self):
+ """Trying to copy this docstring will raise an exception"""
+ return 1
+ print('\n',spam.__doc__)
+ except AttributeError:
+ pass
+ else:
+ raise Exception("AttributeError not raised")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_docstring_copy(self):
+ class Foo(object):
+ @PropertySub
+ def spam(self):
+ """spam wrapped in DynamicClassAttribute subclass"""
+ return 1
+ self.assertEqual(
+ Foo.__dict__['spam'].__doc__,
+ "spam wrapped in DynamicClassAttribute subclass")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_property_setter_copies_getter_docstring(self):
+ class Foo(object):
+ def __init__(self): self._spam = 1
+ @PropertySub
+ def spam(self):
+ """spam wrapped in DynamicClassAttribute subclass"""
+ return self._spam
+ @spam.setter
+ def spam(self, value):
+ """this docstring is ignored"""
+ self._spam = value
+ foo = Foo()
+ self.assertEqual(foo.spam, 1)
+ foo.spam = 2
+ self.assertEqual(foo.spam, 2)
+ self.assertEqual(
+ Foo.__dict__['spam'].__doc__,
+ "spam wrapped in DynamicClassAttribute subclass")
+ class FooSub(Foo):
+ spam = Foo.__dict__['spam']
+ @spam.setter
+ def spam(self, value):
+ """another ignored docstring"""
+ self._spam = 'eggs'
+ foosub = FooSub()
+ self.assertEqual(foosub.spam, 1)
+ foosub.spam = 7
+ self.assertEqual(foosub.spam, 'eggs')
+ self.assertEqual(
+ FooSub.__dict__['spam'].__doc__,
+ "spam wrapped in DynamicClassAttribute subclass")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ def test_property_new_getter_new_docstring(self):
+
+ class Foo(object):
+ @PropertySub
+ def spam(self):
+ """a docstring"""
+ return 1
+ @spam.getter
+ def spam(self):
+ """a new docstring"""
+ return 2
+ self.assertEqual(Foo.__dict__['spam'].__doc__, "a new docstring")
+ class FooBase(object):
+ @PropertySub
+ def spam(self):
+ """a docstring"""
+ return 1
+ class Foo2(FooBase):
+ spam = FooBase.__dict__['spam']
+ @spam.getter
+ def spam(self):
+ """a new docstring"""
+ return 2
+ self.assertEqual(Foo.__dict__['spam'].__doc__, "a new docstring")
+
+
+
+def test_main():
+ run_unittest(PropertyTests, PropertySubclassTests)
+
+if __name__ == '__main__':
+ test_main()
diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py
index f206ace..d8896ee 100644
--- a/Lib/test/test_email/__init__.py
+++ b/Lib/test/test_email/__init__.py
@@ -2,6 +2,7 @@ import os
import sys
import unittest
import test.support
+import collections
import email
from email.message import Message
from email._policybase import compat32
@@ -42,6 +43,8 @@ class TestEmailBase(unittest.TestCase):
# here we make minimal changes in the test_email tests compared to their
# pre-3.3 state.
policy = compat32
+ # Likewise, the default message object is Message.
+ message = Message
def __init__(self, *args, **kw):
super().__init__(*args, **kw)
@@ -54,11 +57,23 @@ class TestEmailBase(unittest.TestCase):
with openfile(filename) as fp:
return email.message_from_file(fp, policy=self.policy)
- def _str_msg(self, string, message=Message, policy=None):
+ def _str_msg(self, string, message=None, policy=None):
if policy is None:
policy = self.policy
+ if message is None:
+ message = self.message
return email.message_from_string(string, message, policy=policy)
+ def _bytes_msg(self, bytestring, message=None, policy=None):
+ if policy is None:
+ policy = self.policy
+ if message is None:
+ message = self.message
+ return email.message_from_bytes(bytestring, message, policy=policy)
+
+ def _make_message(self):
+ return self.message(policy=self.policy)
+
def _bytes_repr(self, b):
return [repr(x) for x in b.splitlines(keepends=True)]
@@ -123,6 +138,7 @@ def parameterize(cls):
"""
paramdicts = {}
+ testers = collections.defaultdict(list)
for name, attr in cls.__dict__.items():
if name.endswith('_params'):
if not hasattr(attr, 'keys'):
@@ -134,7 +150,15 @@ def parameterize(cls):
d[n] = x
attr = d
paramdicts[name[:-7] + '_as_'] = attr
+ if '_as_' in name:
+ testers[name.split('_as_')[0] + '_as_'].append(name)
testfuncs = {}
+ for name in paramdicts:
+ if name not in testers:
+ raise ValueError("No tester found for {}".format(name))
+ for name in testers:
+ if name not in paramdicts:
+ raise ValueError("No params found for {}".format(name))
for name, attr in cls.__dict__.items():
for paramsname, paramsdict in paramdicts.items():
if name.startswith(paramsname):
diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py
new file mode 100644
index 0000000..cdb04e4
--- /dev/null
+++ b/Lib/test/test_email/test_contentmanager.py
@@ -0,0 +1,796 @@
+import unittest
+from test.test_email import TestEmailBase, parameterize
+import textwrap
+from email import policy
+from email.message import EmailMessage
+from email.contentmanager import ContentManager, raw_data_manager
+
+
+@parameterize
+class TestContentManager(TestEmailBase):
+
+ policy = policy.default
+ message = EmailMessage
+
+ get_key_params = {
+ 'full_type': (1, 'text/plain',),
+ 'maintype_only': (2, 'text',),
+ 'null_key': (3, '',),
+ }
+
+ def get_key_as_get_content_key(self, order, key):
+ def foo_getter(msg, foo=None):
+ bar = msg['X-Bar-Header']
+ return foo, bar
+ cm = ContentManager()
+ cm.add_get_handler(key, foo_getter)
+ m = self._make_message()
+ m['Content-Type'] = 'text/plain'
+ m['X-Bar-Header'] = 'foo'
+ self.assertEqual(cm.get_content(m, foo='bar'), ('bar', 'foo'))
+
+ def get_key_as_get_content_key_order(self, order, key):
+ def bar_getter(msg):
+ return msg['X-Bar-Header']
+ def foo_getter(msg):
+ return msg['X-Foo-Header']
+ cm = ContentManager()
+ cm.add_get_handler(key, foo_getter)
+ for precedence, key in self.get_key_params.values():
+ if precedence > order:
+ cm.add_get_handler(key, bar_getter)
+ m = self._make_message()
+ m['Content-Type'] = 'text/plain'
+ m['X-Bar-Header'] = 'bar'
+ m['X-Foo-Header'] = 'foo'
+ self.assertEqual(cm.get_content(m), ('foo'))
+
+ def test_get_content_raises_if_unknown_mimetype_and_no_default(self):
+ cm = ContentManager()
+ m = self._make_message()
+ m['Content-Type'] = 'text/plain'
+ with self.assertRaisesRegex(KeyError, 'text/plain'):
+ cm.get_content(m)
+
+ class BaseThing(str):
+ pass
+ baseobject_full_path = __name__ + '.' + 'TestContentManager.BaseThing'
+ class Thing(BaseThing):
+ pass
+ testobject_full_path = __name__ + '.' + 'TestContentManager.Thing'
+
+ set_key_params = {
+ 'type': (0, Thing,),
+ 'full_path': (1, testobject_full_path,),
+ 'qualname': (2, 'TestContentManager.Thing',),
+ 'name': (3, 'Thing',),
+ 'base_type': (4, BaseThing,),
+ 'base_full_path': (5, baseobject_full_path,),
+ 'base_qualname': (6, 'TestContentManager.BaseThing',),
+ 'base_name': (7, 'BaseThing',),
+ 'str_type': (8, str,),
+ 'str_full_path': (9, 'builtins.str',),
+ 'str_name': (10, 'str',), # str name and qualname are the same
+ 'null_key': (11, None,),
+ }
+
+ def set_key_as_set_content_key(self, order, key):
+ def foo_setter(msg, obj, foo=None):
+ msg['X-Foo-Header'] = foo
+ msg.set_payload(obj)
+ cm = ContentManager()
+ cm.add_set_handler(key, foo_setter)
+ m = self._make_message()
+ msg_obj = self.Thing()
+ cm.set_content(m, msg_obj, foo='bar')
+ self.assertEqual(m['X-Foo-Header'], 'bar')
+ self.assertEqual(m.get_payload(), msg_obj)
+
+ def set_key_as_set_content_key_order(self, order, key):
+ def foo_setter(msg, obj):
+ msg['X-FooBar-Header'] = 'foo'
+ msg.set_payload(obj)
+ def bar_setter(msg, obj):
+ msg['X-FooBar-Header'] = 'bar'
+ cm = ContentManager()
+ cm.add_set_handler(key, foo_setter)
+ for precedence, key in self.get_key_params.values():
+ if precedence > order:
+ cm.add_set_handler(key, bar_setter)
+ m = self._make_message()
+ msg_obj = self.Thing()
+ cm.set_content(m, msg_obj)
+ self.assertEqual(m['X-FooBar-Header'], 'foo')
+ self.assertEqual(m.get_payload(), msg_obj)
+
+ def test_set_content_raises_if_unknown_type_and_no_default(self):
+ cm = ContentManager()
+ m = self._make_message()
+ msg_obj = self.Thing()
+ with self.assertRaisesRegex(KeyError, self.testobject_full_path):
+ cm.set_content(m, msg_obj)
+
+ def test_set_content_raises_if_called_on_multipart(self):
+ cm = ContentManager()
+ m = self._make_message()
+ m['Content-Type'] = 'multipart/foo'
+ with self.assertRaises(TypeError):
+ cm.set_content(m, 'test')
+
+ def test_set_content_calls_clear_content(self):
+ m = self._make_message()
+ m['Content-Foo'] = 'bar'
+ m['Content-Type'] = 'text/html'
+ m['To'] = 'test'
+ m.set_payload('abc')
+ cm = ContentManager()
+ cm.add_set_handler(str, lambda *args, **kw: None)
+ m.set_content('xyz', content_manager=cm)
+ self.assertIsNone(m['Content-Foo'])
+ self.assertIsNone(m['Content-Type'])
+ self.assertEqual(m['To'], 'test')
+ self.assertIsNone(m.get_payload())
+
+
+@parameterize
+class TestRawDataManager(TestEmailBase):
+ # Note: these tests are dependent on the order in which headers are added
+ # to the message objects by the code. There's no defined ordering in
+ # RFC5322/MIME, so this makes the tests more fragile than the standards
+ # require. However, if the header order changes it is best to understand
+ # *why*, and make sure it isn't a subtle bug in whatever change was
+ # applied.
+
+ policy = policy.default.clone(max_line_length=60,
+ content_manager=raw_data_manager)
+ message = EmailMessage
+
+ def test_get_text_plain(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain
+
+ Basic text.
+ """))
+ self.assertEqual(raw_data_manager.get_content(m), "Basic text.\n")
+
+ def test_get_text_html(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/html
+
+ <p>Basic text.</p>
+ """))
+ self.assertEqual(raw_data_manager.get_content(m),
+ "<p>Basic text.</p>\n")
+
+ def test_get_text_plain_latin1(self):
+ m = self._bytes_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset=latin1
+
+ Basìc tëxt.
+ """).encode('latin1'))
+ self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n")
+
+ def test_get_text_plain_latin1_quoted_printable(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset="latin-1"
+ Content-Transfer-Encoding: quoted-printable
+
+ Bas=ECc t=EBxt.
+ """))
+ self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n")
+
+ def test_get_text_plain_utf8_base64(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf8"
+ Content-Transfer-Encoding: base64
+
+ QmFzw6xjIHTDq3h0Lgo=
+ """))
+ self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt.\n")
+
+ def test_get_text_plain_bad_utf8_quoted_printable(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf8"
+ Content-Transfer-Encoding: quoted-printable
+
+ Bas=c3=acc t=c3=abxt=fd.
+ """))
+ self.assertEqual(raw_data_manager.get_content(m), "Basìc tëxt�.\n")
+
+ def test_get_text_plain_bad_utf8_quoted_printable_ignore_errors(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf8"
+ Content-Transfer-Encoding: quoted-printable
+
+ Bas=c3=acc t=c3=abxt=fd.
+ """))
+ self.assertEqual(raw_data_manager.get_content(m, errors='ignore'),
+ "Basìc tëxt.\n")
+
+ def test_get_text_plain_utf8_base64_recoverable_bad_CTE_data(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf8"
+ Content-Transfer-Encoding: base64
+
+ QmFzw6xjIHTDq3h0Lgo\xFF=
+ """))
+ self.assertEqual(raw_data_manager.get_content(m, errors='ignore'),
+ "Basìc tëxt.\n")
+
+ def test_get_text_invalid_keyword(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: text/plain
+
+ Basic text.
+ """))
+ with self.assertRaises(TypeError):
+ raw_data_manager.get_content(m, foo='ignore')
+
+ def test_get_non_text(self):
+ template = textwrap.dedent("""\
+ Content-Type: {}
+ Content-Transfer-Encoding: base64
+
+ Ym9ndXMgZGF0YQ==
+ """)
+ for maintype in 'audio image video application'.split():
+ with self.subTest(maintype=maintype):
+ m = self._str_msg(template.format(maintype+'/foo'))
+ self.assertEqual(raw_data_manager.get_content(m), b"bogus data")
+
+ def test_get_non_text_invalid_keyword(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: image/jpg
+ Content-Transfer-Encoding: base64
+
+ Ym9ndXMgZGF0YQ==
+ """))
+ with self.assertRaises(TypeError):
+ raw_data_manager.get_content(m, errors='ignore')
+
+ def test_get_raises_on_multipart(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ --===--
+ """))
+ with self.assertRaises(KeyError):
+ raw_data_manager.get_content(m)
+
+ def test_get_message_rfc822_and_external_body(self):
+ template = textwrap.dedent("""\
+ Content-Type: message/{}
+
+ To: foo@example.com
+ From: bar@example.com
+ Subject: example
+
+ an example message
+ """)
+ for subtype in 'rfc822 external-body'.split():
+ with self.subTest(subtype=subtype):
+ m = self._str_msg(template.format(subtype))
+ sub_msg = raw_data_manager.get_content(m)
+ self.assertIsInstance(sub_msg, self.message)
+ self.assertEqual(raw_data_manager.get_content(sub_msg),
+ "an example message\n")
+ self.assertEqual(sub_msg['to'], 'foo@example.com')
+ self.assertEqual(sub_msg['from'].addresses[0].username, 'bar')
+
+ def test_get_message_non_rfc822_or_external_body_yields_bytes(self):
+ m = self._str_msg(textwrap.dedent("""\
+ Content-Type: message/partial
+
+ To: foo@example.com
+ From: bar@example.com
+ Subject: example
+
+ The real body is in another message.
+ """))
+ self.assertEqual(raw_data_manager.get_content(m)[:10], b'To: foo@ex')
+
+ def test_set_text_plain(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 7bit
+
+ Simple message.
+ """))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_html(self):
+ m = self._make_message()
+ content = "<p>Simple message.</p>\n"
+ raw_data_manager.set_content(m, content, subtype='html')
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/html; charset="utf-8"
+ Content-Transfer-Encoding: 7bit
+
+ <p>Simple message.</p>
+ """))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_charset_latin_1(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ raw_data_manager.set_content(m, content, charset='latin-1')
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="iso-8859-1"
+ Content-Transfer-Encoding: 7bit
+
+ Simple message.
+ """))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_short_line_minimal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = "et là il est monté sur moi et il commence à m'éto.\n"
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 8bit
+
+ et là il est monté sur moi et il commence à m'éto.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_long_line_minimal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = ("j'ai un problème de python. il est sorti de son"
+ " vivarium. et là il est monté sur moi et il commence"
+ " à m'éto.\n")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: quoted-printable
+
+ j'ai un probl=C3=A8me de python. il est sorti de son vivari=
+ um. et l=C3=A0 il est mont=C3=A9 sur moi et il commence =
+ =C3=A0 m'=C3=A9to.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_11_lines_long_line_minimal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = '\n'*10 + (
+ "j'ai un problème de python. il est sorti de son"
+ " vivarium. et là il est monté sur moi et il commence"
+ " à m'éto.\n")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: quoted-printable
+ """ + '\n'*10 + """
+ j'ai un probl=C3=A8me de python. il est sorti de son vivari=
+ um. et l=C3=A0 il est mont=C3=A9 sur moi et il commence =
+ =C3=A0 m'=C3=A9to.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_maximal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = "áàäéèęöő.\n"
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 8bit
+
+ áàäéèęöő.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_11_lines_maximal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = '\n'*10 + "áàäéèęöő.\n"
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 8bit
+ """ + '\n'*10 + """
+ áàäéèęöő.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_long_line_maximal_non_ascii_heuristics(self):
+ m = self._make_message()
+ content = ("áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: base64
+
+ w6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TDqcOoxJnD
+ tsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TDqcOo
+ xJnDtsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOgw6TD
+ qcOoxJnDtsWRw6HDoMOkw6nDqMSZw7bFkcOhw6DDpMOpw6jEmcO2xZHDocOg
+ w6TDqcOoxJnDtsWRLgo=
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_11_lines_long_line_maximal_non_ascii_heuristics(self):
+ # Yes, it chooses "wrong" here. It's a heuristic. So this result
+ # could change if we come up with a better heuristic.
+ m = self._make_message()
+ content = ('\n'*10 +
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n")
+ raw_data_manager.set_content(m, "\n"*10 +
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő"
+ "áàäéèęöőáàäéèęöőáàäéèęöőáàäéèęöő.\n")
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: quoted-printable
+ """ + '\n'*10 + """
+ =C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=
+ =A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=
+ =C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=
+ =A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=
+ =C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=
+ =91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=
+ =C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=
+ =A4=C3=A9=C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=
+ =C3=A8=C4=99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=
+ =99=C3=B6=C5=91=C3=A1=C3=A0=C3=A4=C3=A9=C3=A8=C4=99=C3=B6=
+ =C5=91.
+ """).encode('utf-8'))
+ self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_text_non_ascii_with_cte_7bit_raises(self):
+ m = self._make_message()
+ with self.assertRaises(UnicodeError):
+ raw_data_manager.set_content(m,"áàäéèęöő.\n", cte='7bit')
+
+ def test_set_text_non_ascii_with_charset_ascii_raises(self):
+ m = self._make_message()
+ with self.assertRaises(UnicodeError):
+ raw_data_manager.set_content(m,"áàäéèęöő.\n", charset='ascii')
+
+ def test_set_text_non_ascii_with_cte_7bit_and_charset_ascii_raises(self):
+ m = self._make_message()
+ with self.assertRaises(UnicodeError):
+ raw_data_manager.set_content(m,"áàäéèęöő.\n", cte='7bit', charset='ascii')
+
+ def test_set_message(self):
+ m = self._make_message()
+ m['Subject'] = "Forwarded message"
+ content = self._make_message()
+ content['To'] = 'python@vivarium.org'
+ content['From'] = 'police@monty.org'
+ content['Subject'] = "get back in your box"
+ content.set_content("Or face the comfy chair.")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Subject: Forwarded message
+ Content-Type: message/rfc822
+ Content-Transfer-Encoding: 8bit
+
+ To: python@vivarium.org
+ From: police@monty.org
+ Subject: get back in your box
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 7bit
+ MIME-Version: 1.0
+
+ Or face the comfy chair.
+ """))
+ payload = m.get_payload(0)
+ self.assertIsInstance(payload, self.message)
+ self.assertEqual(str(payload), str(content))
+ self.assertIsInstance(m.get_content(), self.message)
+ self.assertEqual(str(m.get_content()), str(content))
+
+ def test_set_message_with_non_ascii_and_coercion_to_7bit(self):
+ m = self._make_message()
+ m['Subject'] = "Escape report"
+ content = self._make_message()
+ content['To'] = 'police@monty.org'
+ content['From'] = 'victim@monty.org'
+ content['Subject'] = "Help"
+ content.set_content("j'ai un problème de python. il est sorti de son"
+ " vivarium.")
+ raw_data_manager.set_content(m, content)
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Subject: Escape report
+ Content-Type: message/rfc822
+ Content-Transfer-Encoding: 8bit
+
+ To: police@monty.org
+ From: victim@monty.org
+ Subject: Help
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 8bit
+ MIME-Version: 1.0
+
+ j'ai un problème de python. il est sorti de son vivarium.
+ """).encode('utf-8'))
+ # The choice of base64 for the body encoding is because generator
+ # doesn't bother with heuristics and uses it unconditionally for utf-8
+ # text.
+ # XXX: the first cte should be 7bit, too...that's a generator bug.
+ # XXX: the line length in the body also looks like a generator bug.
+ self.assertEqual(m.as_string(maxheaderlen=self.policy.max_line_length),
+ textwrap.dedent("""\
+ Subject: Escape report
+ Content-Type: message/rfc822
+ Content-Transfer-Encoding: 8bit
+
+ To: police@monty.org
+ From: victim@monty.org
+ Subject: Help
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: base64
+ MIME-Version: 1.0
+
+ aidhaSB1biBwcm9ibMOobWUgZGUgcHl0aG9uLiBpbCBlc3Qgc29ydGkgZGUgc29uIHZpdmFyaXVt
+ Lgo=
+ """))
+ self.assertIsInstance(m.get_content(), self.message)
+ self.assertEqual(str(m.get_content()), str(content))
+
+ def test_set_message_invalid_cte_raises(self):
+ m = self._make_message()
+ content = self._make_message()
+ for cte in 'quoted-printable base64'.split():
+ for subtype in 'rfc822 external-body'.split():
+ with self.subTest(cte=cte, subtype=subtype):
+ with self.assertRaises(ValueError) as ar:
+ m.set_content(content, subtype, cte=cte)
+ exc = str(ar.exception)
+ self.assertIn(cte, exc)
+ self.assertIn(subtype, exc)
+ subtype = 'external-body'
+ for cte in '8bit binary'.split():
+ with self.subTest(cte=cte, subtype=subtype):
+ with self.assertRaises(ValueError) as ar:
+ m.set_content(content, subtype, cte=cte)
+ exc = str(ar.exception)
+ self.assertIn(cte, exc)
+ self.assertIn(subtype, exc)
+
+ def test_set_image_jpg(self):
+ for content in (b"bogus content",
+ bytearray(b"bogus content"),
+ memoryview(b"bogus content")):
+ with self.subTest(content=content):
+ m = self._make_message()
+ raw_data_manager.set_content(m, content, 'image', 'jpeg')
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: image/jpeg
+ Content-Transfer-Encoding: base64
+
+ Ym9ndXMgY29udGVudA==
+ """))
+ self.assertEqual(m.get_payload(decode=True), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_audio_aif_with_quoted_printable_cte(self):
+ # Why you would use qp, I don't know, but it is technically supported.
+ # XXX: the incorrect line length is because binascii.b2a_qp doesn't
+ # support a line length parameter, but we must use it to get newline
+ # encoding.
+ # XXX: what about that lack of tailing newline? Do we actually handle
+ # that correctly in all cases? That is, if the *source* has an
+ # unencoded newline, do we add an extra newline to the returned payload
+ # or not? And can that actually be disambiguated based on the RFC?
+ m = self._make_message()
+ content = b'b\xFFgus\tcon\nt\rent ' + b'z'*100
+ m.set_content(content, 'audio', 'aif', cte='quoted-printable')
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: audio/aif
+ Content-Transfer-Encoding: quoted-printable
+ MIME-Version: 1.0
+
+ b=FFgus=09con=0At=0Dent=20zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=
+ zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz""").encode('latin-1'))
+ self.assertEqual(m.get_payload(decode=True), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_video_mpeg_with_binary_cte(self):
+ m = self._make_message()
+ content = b'b\xFFgus\tcon\nt\rent ' + b'z'*100
+ m.set_content(content, 'video', 'mpeg', cte='binary')
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: video/mpeg
+ Content-Transfer-Encoding: binary
+ MIME-Version: 1.0
+
+ """).encode('ascii') +
+ # XXX: the second \n ought to be a \r, but generator gets it wrong.
+ # THIS MEANS WE DON'T ACTUALLY SUPPORT THE 'binary' CTE.
+ b'b\xFFgus\tcon\nt\nent zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz' +
+ b'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz')
+ self.assertEqual(m.get_payload(decode=True), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_application_octet_stream_with_8bit_cte(self):
+ # In 8bit mode, univeral line end logic applies. It is up to the
+ # application to make sure the lines are short enough; we don't check.
+ m = self._make_message()
+ content = b'b\xFFgus\tcon\nt\rent\n' + b'z'*60 + b'\n'
+ m.set_content(content, 'application', 'octet-stream', cte='8bit')
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: application/octet-stream
+ Content-Transfer-Encoding: 8bit
+ MIME-Version: 1.0
+
+ """).encode('ascii') +
+ b'b\xFFgus\tcon\nt\nent\n' +
+ b'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n')
+ self.assertEqual(m.get_payload(decode=True), content)
+ self.assertEqual(m.get_content(), content)
+
+ def test_set_headers_from_header_objects(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ header_factory = self.policy.header_factory
+ raw_data_manager.set_content(m, content, headers=(
+ header_factory("To", "foo@example.com"),
+ header_factory("From", "foo@example.com"),
+ header_factory("Subject", "I'm talking to myself.")))
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ To: foo@example.com
+ From: foo@example.com
+ Subject: I'm talking to myself.
+ Content-Transfer-Encoding: 7bit
+
+ Simple message.
+ """))
+
+ def test_set_headers_from_strings(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ raw_data_manager.set_content(m, content, headers=(
+ "X-Foo-Header: foo",
+ "X-Bar-Header: bar",))
+ self.assertEqual(str(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ X-Foo-Header: foo
+ X-Bar-Header: bar
+ Content-Transfer-Encoding: 7bit
+
+ Simple message.
+ """))
+
+ def test_set_headers_with_invalid_duplicate_string_header_raises(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ with self.assertRaisesRegex(ValueError, 'Content-Type'):
+ raw_data_manager.set_content(m, content, headers=(
+ "Content-Type: foo/bar",)
+ )
+
+ def test_set_headers_with_invalid_duplicate_header_header_raises(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ header_factory = self.policy.header_factory
+ with self.assertRaisesRegex(ValueError, 'Content-Type'):
+ raw_data_manager.set_content(m, content, headers=(
+ header_factory("Content-Type", " foo/bar"),)
+ )
+
+ def test_set_headers_with_defective_string_header_raises(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ with self.assertRaisesRegex(ValueError, 'a@fairly@@invalid@address'):
+ raw_data_manager.set_content(m, content, headers=(
+ 'To: a@fairly@@invalid@address',)
+ )
+ print(m['To'].defects)
+
+ def test_set_headers_with_defective_header_header_raises(self):
+ m = self._make_message()
+ content = "Simple message.\n"
+ header_factory = self.policy.header_factory
+ with self.assertRaisesRegex(ValueError, 'a@fairly@@invalid@address'):
+ raw_data_manager.set_content(m, content, headers=(
+ header_factory('To', 'a@fairly@@invalid@address'),)
+ )
+ print(m['To'].defects)
+
+ def test_set_disposition_inline(self):
+ m = self._make_message()
+ m.set_content('foo', disposition='inline')
+ self.assertEqual(m['Content-Disposition'], 'inline')
+
+ def test_set_disposition_attachment(self):
+ m = self._make_message()
+ m.set_content('foo', disposition='attachment')
+ self.assertEqual(m['Content-Disposition'], 'attachment')
+
+ def test_set_disposition_foo(self):
+ m = self._make_message()
+ m.set_content('foo', disposition='foo')
+ self.assertEqual(m['Content-Disposition'], 'foo')
+
+ # XXX: we should have a 'strict' policy mode (beyond raise_on_defect) that
+ # would cause 'foo' above to raise.
+
+ def test_set_filename(self):
+ m = self._make_message()
+ m.set_content('foo', filename='bar.txt')
+ self.assertEqual(m['Content-Disposition'],
+ 'attachment; filename="bar.txt"')
+
+ def test_set_filename_and_disposition_inline(self):
+ m = self._make_message()
+ m.set_content('foo', disposition='inline', filename='bar.txt')
+ self.assertEqual(m['Content-Disposition'], 'inline; filename="bar.txt"')
+
+ def test_set_non_ascii_filename(self):
+ m = self._make_message()
+ m.set_content('foo', filename='ábárî.txt')
+ self.assertEqual(bytes(m), textwrap.dedent("""\
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 7bit
+ Content-Disposition: attachment;
+ filename*=utf-8''%C3%A1b%C3%A1r%C3%AE.txt
+ MIME-Version: 1.0
+
+ foo
+ """).encode('ascii'))
+
+ content_object_params = {
+ 'text_plain': ('content', ()),
+ 'text_html': ('content', ('html',)),
+ 'application_octet_stream': (b'content',
+ ('application', 'octet_stream')),
+ 'image_jpeg': (b'content', ('image', 'jpeg')),
+ 'message_rfc822': (message(), ()),
+ 'message_external_body': (message(), ('external-body',)),
+ }
+
+ def content_object_as_header_receiver(self, obj, mimetype):
+ m = self._make_message()
+ m.set_content(obj, *mimetype, headers=(
+ 'To: foo@example.com',
+ 'From: bar@simple.net'))
+ self.assertEqual(m['to'], 'foo@example.com')
+ self.assertEqual(m['from'], 'bar@simple.net')
+
+ def content_object_as_disposition_inline_receiver(self, obj, mimetype):
+ m = self._make_message()
+ m.set_content(obj, *mimetype, disposition='inline')
+ self.assertEqual(m['Content-Disposition'], 'inline')
+
+ def content_object_as_non_ascii_filename_receiver(self, obj, mimetype):
+ m = self._make_message()
+ m.set_content(obj, *mimetype, disposition='inline', filename='bár.txt')
+ self.assertEqual(m['Content-Disposition'], 'inline; filename="bár.txt"')
+ self.assertEqual(m.get_filename(), "bár.txt")
+ self.assertEqual(m['Content-Disposition'].params['filename'], "bár.txt")
+
+ def content_object_as_cid_receiver(self, obj, mimetype):
+ m = self._make_message()
+ m.set_content(obj, *mimetype, cid='some_random_stuff')
+ self.assertEqual(m['Content-ID'], 'some_random_stuff')
+
+ def content_object_as_params_receiver(self, obj, mimetype):
+ m = self._make_message()
+ params = {'foo': 'bár', 'abc': 'xyz'}
+ m.set_content(obj, *mimetype, params=params)
+ if isinstance(obj, str):
+ params['charset'] = 'utf-8'
+ self.assertEqual(m['Content-Type'].params, params)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index 51fe756..2f89a10 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -124,6 +124,14 @@ class TestMessageAPI(TestEmailBase):
msg.set_payload([])
self.assertEqual(msg.get_payload(), [])
+ def test_attach_when_payload_is_string(self):
+ msg = Message()
+ msg['Content-Type'] = 'multipart/mixed'
+ msg.set_payload('string payload')
+ sub_msg = MIMEMessage(Message())
+ self.assertRaisesRegex(TypeError, "[Aa]ttach.*non-multipart",
+ msg.attach, sub_msg)
+
def test_get_charsets(self):
eq = self.assertEqual
@@ -281,15 +289,42 @@ class TestMessageAPI(TestEmailBase):
self.assertIn('TO', msg)
def test_as_string(self):
- eq = self.ndiffAssertEqual
msg = self._msgobj('msg_01.txt')
with openfile('msg_01.txt') as fp:
text = fp.read()
- eq(text, str(msg))
+ self.assertEqual(text, str(msg))
fullrepr = msg.as_string(unixfrom=True)
lines = fullrepr.split('\n')
self.assertTrue(lines[0].startswith('From '))
- eq(text, NL.join(lines[1:]))
+ self.assertEqual(text, NL.join(lines[1:]))
+
+ def test_as_string_policy(self):
+ msg = self._msgobj('msg_01.txt')
+ newpolicy = msg.policy.clone(linesep='\r\n')
+ fullrepr = msg.as_string(policy=newpolicy)
+ s = StringIO()
+ g = Generator(s, policy=newpolicy)
+ g.flatten(msg)
+ self.assertEqual(fullrepr, s.getvalue())
+
+ def test_as_bytes(self):
+ msg = self._msgobj('msg_01.txt')
+ with openfile('msg_01.txt') as fp:
+ data = fp.read().encode('ascii')
+ self.assertEqual(data, bytes(msg))
+ fullrepr = msg.as_bytes(unixfrom=True)
+ lines = fullrepr.split(b'\n')
+ self.assertTrue(lines[0].startswith(b'From '))
+ self.assertEqual(data, b'\n'.join(lines[1:]))
+
+ def test_as_bytes_policy(self):
+ msg = self._msgobj('msg_01.txt')
+ newpolicy = msg.policy.clone(linesep='\r\n')
+ fullrepr = msg.as_bytes(policy=newpolicy)
+ s = BytesIO()
+ g = BytesGenerator(s,policy=newpolicy)
+ g.flatten(msg)
+ self.assertEqual(fullrepr, s.getvalue())
# test_headerregistry.TestContentTypeHeader.bad_params
def test_bad_param(self):
@@ -742,8 +777,15 @@ class TestEncoders(unittest.TestCase):
# whose output character set is 7bit gets a transfer-encoding
# of 7bit.
eq = self.assertEqual
- msg = MIMEText('文', _charset='euc-jp')
+ msg = MIMEText('文\n', _charset='euc-jp')
eq(msg['content-transfer-encoding'], '7bit')
+ eq(msg.as_string(), textwrap.dedent("""\
+ MIME-Version: 1.0
+ Content-Type: text/plain; charset="iso-2022-jp"
+ Content-Transfer-Encoding: 7bit
+
+ \x1b$BJ8\x1b(B
+ """))
def test_qp_encode_latin1(self):
msg = MIMEText('\xe1\xf6\n', 'text', 'ISO-8859-1')
diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py
index adaf3e8..7d64a38 100644
--- a/Lib/test/test_email/test_headerregistry.py
+++ b/Lib/test/test_email/test_headerregistry.py
@@ -661,7 +661,7 @@ class TestContentTypeHeader(TestHeaderBase):
'text/plain; name="ascii_is_the_default"'),
'rfc2231_bad_character_in_charset_parameter_value': (
- "text/plain; charset*=ascii''utf-8%E2%80%9D",
+ "text/plain; charset*=ascii''utf-8%F1%F2%F3",
'text/plain',
'text',
'plain',
@@ -669,6 +669,18 @@ class TestContentTypeHeader(TestHeaderBase):
[errors.UndecodableBytesDefect],
'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"'),
+ 'rfc2231_utf_8_in_supposedly_ascii_charset_parameter_value': (
+ "text/plain; charset*=ascii''utf-8%E2%80%9D",
+ 'text/plain',
+ 'text',
+ 'plain',
+ {'charset': 'utf-8”'},
+ [errors.UndecodableBytesDefect],
+ 'text/plain; charset="utf-8”"',
+ ),
+ # XXX: if the above were *re*folded, it would get tagged as utf-8
+ # instead of ascii in the param, since it now contains non-ASCII.
+
'rfc2231_encoded_then_unencoded_segments': (
('application/x-foo;'
'\tname*0*="us-ascii\'en-us\'My";'
diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py
index 8cc3f80..c761c62 100644
--- a/Lib/test/test_email/test_message.py
+++ b/Lib/test/test_email/test_message.py
@@ -1,6 +1,13 @@
import unittest
-from email import policy
-from test.test_email import TestEmailBase
+import textwrap
+from email import policy, message_from_string
+from email.message import EmailMessage, MIMEPart
+from test.test_email import TestEmailBase, parameterize
+
+
+# Helper.
+def first(iterable):
+ return next(filter(lambda x: x is not None, iterable), None)
class Test(TestEmailBase):
@@ -13,6 +20,753 @@ class Test(TestEmailBase):
with self.assertRaises(ValueError):
m['To'] = 'xyz@abc'
+ def test_rfc2043_auto_decoded_and_emailmessage_used(self):
+ m = message_from_string(textwrap.dedent("""\
+ Subject: Ayons asperges pour le =?utf-8?q?d=C3=A9jeuner?=
+ From: =?utf-8?q?Pep=C3=A9?= Le Pew <pepe@example.com>
+ To: "Penelope Pussycat" <"penelope@example.com">
+ MIME-Version: 1.0
+ Content-Type: text/plain; charset="utf-8"
+
+ sample text
+ """), policy=policy.default)
+ self.assertEqual(m['subject'], "Ayons asperges pour le déjeuner")
+ self.assertEqual(m['from'], "Pepé Le Pew <pepe@example.com>")
+ self.assertIsInstance(m, EmailMessage)
+
+
+@parameterize
+class TestEmailMessageBase:
+
+ policy = policy.default
+
+ # The first argument is a triple (related, html, plain) of indices into the
+ # list returned by 'walk' called on a Message constructed from the third.
+ # The indices indicate which part should match the corresponding part-type
+ # when passed to get_body (ie: the "first" part of that type in the
+ # message). The second argument is a list of indices into the 'walk' list
+ # of the attachments that should be returned by a call to
+ # 'iter_attachments'. The third argument is a list of indices into 'walk'
+ # that should be returned by a call to 'iter_parts'. Note that the first
+ # item returned by 'walk' is the Message itself.
+
+ message_params = {
+
+ 'empty_message': (
+ (None, None, 0),
+ (),
+ (),
+ ""),
+
+ 'non_mime_plain': (
+ (None, None, 0),
+ (),
+ (),
+ textwrap.dedent("""\
+ To: foo@example.com
+
+ simple text body
+ """)),
+
+ 'mime_non_text': (
+ (None, None, None),
+ (),
+ (),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: image/jpg
+
+ bogus body.
+ """)),
+
+ 'plain_html_alternative': (
+ (None, 2, 1),
+ (),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/alternative; boundary="==="
+
+ preamble
+
+ --===
+ Content-Type: text/plain
+
+ simple body
+
+ --===
+ Content-Type: text/html
+
+ <p>simple body</p>
+ --===--
+ """)),
+
+ 'plain_html_mixed': (
+ (None, 2, 1),
+ (),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ preamble
+
+ --===
+ Content-Type: text/plain
+
+ simple body
+
+ --===
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --===--
+ """)),
+
+ 'plain_html_attachment_mixed': (
+ (None, None, 1),
+ (2,),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: text/plain
+
+ simple body
+
+ --===
+ Content-Type: text/html
+ Content-Disposition: attachment
+
+ <p>simple body</p>
+
+ --===--
+ """)),
+
+ 'html_text_attachment_mixed': (
+ (None, 2, None),
+ (1,),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: text/plain
+ Content-Disposition: AtTaChment
+
+ simple body
+
+ --===
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --===--
+ """)),
+
+ 'html_text_attachment_inline_mixed': (
+ (None, 2, 1),
+ (),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: text/plain
+ Content-Disposition: InLine
+
+ simple body
+
+ --===
+ Content-Type: text/html
+ Content-Disposition: inline
+
+ <p>simple body</p>
+
+ --===--
+ """)),
+
+ # RFC 2387
+ 'related': (
+ (0, 1, None),
+ (2,),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/related; boundary="==="; type=text/html
+
+ --===
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --===
+ Content-Type: image/jpg
+ Content-ID: <image1>
+
+ bogus data
+
+ --===--
+ """)),
+
+ # This message structure will probably never be seen in the wild, but
+ # it proves we distinguish between text parts based on 'start'. The
+ # content would not, of course, actually work :)
+ 'related_with_start': (
+ (0, 2, None),
+ (1,),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/related; boundary="==="; type=text/html;
+ start="<body>"
+
+ --===
+ Content-Type: text/html
+ Content-ID: <include>
+
+ useless text
+
+ --===
+ Content-Type: text/html
+ Content-ID: <body>
+
+ <p>simple body</p>
+ <!--#include file="<include>"-->
+
+ --===--
+ """)),
+
+
+ 'mixed_alternative_plain_related': (
+ (3, 4, 2),
+ (6, 7),
+ (1, 6, 7),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: multipart/alternative; boundary="+++"
+
+ --+++
+ Content-Type: text/plain
+
+ simple body
+
+ --+++
+ Content-Type: multipart/related; boundary="___"
+
+ --___
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --___
+ Content-Type: image/jpg
+ Content-ID: <image1@cid>
+
+ bogus jpg body
+
+ --___--
+
+ --+++--
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: attachment
+
+ bogus jpg body
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: AttacHmenT
+
+ another bogus jpg body
+
+ --===--
+ """)),
+
+ # This structure suggested by Stephen J. Turnbull...may not exist/be
+ # supported in the wild, but we want to support it.
+ 'mixed_related_alternative_plain_html': (
+ (1, 4, 3),
+ (6, 7),
+ (1, 6, 7),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: multipart/related; boundary="+++"
+
+ --+++
+ Content-Type: multipart/alternative; boundary="___"
+
+ --___
+ Content-Type: text/plain
+
+ simple body
+
+ --___
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --___--
+
+ --+++
+ Content-Type: image/jpg
+ Content-ID: <image1@cid>
+
+ bogus jpg body
+
+ --+++--
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: attachment
+
+ bogus jpg body
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: attachment
+
+ another bogus jpg body
+
+ --===--
+ """)),
+
+ # Same thing, but proving we only look at the root part, which is the
+ # first one if there isn't any start parameter. That is, this is a
+ # broken related.
+ 'mixed_related_alternative_plain_html_wrong_order': (
+ (1, None, None),
+ (6, 7),
+ (1, 6, 7),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: multipart/related; boundary="+++"
+
+ --+++
+ Content-Type: image/jpg
+ Content-ID: <image1@cid>
+
+ bogus jpg body
+
+ --+++
+ Content-Type: multipart/alternative; boundary="___"
+
+ --___
+ Content-Type: text/plain
+
+ simple body
+
+ --___
+ Content-Type: text/html
+
+ <p>simple body</p>
+
+ --___--
+
+ --+++--
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: attachment
+
+ bogus jpg body
+
+ --===
+ Content-Type: image/jpg
+ Content-Disposition: attachment
+
+ another bogus jpg body
+
+ --===--
+ """)),
+
+ 'message_rfc822': (
+ (None, None, None),
+ (),
+ (),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: message/rfc822
+
+ To: bar@example.com
+ From: robot@examp.com
+
+ this is a message body.
+ """)),
+
+ 'mixed_text_message_rfc822': (
+ (None, None, 1),
+ (2,),
+ (1, 2),
+ textwrap.dedent("""\
+ To: foo@example.com
+ MIME-Version: 1.0
+ Content-Type: multipart/mixed; boundary="==="
+
+ --===
+ Content-Type: text/plain
+
+ Your message has bounced, ser.
+
+ --===
+ Content-Type: message/rfc822
+
+ To: bar@example.com
+ From: robot@examp.com
+
+ this is a message body.
+
+ --===--
+ """)),
+
+ }
+
+ def message_as_get_body(self, body_parts, attachments, parts, msg):
+ m = self._str_msg(msg)
+ allparts = list(m.walk())
+ expected = [None if n is None else allparts[n] for n in body_parts]
+ related = 0; html = 1; plain = 2
+ self.assertEqual(m.get_body(), first(expected))
+ self.assertEqual(m.get_body(preferencelist=(
+ 'related', 'html', 'plain')),
+ first(expected))
+ self.assertEqual(m.get_body(preferencelist=('related', 'html')),
+ first(expected[related:html+1]))
+ self.assertEqual(m.get_body(preferencelist=('related', 'plain')),
+ first([expected[related], expected[plain]]))
+ self.assertEqual(m.get_body(preferencelist=('html', 'plain')),
+ first(expected[html:plain+1]))
+ self.assertEqual(m.get_body(preferencelist=['related']),
+ expected[related])
+ self.assertEqual(m.get_body(preferencelist=['html']), expected[html])
+ self.assertEqual(m.get_body(preferencelist=['plain']), expected[plain])
+ self.assertEqual(m.get_body(preferencelist=('plain', 'html')),
+ first(expected[plain:html-1:-1]))
+ self.assertEqual(m.get_body(preferencelist=('plain', 'related')),
+ first([expected[plain], expected[related]]))
+ self.assertEqual(m.get_body(preferencelist=('html', 'related')),
+ first(expected[html::-1]))
+ self.assertEqual(m.get_body(preferencelist=('plain', 'html', 'related')),
+ first(expected[::-1]))
+ self.assertEqual(m.get_body(preferencelist=('html', 'plain', 'related')),
+ first([expected[html],
+ expected[plain],
+ expected[related]]))
+
+ def message_as_iter_attachment(self, body_parts, attachments, parts, msg):
+ m = self._str_msg(msg)
+ allparts = list(m.walk())
+ attachments = [allparts[n] for n in attachments]
+ self.assertEqual(list(m.iter_attachments()), attachments)
+
+ def message_as_iter_parts(self, body_parts, attachments, parts, msg):
+ m = self._str_msg(msg)
+ allparts = list(m.walk())
+ parts = [allparts[n] for n in parts]
+ self.assertEqual(list(m.iter_parts()), parts)
+
+ class _TestContentManager:
+ def get_content(self, msg, *args, **kw):
+ return msg, args, kw
+ def set_content(self, msg, *args, **kw):
+ self.msg = msg
+ self.args = args
+ self.kw = kw
+
+ def test_get_content_with_cm(self):
+ m = self._str_msg('')
+ cm = self._TestContentManager()
+ self.assertEqual(m.get_content(content_manager=cm), (m, (), {}))
+ msg, args, kw = m.get_content('foo', content_manager=cm, bar=1, k=2)
+ self.assertEqual(msg, m)
+ self.assertEqual(args, ('foo',))
+ self.assertEqual(kw, dict(bar=1, k=2))
+
+ def test_get_content_default_cm_comes_from_policy(self):
+ p = policy.default.clone(content_manager=self._TestContentManager())
+ m = self._str_msg('', policy=p)
+ self.assertEqual(m.get_content(), (m, (), {}))
+ msg, args, kw = m.get_content('foo', bar=1, k=2)
+ self.assertEqual(msg, m)
+ self.assertEqual(args, ('foo',))
+ self.assertEqual(kw, dict(bar=1, k=2))
+
+ def test_set_content_with_cm(self):
+ m = self._str_msg('')
+ cm = self._TestContentManager()
+ m.set_content(content_manager=cm)
+ self.assertEqual(cm.msg, m)
+ self.assertEqual(cm.args, ())
+ self.assertEqual(cm.kw, {})
+ m.set_content('foo', content_manager=cm, bar=1, k=2)
+ self.assertEqual(cm.msg, m)
+ self.assertEqual(cm.args, ('foo',))
+ self.assertEqual(cm.kw, dict(bar=1, k=2))
+
+ def test_set_content_default_cm_comes_from_policy(self):
+ cm = self._TestContentManager()
+ p = policy.default.clone(content_manager=cm)
+ m = self._str_msg('', policy=p)
+ m.set_content()
+ self.assertEqual(cm.msg, m)
+ self.assertEqual(cm.args, ())
+ self.assertEqual(cm.kw, {})
+ m.set_content('foo', bar=1, k=2)
+ self.assertEqual(cm.msg, m)
+ self.assertEqual(cm.args, ('foo',))
+ self.assertEqual(cm.kw, dict(bar=1, k=2))
+
+ # outcome is whether xxx_method should raise ValueError error when called
+ # on multipart/subtype. Blank outcome means it depends on xxx (add
+ # succeeds, make raises). Note: 'none' means there are content-type
+ # headers but payload is None...this happening in practice would be very
+ # unusual, so treating it as if there were content seems reasonable.
+ # method subtype outcome
+ subtype_params = (
+ ('related', 'no_content', 'succeeds'),
+ ('related', 'none', 'succeeds'),
+ ('related', 'plain', 'succeeds'),
+ ('related', 'related', ''),
+ ('related', 'alternative', 'raises'),
+ ('related', 'mixed', 'raises'),
+ ('alternative', 'no_content', 'succeeds'),
+ ('alternative', 'none', 'succeeds'),
+ ('alternative', 'plain', 'succeeds'),
+ ('alternative', 'related', 'succeeds'),
+ ('alternative', 'alternative', ''),
+ ('alternative', 'mixed', 'raises'),
+ ('mixed', 'no_content', 'succeeds'),
+ ('mixed', 'none', 'succeeds'),
+ ('mixed', 'plain', 'succeeds'),
+ ('mixed', 'related', 'succeeds'),
+ ('mixed', 'alternative', 'succeeds'),
+ ('mixed', 'mixed', ''),
+ )
+
+ def _make_subtype_test_message(self, subtype):
+ m = self.message()
+ payload = None
+ msg_headers = [
+ ('To', 'foo@bar.com'),
+ ('From', 'bar@foo.com'),
+ ]
+ if subtype != 'no_content':
+ ('content-shadow', 'Logrus'),
+ msg_headers.append(('X-Random-Header', 'Corwin'))
+ if subtype == 'text':
+ payload = ''
+ msg_headers.append(('Content-Type', 'text/plain'))
+ m.set_payload('')
+ elif subtype != 'no_content':
+ payload = []
+ msg_headers.append(('Content-Type', 'multipart/' + subtype))
+ msg_headers.append(('X-Trump', 'Random'))
+ m.set_payload(payload)
+ for name, value in msg_headers:
+ m[name] = value
+ return m, msg_headers, payload
+
+ def _check_disallowed_subtype_raises(self, m, method_name, subtype, method):
+ with self.assertRaises(ValueError) as ar:
+ getattr(m, method)()
+ exc_text = str(ar.exception)
+ self.assertIn(subtype, exc_text)
+ self.assertIn(method_name, exc_text)
+
+ def _check_make_multipart(self, m, msg_headers, payload):
+ count = 0
+ for name, value in msg_headers:
+ if not name.lower().startswith('content-'):
+ self.assertEqual(m[name], value)
+ count += 1
+ self.assertEqual(len(m), count+1) # +1 for new Content-Type
+ part = next(m.iter_parts())
+ count = 0
+ for name, value in msg_headers:
+ if name.lower().startswith('content-'):
+ self.assertEqual(part[name], value)
+ count += 1
+ self.assertEqual(len(part), count)
+ self.assertEqual(part.get_payload(), payload)
+
+ def subtype_as_make(self, method, subtype, outcome):
+ m, msg_headers, payload = self._make_subtype_test_message(subtype)
+ make_method = 'make_' + method
+ if outcome in ('', 'raises'):
+ self._check_disallowed_subtype_raises(m, method, subtype, make_method)
+ return
+ getattr(m, make_method)()
+ self.assertEqual(m.get_content_maintype(), 'multipart')
+ self.assertEqual(m.get_content_subtype(), method)
+ if subtype == 'no_content':
+ self.assertEqual(len(m.get_payload()), 0)
+ self.assertEqual(m.items(),
+ msg_headers + [('Content-Type',
+ 'multipart/'+method)])
+ else:
+ self.assertEqual(len(m.get_payload()), 1)
+ self._check_make_multipart(m, msg_headers, payload)
+
+ def subtype_as_make_with_boundary(self, method, subtype, outcome):
+ # Doing all variation is a bit of overkill...
+ m = self.message()
+ if outcome in ('', 'raises'):
+ m['Content-Type'] = 'multipart/' + subtype
+ with self.assertRaises(ValueError) as cm:
+ getattr(m, 'make_' + method)()
+ return
+ if subtype == 'plain':
+ m['Content-Type'] = 'text/plain'
+ elif subtype != 'no_content':
+ m['Content-Type'] = 'multipart/' + subtype
+ getattr(m, 'make_' + method)(boundary="abc")
+ self.assertTrue(m.is_multipart())
+ self.assertEqual(m.get_boundary(), 'abc')
+
+ def test_policy_on_part_made_by_make_comes_from_message(self):
+ for method in ('make_related', 'make_alternative', 'make_mixed'):
+ m = self.message(policy=self.policy.clone(content_manager='foo'))
+ m['Content-Type'] = 'text/plain'
+ getattr(m, method)()
+ self.assertEqual(m.get_payload(0).policy.content_manager, 'foo')
+
+ class _TestSetContentManager:
+ def set_content(self, msg, content, *args, **kw):
+ msg['Content-Type'] = 'text/plain'
+ msg.set_payload(content)
+
+ def subtype_as_add(self, method, subtype, outcome):
+ m, msg_headers, payload = self._make_subtype_test_message(subtype)
+ cm = self._TestSetContentManager()
+ add_method = 'add_attachment' if method=='mixed' else 'add_' + method
+ if outcome == 'raises':
+ self._check_disallowed_subtype_raises(m, method, subtype, add_method)
+ return
+ getattr(m, add_method)('test', content_manager=cm)
+ self.assertEqual(m.get_content_maintype(), 'multipart')
+ self.assertEqual(m.get_content_subtype(), method)
+ if method == subtype or subtype == 'no_content':
+ self.assertEqual(len(m.get_payload()), 1)
+ for name, value in msg_headers:
+ self.assertEqual(m[name], value)
+ part = m.get_payload()[0]
+ else:
+ self.assertEqual(len(m.get_payload()), 2)
+ self._check_make_multipart(m, msg_headers, payload)
+ part = m.get_payload()[1]
+ self.assertEqual(part.get_content_type(), 'text/plain')
+ self.assertEqual(part.get_payload(), 'test')
+ if method=='mixed':
+ self.assertEqual(part['Content-Disposition'], 'attachment')
+ elif method=='related':
+ self.assertEqual(part['Content-Disposition'], 'inline')
+ else:
+ # Otherwise we don't guess.
+ self.assertIsNone(part['Content-Disposition'])
+
+ class _TestSetRaisingContentManager:
+ def set_content(self, msg, content, *args, **kw):
+ raise Exception('test')
+
+ def test_default_content_manager_for_add_comes_from_policy(self):
+ cm = self._TestSetRaisingContentManager()
+ m = self.message(policy=self.policy.clone(content_manager=cm))
+ for method in ('add_related', 'add_alternative', 'add_attachment'):
+ with self.assertRaises(Exception) as ar:
+ getattr(m, method)('')
+ self.assertEqual(str(ar.exception), 'test')
+
+ def message_as_clear(self, body_parts, attachments, parts, msg):
+ m = self._str_msg(msg)
+ m.clear()
+ self.assertEqual(len(m), 0)
+ self.assertEqual(list(m.items()), [])
+ self.assertIsNone(m.get_payload())
+ self.assertEqual(list(m.iter_parts()), [])
+
+ def message_as_clear_content(self, body_parts, attachments, parts, msg):
+ m = self._str_msg(msg)
+ expected_headers = [h for h in m.keys()
+ if not h.lower().startswith('content-')]
+ m.clear_content()
+ self.assertEqual(list(m.keys()), expected_headers)
+ self.assertIsNone(m.get_payload())
+ self.assertEqual(list(m.iter_parts()), [])
+
+ def test_is_attachment(self):
+ m = self._make_message()
+ self.assertFalse(m.is_attachment)
+ m['Content-Disposition'] = 'inline'
+ self.assertFalse(m.is_attachment)
+ m.replace_header('Content-Disposition', 'attachment')
+ self.assertTrue(m.is_attachment)
+ m.replace_header('Content-Disposition', 'AtTachMent')
+ self.assertTrue(m.is_attachment)
+
+
+
+class TestEmailMessage(TestEmailMessageBase, TestEmailBase):
+ message = EmailMessage
+
+ def test_set_content_adds_MIME_Version(self):
+ m = self._str_msg('')
+ cm = self._TestContentManager()
+ self.assertNotIn('MIME-Version', m)
+ m.set_content(content_manager=cm)
+ self.assertEqual(m['MIME-Version'], '1.0')
+
+ class _MIME_Version_adding_CM:
+ def set_content(self, msg, *args, **kw):
+ msg['MIME-Version'] = '1.0'
+
+ def test_set_content_does_not_duplicate_MIME_Version(self):
+ m = self._str_msg('')
+ cm = self._MIME_Version_adding_CM()
+ self.assertNotIn('MIME-Version', m)
+ m.set_content(content_manager=cm)
+ self.assertEqual(m['MIME-Version'], '1.0')
+
+
+class TestMIMEPart(TestEmailMessageBase, TestEmailBase):
+ # Doing the full test run here may seem a bit redundant, since the two
+ # classes are almost identical. But what if they drift apart? So we do
+ # the full tests so that any future drift doesn't introduce bugs.
+ message = MIMEPart
+
+ def test_set_content_does_not_add_MIME_Version(self):
+ m = self._str_msg('')
+ cm = self._TestContentManager()
+ self.assertNotIn('MIME-Version', m)
+ m.set_content(content_manager=cm)
+ self.assertNotIn('MIME-Version', m)
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py
index 983bd49..e797f36 100644
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -30,6 +30,7 @@ class PolicyAPITests(unittest.TestCase):
'raise_on_defect': False,
'header_factory': email.policy.EmailPolicy.header_factory,
'refold_source': 'long',
+ 'content_manager': email.policy.EmailPolicy.content_manager,
})
# For each policy under test, we give here what we expect the defaults to
@@ -318,5 +319,14 @@ class TestPolicyPropagation(unittest.TestCase):
self.assertEqual(msg.as_string(), "Subject: testXTo: fooXX")
+class TestConcretePolicies(unittest.TestCase):
+
+ def test_header_store_parse_rejects_newlines(self):
+ instance = email.policy.EmailPolicy()
+ self.assertRaises(ValueError,
+ instance.header_store_parse,
+ 'From', 'spam\negg@foo.py')
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py
index 544b1bb..19cf64f 100644
--- a/Lib/test/test_email/torture_test.py
+++ b/Lib/test/test_email/torture_test.py
@@ -27,7 +27,7 @@ def openfile(filename):
# Prevent this test from running in the Python distro
try:
openfile('crispin-torture.txt')
-except IOError:
+except OSError:
raise TestSkipped
diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py
new file mode 100644
index 0000000..70b4153
--- /dev/null
+++ b/Lib/test/test_ensurepip.py
@@ -0,0 +1,345 @@
+import unittest
+import unittest.mock
+import test.support
+import os
+import os.path
+import contextlib
+import sys
+
+import ensurepip
+import ensurepip._uninstall
+
+# pip currently requires ssl support, so we ensure we handle
+# it being missing (http://bugs.python.org/issue19744)
+ensurepip_no_ssl = test.support.import_fresh_module("ensurepip",
+ blocked=["ssl"])
+try:
+ import ssl
+except ImportError:
+ def requires_usable_pip(f):
+ deco = unittest.skip(ensurepip._MISSING_SSL_MESSAGE)
+ return deco(f)
+else:
+ def requires_usable_pip(f):
+ return f
+
+class TestEnsurePipVersion(unittest.TestCase):
+
+ def test_returns_version(self):
+ self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
+
+class EnsurepipMixin:
+
+ def setUp(self):
+ run_pip_patch = unittest.mock.patch("ensurepip._run_pip")
+ self.run_pip = run_pip_patch.start()
+ self.addCleanup(run_pip_patch.stop)
+
+ # Avoid side effects on the actual os module
+ real_devnull = os.devnull
+ os_patch = unittest.mock.patch("ensurepip.os")
+ patched_os = os_patch.start()
+ self.addCleanup(os_patch.stop)
+ patched_os.devnull = real_devnull
+ patched_os.path = os.path
+ self.os_environ = patched_os.environ = os.environ.copy()
+
+
+class TestBootstrap(EnsurepipMixin, unittest.TestCase):
+
+ @requires_usable_pip
+ def test_basic_bootstrapping(self):
+ ensurepip.bootstrap()
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ additional_paths = self.run_pip.call_args[0][1]
+ self.assertEqual(len(additional_paths), 2)
+
+ @requires_usable_pip
+ def test_bootstrapping_with_root(self):
+ ensurepip.bootstrap(root="/foo/bar/")
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "--root", "/foo/bar/",
+ "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_user(self):
+ ensurepip.bootstrap(user=True)
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "--user", "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_upgrade(self):
+ ensurepip.bootstrap(upgrade=True)
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "--upgrade", "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_verbosity_1(self):
+ ensurepip.bootstrap(verbosity=1)
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "-v", "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_verbosity_2(self):
+ ensurepip.bootstrap(verbosity=2)
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "-vv", "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_verbosity_3(self):
+ ensurepip.bootstrap(verbosity=3)
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "-vvv", "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ @requires_usable_pip
+ def test_bootstrapping_with_regular_install(self):
+ ensurepip.bootstrap()
+ self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "install")
+
+ @requires_usable_pip
+ def test_bootstrapping_with_alt_install(self):
+ ensurepip.bootstrap(altinstall=True)
+ self.assertEqual(self.os_environ["ENSUREPIP_OPTIONS"], "altinstall")
+
+ @requires_usable_pip
+ def test_bootstrapping_with_default_pip(self):
+ ensurepip.bootstrap(default_pip=True)
+ self.assertNotIn("ENSUREPIP_OPTIONS", self.os_environ)
+
+ def test_altinstall_default_pip_conflict(self):
+ with self.assertRaises(ValueError):
+ ensurepip.bootstrap(altinstall=True, default_pip=True)
+ self.run_pip.assert_not_called()
+
+ @requires_usable_pip
+ def test_pip_environment_variables_removed(self):
+ # ensurepip deliberately ignores all pip environment variables
+ # See http://bugs.python.org/issue19734 for details
+ self.os_environ["PIP_THIS_SHOULD_GO_AWAY"] = "test fodder"
+ ensurepip.bootstrap()
+ self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ)
+
+ @requires_usable_pip
+ def test_pip_config_file_disabled(self):
+ # ensurepip deliberately ignores the pip config file
+ # See http://bugs.python.org/issue20053 for details
+ ensurepip.bootstrap()
+ self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
+
+@contextlib.contextmanager
+def fake_pip(version=ensurepip._PIP_VERSION):
+ if version is None:
+ pip = None
+ else:
+ class FakePip():
+ __version__ = version
+ pip = FakePip()
+ sentinel = object()
+ orig_pip = sys.modules.get("pip", sentinel)
+ sys.modules["pip"] = pip
+ try:
+ yield pip
+ finally:
+ if orig_pip is sentinel:
+ del sys.modules["pip"]
+ else:
+ sys.modules["pip"] = orig_pip
+
+class TestUninstall(EnsurepipMixin, unittest.TestCase):
+
+ def test_uninstall_skipped_when_not_installed(self):
+ with fake_pip(None):
+ ensurepip._uninstall_helper()
+ self.run_pip.assert_not_called()
+
+ def test_uninstall_skipped_with_warning_for_wrong_version(self):
+ with fake_pip("not a valid version"):
+ with test.support.captured_stderr() as stderr:
+ ensurepip._uninstall_helper()
+ warning = stderr.getvalue().strip()
+ self.assertIn("only uninstall a matching version", warning)
+ self.run_pip.assert_not_called()
+
+
+ @requires_usable_pip
+ def test_uninstall(self):
+ with fake_pip():
+ ensurepip._uninstall_helper()
+
+ self.run_pip.assert_called_once_with(
+ ["uninstall", "-y", "pip", "setuptools"]
+ )
+
+ @requires_usable_pip
+ def test_uninstall_with_verbosity_1(self):
+ with fake_pip():
+ ensurepip._uninstall_helper(verbosity=1)
+
+ self.run_pip.assert_called_once_with(
+ ["uninstall", "-y", "-v", "pip", "setuptools"]
+ )
+
+ @requires_usable_pip
+ def test_uninstall_with_verbosity_2(self):
+ with fake_pip():
+ ensurepip._uninstall_helper(verbosity=2)
+
+ self.run_pip.assert_called_once_with(
+ ["uninstall", "-y", "-vv", "pip", "setuptools"]
+ )
+
+ @requires_usable_pip
+ def test_uninstall_with_verbosity_3(self):
+ with fake_pip():
+ ensurepip._uninstall_helper(verbosity=3)
+
+ self.run_pip.assert_called_once_with(
+ ["uninstall", "-y", "-vvv", "pip", "setuptools"]
+ )
+
+ @requires_usable_pip
+ def test_pip_environment_variables_removed(self):
+ # ensurepip deliberately ignores all pip environment variables
+ # See http://bugs.python.org/issue19734 for details
+ self.os_environ["PIP_THIS_SHOULD_GO_AWAY"] = "test fodder"
+ with fake_pip():
+ ensurepip._uninstall_helper()
+ self.assertNotIn("PIP_THIS_SHOULD_GO_AWAY", self.os_environ)
+
+ @requires_usable_pip
+ def test_pip_config_file_disabled(self):
+ # ensurepip deliberately ignores the pip config file
+ # See http://bugs.python.org/issue20053 for details
+ with fake_pip():
+ ensurepip._uninstall_helper()
+ self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
+
+
+class TestMissingSSL(EnsurepipMixin, unittest.TestCase):
+
+ def setUp(self):
+ sys.modules["ensurepip"] = ensurepip_no_ssl
+ @self.addCleanup
+ def restore_module():
+ sys.modules["ensurepip"] = ensurepip
+ super().setUp()
+
+ def test_bootstrap_requires_ssl(self):
+ self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder"
+ with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"):
+ ensurepip_no_ssl.bootstrap()
+ self.run_pip.assert_not_called()
+ self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ)
+
+ def test_uninstall_requires_ssl(self):
+ self.os_environ["PIP_THIS_SHOULD_STAY"] = "test fodder"
+ with self.assertRaisesRegex(RuntimeError, "requires SSL/TLS"):
+ with fake_pip():
+ ensurepip_no_ssl._uninstall_helper()
+ self.run_pip.assert_not_called()
+ self.assertIn("PIP_THIS_SHOULD_STAY", self.os_environ)
+
+ def test_main_exits_early_with_warning(self):
+ with test.support.captured_stderr() as stderr:
+ ensurepip_no_ssl._main(["--version"])
+ warning = stderr.getvalue().strip()
+ self.assertTrue(warning.endswith("requires SSL/TLS"), warning)
+ self.run_pip.assert_not_called()
+
+# Basic testing of the main functions and their argument parsing
+
+EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION
+
+class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase):
+
+ @requires_usable_pip
+ def test_bootstrap_version(self):
+ with test.support.captured_stdout() as stdout:
+ with self.assertRaises(SystemExit):
+ ensurepip._main(["--version"])
+ result = stdout.getvalue().strip()
+ self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+ self.run_pip.assert_not_called()
+
+ @requires_usable_pip
+ def test_basic_bootstrapping(self):
+ ensurepip._main([])
+
+ self.run_pip.assert_called_once_with(
+ [
+ "install", "--no-index", "--find-links",
+ unittest.mock.ANY, "setuptools", "pip",
+ ],
+ unittest.mock.ANY,
+ )
+
+ additional_paths = self.run_pip.call_args[0][1]
+ self.assertEqual(len(additional_paths), 2)
+
+class TestUninstallationMainFunction(EnsurepipMixin, unittest.TestCase):
+
+ def test_uninstall_version(self):
+ with test.support.captured_stdout() as stdout:
+ with self.assertRaises(SystemExit):
+ ensurepip._uninstall._main(["--version"])
+ result = stdout.getvalue().strip()
+ self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+ self.run_pip.assert_not_called()
+
+ @requires_usable_pip
+ def test_basic_uninstall(self):
+ with fake_pip():
+ ensurepip._uninstall._main([])
+
+ self.run_pip.assert_called_once_with(
+ ["uninstall", "-y", "pip", "setuptools"]
+ )
+
+
+
+if __name__ == "__main__":
+ test.support.run_unittest(__name__)
diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py
new file mode 100644
index 0000000..b8ef632
--- /dev/null
+++ b/Lib/test/test_enum.py
@@ -0,0 +1,1594 @@
+import enum
+import inspect
+import pydoc
+import unittest
+from collections import OrderedDict
+from enum import Enum, IntEnum, EnumMeta, unique
+from io import StringIO
+from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL
+
+# for pickle tests
+try:
+ class Stooges(Enum):
+ LARRY = 1
+ CURLY = 2
+ MOE = 3
+except Exception as exc:
+ Stooges = exc
+
+try:
+ class IntStooges(int, Enum):
+ LARRY = 1
+ CURLY = 2
+ MOE = 3
+except Exception as exc:
+ IntStooges = exc
+
+try:
+ class FloatStooges(float, Enum):
+ LARRY = 1.39
+ CURLY = 2.72
+ MOE = 3.142596
+except Exception as exc:
+ FloatStooges = exc
+
+# for pickle test and subclass tests
+try:
+ class StrEnum(str, Enum):
+ 'accepts only string values'
+ class Name(StrEnum):
+ BDFL = 'Guido van Rossum'
+ FLUFL = 'Barry Warsaw'
+except Exception as exc:
+ Name = exc
+
+try:
+ Question = Enum('Question', 'who what when where why', module=__name__)
+except Exception as exc:
+ Question = exc
+
+try:
+ Answer = Enum('Answer', 'him this then there because')
+except Exception as exc:
+ Answer = exc
+
+try:
+ Theory = Enum('Theory', 'rule law supposition', qualname='spanish_inquisition')
+except Exception as exc:
+ Theory = exc
+
+# for doctests
+try:
+ class Fruit(Enum):
+ tomato = 1
+ banana = 2
+ cherry = 3
+except Exception:
+ pass
+
+def test_pickle_dump_load(assertion, source, target=None,
+ *, protocol=(0, HIGHEST_PROTOCOL)):
+ start, stop = protocol
+ if target is None:
+ target = source
+ for protocol in range(start, stop+1):
+ assertion(loads(dumps(source, protocol=protocol)), target)
+
+def test_pickle_exception(assertion, exception, obj,
+ *, protocol=(0, HIGHEST_PROTOCOL)):
+ start, stop = protocol
+ for protocol in range(start, stop+1):
+ with assertion(exception):
+ dumps(obj, protocol=protocol)
+
+class TestHelpers(unittest.TestCase):
+ # _is_descriptor, _is_sunder, _is_dunder
+
+ def test_is_descriptor(self):
+ class foo:
+ pass
+ for attr in ('__get__','__set__','__delete__'):
+ obj = foo()
+ self.assertFalse(enum._is_descriptor(obj))
+ setattr(obj, attr, 1)
+ self.assertTrue(enum._is_descriptor(obj))
+
+ def test_is_sunder(self):
+ for s in ('_a_', '_aa_'):
+ self.assertTrue(enum._is_sunder(s))
+
+ for s in ('a', 'a_', '_a', '__a', 'a__', '__a__', '_a__', '__a_', '_',
+ '__', '___', '____', '_____',):
+ self.assertFalse(enum._is_sunder(s))
+
+ def test_is_dunder(self):
+ for s in ('__a__', '__aa__'):
+ self.assertTrue(enum._is_dunder(s))
+ for s in ('a', 'a_', '_a', '__a', 'a__', '_a_', '_a__', '__a_', '_',
+ '__', '___', '____', '_____',):
+ self.assertFalse(enum._is_dunder(s))
+
+
+class TestEnum(unittest.TestCase):
+
+ def setUp(self):
+ class Season(Enum):
+ SPRING = 1
+ SUMMER = 2
+ AUTUMN = 3
+ WINTER = 4
+ self.Season = Season
+
+ class Konstants(float, Enum):
+ E = 2.7182818
+ PI = 3.1415926
+ TAU = 2 * PI
+ self.Konstants = Konstants
+
+ class Grades(IntEnum):
+ A = 5
+ B = 4
+ C = 3
+ D = 2
+ F = 0
+ self.Grades = Grades
+
+ class Directional(str, Enum):
+ EAST = 'east'
+ WEST = 'west'
+ NORTH = 'north'
+ SOUTH = 'south'
+ self.Directional = Directional
+
+ from datetime import date
+ class Holiday(date, Enum):
+ NEW_YEAR = 2013, 1, 1
+ IDES_OF_MARCH = 2013, 3, 15
+ self.Holiday = Holiday
+
+ def test_dir_on_class(self):
+ Season = self.Season
+ self.assertEqual(
+ set(dir(Season)),
+ set(['__class__', '__doc__', '__members__', '__module__',
+ 'SPRING', 'SUMMER', 'AUTUMN', 'WINTER']),
+ )
+
+ def test_dir_on_item(self):
+ Season = self.Season
+ self.assertEqual(
+ set(dir(Season.WINTER)),
+ set(['__class__', '__doc__', '__module__', 'name', 'value']),
+ )
+
+ def test_dir_with_added_behavior(self):
+ class Test(Enum):
+ this = 'that'
+ these = 'those'
+ def wowser(self):
+ return ("Wowser! I'm %s!" % self.name)
+ self.assertEqual(
+ set(dir(Test)),
+ set(['__class__', '__doc__', '__members__', '__module__', 'this', 'these']),
+ )
+ self.assertEqual(
+ set(dir(Test.this)),
+ set(['__class__', '__doc__', '__module__', 'name', 'value', 'wowser']),
+ )
+
+ def test_enum_in_enum_out(self):
+ Season = self.Season
+ self.assertIs(Season(Season.WINTER), Season.WINTER)
+
+ def test_enum_value(self):
+ Season = self.Season
+ self.assertEqual(Season.SPRING.value, 1)
+
+ def test_intenum_value(self):
+ self.assertEqual(IntStooges.CURLY.value, 2)
+
+ def test_enum(self):
+ Season = self.Season
+ lst = list(Season)
+ self.assertEqual(len(lst), len(Season))
+ self.assertEqual(len(Season), 4, Season)
+ self.assertEqual(
+ [Season.SPRING, Season.SUMMER, Season.AUTUMN, Season.WINTER], lst)
+
+ for i, season in enumerate('SPRING SUMMER AUTUMN WINTER'.split(), 1):
+ e = Season(i)
+ self.assertEqual(e, getattr(Season, season))
+ self.assertEqual(e.value, i)
+ self.assertNotEqual(e, i)
+ self.assertEqual(e.name, season)
+ self.assertIn(e, Season)
+ self.assertIs(type(e), Season)
+ self.assertIsInstance(e, Season)
+ self.assertEqual(str(e), 'Season.' + season)
+ self.assertEqual(
+ repr(e),
+ '<Season.{0}: {1}>'.format(season, i),
+ )
+
+ def test_value_name(self):
+ Season = self.Season
+ self.assertEqual(Season.SPRING.name, 'SPRING')
+ self.assertEqual(Season.SPRING.value, 1)
+ with self.assertRaises(AttributeError):
+ Season.SPRING.name = 'invierno'
+ with self.assertRaises(AttributeError):
+ Season.SPRING.value = 2
+
+ def test_changing_member(self):
+ Season = self.Season
+ with self.assertRaises(AttributeError):
+ Season.WINTER = 'really cold'
+
+ def test_attribute_deletion(self):
+ class Season(Enum):
+ SPRING = 1
+ SUMMER = 2
+ AUTUMN = 3
+ WINTER = 4
+
+ def spam(cls):
+ pass
+
+ self.assertTrue(hasattr(Season, 'spam'))
+ del Season.spam
+ self.assertFalse(hasattr(Season, 'spam'))
+
+ with self.assertRaises(AttributeError):
+ del Season.SPRING
+ with self.assertRaises(AttributeError):
+ del Season.DRY
+ with self.assertRaises(AttributeError):
+ del Season.SPRING.name
+
+ def test_invalid_names(self):
+ with self.assertRaises(ValueError):
+ class Wrong(Enum):
+ mro = 9
+ with self.assertRaises(ValueError):
+ class Wrong(Enum):
+ _create_= 11
+ with self.assertRaises(ValueError):
+ class Wrong(Enum):
+ _get_mixins_ = 9
+ with self.assertRaises(ValueError):
+ class Wrong(Enum):
+ _find_new_ = 1
+ with self.assertRaises(ValueError):
+ class Wrong(Enum):
+ _any_name_ = 9
+
+ def test_contains(self):
+ Season = self.Season
+ self.assertIn(Season.AUTUMN, Season)
+ self.assertNotIn(3, Season)
+
+ val = Season(3)
+ self.assertIn(val, Season)
+
+ class OtherEnum(Enum):
+ one = 1; two = 2
+ self.assertNotIn(OtherEnum.two, Season)
+
+ def test_comparisons(self):
+ Season = self.Season
+ with self.assertRaises(TypeError):
+ Season.SPRING < Season.WINTER
+ with self.assertRaises(TypeError):
+ Season.SPRING > 4
+
+ self.assertNotEqual(Season.SPRING, 1)
+
+ class Part(Enum):
+ SPRING = 1
+ CLIP = 2
+ BARREL = 3
+
+ self.assertNotEqual(Season.SPRING, Part.SPRING)
+ with self.assertRaises(TypeError):
+ Season.SPRING < Part.CLIP
+
+ def test_enum_duplicates(self):
+ class Season(Enum):
+ SPRING = 1
+ SUMMER = 2
+ AUTUMN = FALL = 3
+ WINTER = 4
+ ANOTHER_SPRING = 1
+ lst = list(Season)
+ self.assertEqual(
+ lst,
+ [Season.SPRING, Season.SUMMER,
+ Season.AUTUMN, Season.WINTER,
+ ])
+ self.assertIs(Season.FALL, Season.AUTUMN)
+ self.assertEqual(Season.FALL.value, 3)
+ self.assertEqual(Season.AUTUMN.value, 3)
+ self.assertIs(Season(3), Season.AUTUMN)
+ self.assertIs(Season(1), Season.SPRING)
+ self.assertEqual(Season.FALL.name, 'AUTUMN')
+ self.assertEqual(
+ [k for k,v in Season.__members__.items() if v.name != k],
+ ['FALL', 'ANOTHER_SPRING'],
+ )
+
+ def test_duplicate_name(self):
+ with self.assertRaises(TypeError):
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+ red = 4
+
+ with self.assertRaises(TypeError):
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+ def red(self):
+ return 'red'
+
+ with self.assertRaises(TypeError):
+ class Color(Enum):
+ @property
+ def red(self):
+ return 'redder'
+ red = 1
+ green = 2
+ blue = 3
+
+
+ def test_enum_with_value_name(self):
+ class Huh(Enum):
+ name = 1
+ value = 2
+ self.assertEqual(
+ list(Huh),
+ [Huh.name, Huh.value],
+ )
+ self.assertIs(type(Huh.name), Huh)
+ self.assertEqual(Huh.name.name, 'name')
+ self.assertEqual(Huh.name.value, 1)
+
+ def test_format_enum(self):
+ Season = self.Season
+ self.assertEqual('{}'.format(Season.SPRING),
+ '{}'.format(str(Season.SPRING)))
+ self.assertEqual( '{:}'.format(Season.SPRING),
+ '{:}'.format(str(Season.SPRING)))
+ self.assertEqual('{:20}'.format(Season.SPRING),
+ '{:20}'.format(str(Season.SPRING)))
+ self.assertEqual('{:^20}'.format(Season.SPRING),
+ '{:^20}'.format(str(Season.SPRING)))
+ self.assertEqual('{:>20}'.format(Season.SPRING),
+ '{:>20}'.format(str(Season.SPRING)))
+ self.assertEqual('{:<20}'.format(Season.SPRING),
+ '{:<20}'.format(str(Season.SPRING)))
+
+ def test_format_enum_custom(self):
+ class TestFloat(float, Enum):
+ one = 1.0
+ two = 2.0
+ def __format__(self, spec):
+ return 'TestFloat success!'
+ self.assertEqual('{}'.format(TestFloat.one), 'TestFloat success!')
+
+ def assertFormatIsValue(self, spec, member):
+ self.assertEqual(spec.format(member), spec.format(member.value))
+
+ def test_format_enum_date(self):
+ Holiday = self.Holiday
+ self.assertFormatIsValue('{}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:20}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:^20}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:>20}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:<20}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:%Y %m}', Holiday.IDES_OF_MARCH)
+ self.assertFormatIsValue('{:%Y %m %M:00}', Holiday.IDES_OF_MARCH)
+
+ def test_format_enum_float(self):
+ Konstants = self.Konstants
+ self.assertFormatIsValue('{}', Konstants.TAU)
+ self.assertFormatIsValue('{:}', Konstants.TAU)
+ self.assertFormatIsValue('{:20}', Konstants.TAU)
+ self.assertFormatIsValue('{:^20}', Konstants.TAU)
+ self.assertFormatIsValue('{:>20}', Konstants.TAU)
+ self.assertFormatIsValue('{:<20}', Konstants.TAU)
+ self.assertFormatIsValue('{:n}', Konstants.TAU)
+ self.assertFormatIsValue('{:5.2}', Konstants.TAU)
+ self.assertFormatIsValue('{:f}', Konstants.TAU)
+
+ def test_format_enum_int(self):
+ Grades = self.Grades
+ self.assertFormatIsValue('{}', Grades.C)
+ self.assertFormatIsValue('{:}', Grades.C)
+ self.assertFormatIsValue('{:20}', Grades.C)
+ self.assertFormatIsValue('{:^20}', Grades.C)
+ self.assertFormatIsValue('{:>20}', Grades.C)
+ self.assertFormatIsValue('{:<20}', Grades.C)
+ self.assertFormatIsValue('{:+}', Grades.C)
+ self.assertFormatIsValue('{:08X}', Grades.C)
+ self.assertFormatIsValue('{:b}', Grades.C)
+
+ def test_format_enum_str(self):
+ Directional = self.Directional
+ self.assertFormatIsValue('{}', Directional.WEST)
+ self.assertFormatIsValue('{:}', Directional.WEST)
+ self.assertFormatIsValue('{:20}', Directional.WEST)
+ self.assertFormatIsValue('{:^20}', Directional.WEST)
+ self.assertFormatIsValue('{:>20}', Directional.WEST)
+ self.assertFormatIsValue('{:<20}', Directional.WEST)
+
+ def test_hash(self):
+ Season = self.Season
+ dates = {}
+ dates[Season.WINTER] = '1225'
+ dates[Season.SPRING] = '0315'
+ dates[Season.SUMMER] = '0704'
+ dates[Season.AUTUMN] = '1031'
+ self.assertEqual(dates[Season.AUTUMN], '1031')
+
+ def test_intenum_from_scratch(self):
+ class phy(int, Enum):
+ pi = 3
+ tau = 2 * pi
+ self.assertTrue(phy.pi < phy.tau)
+
+ def test_intenum_inherited(self):
+ class IntEnum(int, Enum):
+ pass
+ class phy(IntEnum):
+ pi = 3
+ tau = 2 * pi
+ self.assertTrue(phy.pi < phy.tau)
+
+ def test_floatenum_from_scratch(self):
+ class phy(float, Enum):
+ pi = 3.1415926
+ tau = 2 * pi
+ self.assertTrue(phy.pi < phy.tau)
+
+ def test_floatenum_inherited(self):
+ class FloatEnum(float, Enum):
+ pass
+ class phy(FloatEnum):
+ pi = 3.1415926
+ tau = 2 * pi
+ self.assertTrue(phy.pi < phy.tau)
+
+ def test_strenum_from_scratch(self):
+ class phy(str, Enum):
+ pi = 'Pi'
+ tau = 'Tau'
+ self.assertTrue(phy.pi < phy.tau)
+
+ def test_strenum_inherited(self):
+ class StrEnum(str, Enum):
+ pass
+ class phy(StrEnum):
+ pi = 'Pi'
+ tau = 'Tau'
+ self.assertTrue(phy.pi < phy.tau)
+
+
+ def test_intenum(self):
+ class WeekDay(IntEnum):
+ SUNDAY = 1
+ MONDAY = 2
+ TUESDAY = 3
+ WEDNESDAY = 4
+ THURSDAY = 5
+ FRIDAY = 6
+ SATURDAY = 7
+
+ self.assertEqual(['a', 'b', 'c'][WeekDay.MONDAY], 'c')
+ self.assertEqual([i for i in range(WeekDay.TUESDAY)], [0, 1, 2])
+
+ lst = list(WeekDay)
+ self.assertEqual(len(lst), len(WeekDay))
+ self.assertEqual(len(WeekDay), 7)
+ target = 'SUNDAY MONDAY TUESDAY WEDNESDAY THURSDAY FRIDAY SATURDAY'
+ target = target.split()
+ for i, weekday in enumerate(target, 1):
+ e = WeekDay(i)
+ self.assertEqual(e, i)
+ self.assertEqual(int(e), i)
+ self.assertEqual(e.name, weekday)
+ self.assertIn(e, WeekDay)
+ self.assertEqual(lst.index(e)+1, i)
+ self.assertTrue(0 < e < 8)
+ self.assertIs(type(e), WeekDay)
+ self.assertIsInstance(e, int)
+ self.assertIsInstance(e, Enum)
+
+ def test_intenum_duplicates(self):
+ class WeekDay(IntEnum):
+ SUNDAY = 1
+ MONDAY = 2
+ TUESDAY = TEUSDAY = 3
+ WEDNESDAY = 4
+ THURSDAY = 5
+ FRIDAY = 6
+ SATURDAY = 7
+ self.assertIs(WeekDay.TEUSDAY, WeekDay.TUESDAY)
+ self.assertEqual(WeekDay(3).name, 'TUESDAY')
+ self.assertEqual([k for k,v in WeekDay.__members__.items()
+ if v.name != k], ['TEUSDAY', ])
+
+ def test_pickle_enum(self):
+ if isinstance(Stooges, Exception):
+ raise Stooges
+ test_pickle_dump_load(self.assertIs, Stooges.CURLY)
+ test_pickle_dump_load(self.assertIs, Stooges)
+
+ def test_pickle_int(self):
+ if isinstance(IntStooges, Exception):
+ raise IntStooges
+ test_pickle_dump_load(self.assertIs, IntStooges.CURLY)
+ test_pickle_dump_load(self.assertIs, IntStooges)
+
+ def test_pickle_float(self):
+ if isinstance(FloatStooges, Exception):
+ raise FloatStooges
+ test_pickle_dump_load(self.assertIs, FloatStooges.CURLY)
+ test_pickle_dump_load(self.assertIs, FloatStooges)
+
+ def test_pickle_enum_function(self):
+ if isinstance(Answer, Exception):
+ raise Answer
+ test_pickle_dump_load(self.assertIs, Answer.him)
+ test_pickle_dump_load(self.assertIs, Answer)
+
+ def test_pickle_enum_function_with_module(self):
+ if isinstance(Question, Exception):
+ raise Question
+ test_pickle_dump_load(self.assertIs, Question.who)
+ test_pickle_dump_load(self.assertIs, Question)
+
+ def test_enum_function_with_qualname(self):
+ if isinstance(Theory, Exception):
+ raise Theory
+ self.assertEqual(Theory.__qualname__, 'spanish_inquisition')
+
+ def test_class_nested_enum_and_pickle_protocol_four(self):
+ # would normally just have this directly in the class namespace
+ class NestedEnum(Enum):
+ twigs = 'common'
+ shiny = 'rare'
+
+ self.__class__.NestedEnum = NestedEnum
+ self.NestedEnum.__qualname__ = '%s.NestedEnum' % self.__class__.__name__
+ test_pickle_exception(
+ self.assertRaises, PicklingError, self.NestedEnum.twigs,
+ protocol=(0, 3))
+ test_pickle_dump_load(self.assertIs, self.NestedEnum.twigs,
+ protocol=(4, HIGHEST_PROTOCOL))
+
+ def test_exploding_pickle(self):
+ BadPickle = Enum(
+ 'BadPickle', 'dill sweet bread-n-butter', module=__name__)
+ globals()['BadPickle'] = BadPickle
+ # now break BadPickle to test exception raising
+ enum._make_class_unpicklable(BadPickle)
+ test_pickle_exception(self.assertRaises, TypeError, BadPickle.dill)
+ test_pickle_exception(self.assertRaises, PicklingError, BadPickle)
+
+ def test_string_enum(self):
+ class SkillLevel(str, Enum):
+ master = 'what is the sound of one hand clapping?'
+ journeyman = 'why did the chicken cross the road?'
+ apprentice = 'knock, knock!'
+ self.assertEqual(SkillLevel.apprentice, 'knock, knock!')
+
+ def test_getattr_getitem(self):
+ class Period(Enum):
+ morning = 1
+ noon = 2
+ evening = 3
+ night = 4
+ self.assertIs(Period(2), Period.noon)
+ self.assertIs(getattr(Period, 'night'), Period.night)
+ self.assertIs(Period['morning'], Period.morning)
+
+ def test_getattr_dunder(self):
+ Season = self.Season
+ self.assertTrue(getattr(Season, '__eq__'))
+
+ def test_iteration_order(self):
+ class Season(Enum):
+ SUMMER = 2
+ WINTER = 4
+ AUTUMN = 3
+ SPRING = 1
+ self.assertEqual(
+ list(Season),
+ [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING],
+ )
+
+ def test_reversed_iteration_order(self):
+ self.assertEqual(
+ list(reversed(self.Season)),
+ [self.Season.WINTER, self.Season.AUTUMN, self.Season.SUMMER,
+ self.Season.SPRING]
+ )
+
+ def test_programatic_function_string(self):
+ SummerMonth = Enum('SummerMonth', 'june july august')
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(int(e.value), i)
+ self.assertNotEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_programatic_function_string_list(self):
+ SummerMonth = Enum('SummerMonth', ['june', 'july', 'august'])
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(int(e.value), i)
+ self.assertNotEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_programatic_function_iterable(self):
+ SummerMonth = Enum(
+ 'SummerMonth',
+ (('june', 1), ('july', 2), ('august', 3))
+ )
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(int(e.value), i)
+ self.assertNotEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_programatic_function_from_dict(self):
+ SummerMonth = Enum(
+ 'SummerMonth',
+ OrderedDict((('june', 1), ('july', 2), ('august', 3)))
+ )
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(int(e.value), i)
+ self.assertNotEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_programatic_function_type(self):
+ SummerMonth = Enum('SummerMonth', 'june july august', type=int)
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_programatic_function_type_from_subclass(self):
+ SummerMonth = IntEnum('SummerMonth', 'june july august')
+ lst = list(SummerMonth)
+ self.assertEqual(len(lst), len(SummerMonth))
+ self.assertEqual(len(SummerMonth), 3, SummerMonth)
+ self.assertEqual(
+ [SummerMonth.june, SummerMonth.july, SummerMonth.august],
+ lst,
+ )
+ for i, month in enumerate('june july august'.split(), 1):
+ e = SummerMonth(i)
+ self.assertEqual(e, i)
+ self.assertEqual(e.name, month)
+ self.assertIn(e, SummerMonth)
+ self.assertIs(type(e), SummerMonth)
+
+ def test_subclassing(self):
+ if isinstance(Name, Exception):
+ raise Name
+ self.assertEqual(Name.BDFL, 'Guido van Rossum')
+ self.assertTrue(Name.BDFL, Name('Guido van Rossum'))
+ self.assertIs(Name.BDFL, getattr(Name, 'BDFL'))
+ test_pickle_dump_load(self.assertIs, Name.BDFL)
+
+ def test_extending(self):
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+ with self.assertRaises(TypeError):
+ class MoreColor(Color):
+ cyan = 4
+ magenta = 5
+ yellow = 6
+
+ def test_exclude_methods(self):
+ class whatever(Enum):
+ this = 'that'
+ these = 'those'
+ def really(self):
+ return 'no, not %s' % self.value
+ self.assertIsNot(type(whatever.really), whatever)
+ self.assertEqual(whatever.this.really(), 'no, not that')
+
+ def test_wrong_inheritance_order(self):
+ with self.assertRaises(TypeError):
+ class Wrong(Enum, str):
+ NotHere = 'error before this point'
+
+ def test_intenum_transitivity(self):
+ class number(IntEnum):
+ one = 1
+ two = 2
+ three = 3
+ class numero(IntEnum):
+ uno = 1
+ dos = 2
+ tres = 3
+ self.assertEqual(number.one, numero.uno)
+ self.assertEqual(number.two, numero.dos)
+ self.assertEqual(number.three, numero.tres)
+
+ def test_wrong_enum_in_call(self):
+ class Monochrome(Enum):
+ black = 0
+ white = 1
+ class Gender(Enum):
+ male = 0
+ female = 1
+ self.assertRaises(ValueError, Monochrome, Gender.male)
+
+ def test_wrong_enum_in_mixed_call(self):
+ class Monochrome(IntEnum):
+ black = 0
+ white = 1
+ class Gender(Enum):
+ male = 0
+ female = 1
+ self.assertRaises(ValueError, Monochrome, Gender.male)
+
+ def test_mixed_enum_in_call_1(self):
+ class Monochrome(IntEnum):
+ black = 0
+ white = 1
+ class Gender(IntEnum):
+ male = 0
+ female = 1
+ self.assertIs(Monochrome(Gender.female), Monochrome.white)
+
+ def test_mixed_enum_in_call_2(self):
+ class Monochrome(Enum):
+ black = 0
+ white = 1
+ class Gender(IntEnum):
+ male = 0
+ female = 1
+ self.assertIs(Monochrome(Gender.male), Monochrome.black)
+
+ def test_flufl_enum(self):
+ class Fluflnum(Enum):
+ def __int__(self):
+ return int(self.value)
+ class MailManOptions(Fluflnum):
+ option1 = 1
+ option2 = 2
+ option3 = 3
+ self.assertEqual(int(MailManOptions.option1), 1)
+
+ def test_introspection(self):
+ class Number(IntEnum):
+ one = 100
+ two = 200
+ self.assertIs(Number.one._member_type_, int)
+ self.assertIs(Number._member_type_, int)
+ class String(str, Enum):
+ yarn = 'soft'
+ rope = 'rough'
+ wire = 'hard'
+ self.assertIs(String.yarn._member_type_, str)
+ self.assertIs(String._member_type_, str)
+ class Plain(Enum):
+ vanilla = 'white'
+ one = 1
+ self.assertIs(Plain.vanilla._member_type_, object)
+ self.assertIs(Plain._member_type_, object)
+
+ def test_no_such_enum_member(self):
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+ with self.assertRaises(ValueError):
+ Color(4)
+ with self.assertRaises(KeyError):
+ Color['chartreuse']
+
+ def test_new_repr(self):
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+ def __repr__(self):
+ return "don't you just love shades of %s?" % self.name
+ self.assertEqual(
+ repr(Color.blue),
+ "don't you just love shades of blue?",
+ )
+
+ def test_inherited_repr(self):
+ class MyEnum(Enum):
+ def __repr__(self):
+ return "My name is %s." % self.name
+ class MyIntEnum(int, MyEnum):
+ this = 1
+ that = 2
+ theother = 3
+ self.assertEqual(repr(MyIntEnum.that), "My name is that.")
+
+ def test_multiple_mixin_mro(self):
+ class auto_enum(type(Enum)):
+ def __new__(metacls, cls, bases, classdict):
+ temp = type(classdict)()
+ names = set(classdict._member_names)
+ i = 0
+ for k in classdict._member_names:
+ v = classdict[k]
+ if v is Ellipsis:
+ v = i
+ else:
+ i = v
+ i += 1
+ temp[k] = v
+ for k, v in classdict.items():
+ if k not in names:
+ temp[k] = v
+ return super(auto_enum, metacls).__new__(
+ metacls, cls, bases, temp)
+
+ class AutoNumberedEnum(Enum, metaclass=auto_enum):
+ pass
+
+ class AutoIntEnum(IntEnum, metaclass=auto_enum):
+ pass
+
+ class TestAutoNumber(AutoNumberedEnum):
+ a = ...
+ b = 3
+ c = ...
+
+ class TestAutoInt(AutoIntEnum):
+ a = ...
+ b = 3
+ c = ...
+
+ def test_subclasses_with_getnewargs(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt' # needed for pickle protocol 4
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ def __getnewargs__(self):
+ return self._args
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI' # needed for pickle protocol 4
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ test_pickle_dump_load(self.assertEqual, NI5, 5)
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_dump_load(self.assertIs, NEI.y)
+ test_pickle_dump_load(self.assertIs, NEI)
+
+ def test_subclasses_with_getnewargs_ex(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt' # needed for pickle protocol 4
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ def __getnewargs_ex__(self):
+ return self._args, {}
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI' # needed for pickle protocol 4
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ test_pickle_dump_load(self.assertEqual, NI5, 5, protocol=(4, 4))
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_dump_load(self.assertIs, NEI.y, protocol=(4, 4))
+ test_pickle_dump_load(self.assertIs, NEI)
+
+ def test_subclasses_with_reduce(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt' # needed for pickle protocol 4
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ def __reduce__(self):
+ return self.__class__, self._args
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI' # needed for pickle protocol 4
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ test_pickle_dump_load(self.assertEqual, NI5, 5)
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_dump_load(self.assertIs, NEI.y)
+ test_pickle_dump_load(self.assertIs, NEI)
+
+ def test_subclasses_with_reduce_ex(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt' # needed for pickle protocol 4
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ def __reduce_ex__(self, proto):
+ return self.__class__, self._args
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI' # needed for pickle protocol 4
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ test_pickle_dump_load(self.assertEqual, NI5, 5)
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_dump_load(self.assertIs, NEI.y)
+ test_pickle_dump_load(self.assertIs, NEI)
+
+ def test_subclasses_without_direct_pickle_support(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt'
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI'
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_exception(self.assertRaises, TypeError, NEI.x)
+ test_pickle_exception(self.assertRaises, PicklingError, NEI)
+
+ def test_subclasses_without_direct_pickle_support_using_name(self):
+ class NamedInt(int):
+ __qualname__ = 'NamedInt'
+ def __new__(cls, *args):
+ _args = args
+ name, *args = args
+ if len(args) == 0:
+ raise TypeError("name and value must be specified")
+ self = int.__new__(cls, *args)
+ self._intname = name
+ self._args = _args
+ return self
+ @property
+ def __name__(self):
+ return self._intname
+ def __repr__(self):
+ # repr() is updated to include the name and type info
+ return "{}({!r}, {})".format(type(self).__name__,
+ self.__name__,
+ int.__repr__(self))
+ def __str__(self):
+ # str() is unchanged, even if it relies on the repr() fallback
+ base = int
+ base_str = base.__str__
+ if base_str.__objclass__ is object:
+ return base.__repr__(self)
+ return base_str(self)
+ # for simplicity, we only define one operator that
+ # propagates expressions
+ def __add__(self, other):
+ temp = int(self) + int( other)
+ if isinstance(self, NamedInt) and isinstance(other, NamedInt):
+ return NamedInt(
+ '({0} + {1})'.format(self.__name__, other.__name__),
+ temp )
+ else:
+ return temp
+
+ class NEI(NamedInt, Enum):
+ __qualname__ = 'NEI'
+ x = ('the-x', 1)
+ y = ('the-y', 2)
+ def __reduce_ex__(self, proto):
+ return getattr, (self.__class__, self._name_)
+
+ self.assertIs(NEI.__new__, Enum.__new__)
+ self.assertEqual(repr(NEI.x + NEI.y), "NamedInt('(the-x + the-y)', 3)")
+ globals()['NamedInt'] = NamedInt
+ globals()['NEI'] = NEI
+ NI5 = NamedInt('test', 5)
+ self.assertEqual(NI5, 5)
+ self.assertEqual(NEI.y.value, 2)
+ test_pickle_dump_load(self.assertIs, NEI.y)
+ test_pickle_dump_load(self.assertIs, NEI)
+
+ def test_tuple_subclass(self):
+ class SomeTuple(tuple, Enum):
+ __qualname__ = 'SomeTuple' # needed for pickle protocol 4
+ first = (1, 'for the money')
+ second = (2, 'for the show')
+ third = (3, 'for the music')
+ self.assertIs(type(SomeTuple.first), SomeTuple)
+ self.assertIsInstance(SomeTuple.second, tuple)
+ self.assertEqual(SomeTuple.third, (3, 'for the music'))
+ globals()['SomeTuple'] = SomeTuple
+ test_pickle_dump_load(self.assertIs, SomeTuple.first)
+
+ def test_duplicate_values_give_unique_enum_items(self):
+ class AutoNumber(Enum):
+ first = ()
+ second = ()
+ third = ()
+ def __new__(cls):
+ value = len(cls.__members__) + 1
+ obj = object.__new__(cls)
+ obj._value_ = value
+ return obj
+ def __int__(self):
+ return int(self._value_)
+ self.assertEqual(
+ list(AutoNumber),
+ [AutoNumber.first, AutoNumber.second, AutoNumber.third],
+ )
+ self.assertEqual(int(AutoNumber.second), 2)
+ self.assertEqual(AutoNumber.third.value, 3)
+ self.assertIs(AutoNumber(1), AutoNumber.first)
+
+ def test_inherited_new_from_enhanced_enum(self):
+ class AutoNumber(Enum):
+ def __new__(cls):
+ value = len(cls.__members__) + 1
+ obj = object.__new__(cls)
+ obj._value_ = value
+ return obj
+ def __int__(self):
+ return int(self._value_)
+ class Color(AutoNumber):
+ red = ()
+ green = ()
+ blue = ()
+ self.assertEqual(list(Color), [Color.red, Color.green, Color.blue])
+ self.assertEqual(list(map(int, Color)), [1, 2, 3])
+
+ def test_inherited_new_from_mixed_enum(self):
+ class AutoNumber(IntEnum):
+ def __new__(cls):
+ value = len(cls.__members__) + 1
+ obj = int.__new__(cls, value)
+ obj._value_ = value
+ return obj
+ class Color(AutoNumber):
+ red = ()
+ green = ()
+ blue = ()
+ self.assertEqual(list(Color), [Color.red, Color.green, Color.blue])
+ self.assertEqual(list(map(int, Color)), [1, 2, 3])
+
+ def test_equality(self):
+ class AlwaysEqual:
+ def __eq__(self, other):
+ return True
+ class OrdinaryEnum(Enum):
+ a = 1
+ self.assertEqual(AlwaysEqual(), OrdinaryEnum.a)
+ self.assertEqual(OrdinaryEnum.a, AlwaysEqual())
+
+ def test_ordered_mixin(self):
+ class OrderedEnum(Enum):
+ def __ge__(self, other):
+ if self.__class__ is other.__class__:
+ return self._value_ >= other._value_
+ return NotImplemented
+ def __gt__(self, other):
+ if self.__class__ is other.__class__:
+ return self._value_ > other._value_
+ return NotImplemented
+ def __le__(self, other):
+ if self.__class__ is other.__class__:
+ return self._value_ <= other._value_
+ return NotImplemented
+ def __lt__(self, other):
+ if self.__class__ is other.__class__:
+ return self._value_ < other._value_
+ return NotImplemented
+ class Grade(OrderedEnum):
+ A = 5
+ B = 4
+ C = 3
+ D = 2
+ F = 1
+ self.assertGreater(Grade.A, Grade.B)
+ self.assertLessEqual(Grade.F, Grade.C)
+ self.assertLess(Grade.D, Grade.A)
+ self.assertGreaterEqual(Grade.B, Grade.B)
+ self.assertEqual(Grade.B, Grade.B)
+ self.assertNotEqual(Grade.C, Grade.D)
+
+ def test_extending2(self):
+ class Shade(Enum):
+ def shade(self):
+ print(self.name)
+ class Color(Shade):
+ red = 1
+ green = 2
+ blue = 3
+ with self.assertRaises(TypeError):
+ class MoreColor(Color):
+ cyan = 4
+ magenta = 5
+ yellow = 6
+
+ def test_extending3(self):
+ class Shade(Enum):
+ def shade(self):
+ return self.name
+ class Color(Shade):
+ def hex(self):
+ return '%s hexlified!' % self.value
+ class MoreColor(Color):
+ cyan = 4
+ magenta = 5
+ yellow = 6
+ self.assertEqual(MoreColor.magenta.hex(), '5 hexlified!')
+
+
+ def test_no_duplicates(self):
+ class UniqueEnum(Enum):
+ def __init__(self, *args):
+ cls = self.__class__
+ if any(self.value == e.value for e in cls):
+ a = self.name
+ e = cls(self.value).name
+ raise ValueError(
+ "aliases not allowed in UniqueEnum: %r --> %r"
+ % (a, e)
+ )
+ class Color(UniqueEnum):
+ red = 1
+ green = 2
+ blue = 3
+ with self.assertRaises(ValueError):
+ class Color(UniqueEnum):
+ red = 1
+ green = 2
+ blue = 3
+ grene = 2
+
+ def test_init(self):
+ class Planet(Enum):
+ MERCURY = (3.303e+23, 2.4397e6)
+ VENUS = (4.869e+24, 6.0518e6)
+ EARTH = (5.976e+24, 6.37814e6)
+ MARS = (6.421e+23, 3.3972e6)
+ JUPITER = (1.9e+27, 7.1492e7)
+ SATURN = (5.688e+26, 6.0268e7)
+ URANUS = (8.686e+25, 2.5559e7)
+ NEPTUNE = (1.024e+26, 2.4746e7)
+ def __init__(self, mass, radius):
+ self.mass = mass # in kilograms
+ self.radius = radius # in meters
+ @property
+ def surface_gravity(self):
+ # universal gravitational constant (m3 kg-1 s-2)
+ G = 6.67300E-11
+ return G * self.mass / (self.radius * self.radius)
+ self.assertEqual(round(Planet.EARTH.surface_gravity, 2), 9.80)
+ self.assertEqual(Planet.EARTH.value, (5.976e+24, 6.37814e6))
+
+ def test_nonhash_value(self):
+ class AutoNumberInAList(Enum):
+ def __new__(cls):
+ value = [len(cls.__members__) + 1]
+ obj = object.__new__(cls)
+ obj._value_ = value
+ return obj
+ class ColorInAList(AutoNumberInAList):
+ red = ()
+ green = ()
+ blue = ()
+ self.assertEqual(list(ColorInAList), [ColorInAList.red, ColorInAList.green, ColorInAList.blue])
+ for enum, value in zip(ColorInAList, range(3)):
+ value += 1
+ self.assertEqual(enum.value, [value])
+ self.assertIs(ColorInAList([value]), enum)
+
+ def test_conflicting_types_resolved_in_new(self):
+ class LabelledIntEnum(int, Enum):
+ def __new__(cls, *args):
+ value, label = args
+ obj = int.__new__(cls, value)
+ obj.label = label
+ obj._value_ = value
+ return obj
+
+ class LabelledList(LabelledIntEnum):
+ unprocessed = (1, "Unprocessed")
+ payment_complete = (2, "Payment Complete")
+
+ self.assertEqual(list(LabelledList), [LabelledList.unprocessed, LabelledList.payment_complete])
+ self.assertEqual(LabelledList.unprocessed, 1)
+ self.assertEqual(LabelledList(1), LabelledList.unprocessed)
+
+
+class TestUnique(unittest.TestCase):
+
+ def test_unique_clean(self):
+ @unique
+ class Clean(Enum):
+ one = 1
+ two = 'dos'
+ tres = 4.0
+ @unique
+ class Cleaner(IntEnum):
+ single = 1
+ double = 2
+ triple = 3
+
+ def test_unique_dirty(self):
+ with self.assertRaisesRegex(ValueError, 'tres.*one'):
+ @unique
+ class Dirty(Enum):
+ one = 1
+ two = 'dos'
+ tres = 1
+ with self.assertRaisesRegex(
+ ValueError,
+ 'double.*single.*turkey.*triple',
+ ):
+ @unique
+ class Dirtier(IntEnum):
+ single = 1
+ double = 1
+ triple = 3
+ turkey = 3
+
+
+expected_help_output = """
+Help on class Color in module %s:
+
+class Color(enum.Enum)
+ | Method resolution order:
+ | Color
+ | enum.Enum
+ | builtins.object
+ |\x20\x20
+ | Data and other attributes defined here:
+ |\x20\x20
+ | blue = <Color.blue: 3>
+ |\x20\x20
+ | green = <Color.green: 2>
+ |\x20\x20
+ | red = <Color.red: 1>
+ |\x20\x20
+ | ----------------------------------------------------------------------
+ | Data descriptors inherited from enum.Enum:
+ |\x20\x20
+ | name
+ | The name of the Enum member.
+ |\x20\x20
+ | value
+ | The value of the Enum member.
+ |\x20\x20
+ | ----------------------------------------------------------------------
+ | Data descriptors inherited from enum.EnumMeta:
+ |\x20\x20
+ | __members__
+ | Returns a mapping of member name->value.
+ |\x20\x20\x20\x20\x20\x20
+ | This mapping lists all enum members, including aliases. Note that this
+ | is a read-only view of the internal mapping.
+""".strip()
+
+class TestStdLib(unittest.TestCase):
+
+ class Color(Enum):
+ red = 1
+ green = 2
+ blue = 3
+
+ def test_pydoc(self):
+ # indirectly test __objclass__
+ expected_text = expected_help_output % __name__
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(self.Color)
+ result = output.getvalue().strip()
+ if result != expected_text:
+ print_diffs(expected_text, result)
+ self.fail("outputs are not equal, see diff above")
+
+ def test_inspect_getmembers(self):
+ values = dict((
+ ('__class__', EnumMeta),
+ ('__doc__', None),
+ ('__members__', self.Color.__members__),
+ ('__module__', __name__),
+ ('blue', self.Color.blue),
+ ('green', self.Color.green),
+ ('name', Enum.__dict__['name']),
+ ('red', self.Color.red),
+ ('value', Enum.__dict__['value']),
+ ))
+ result = dict(inspect.getmembers(self.Color))
+ self.assertEqual(values.keys(), result.keys())
+ failed = False
+ for k in values.keys():
+ if result[k] != values[k]:
+ print()
+ print('\n%s\n key: %s\n result: %s\nexpected: %s\n%s\n' %
+ ('=' * 75, k, result[k], values[k], '=' * 75), sep='')
+ failed = True
+ if failed:
+ self.fail("result does not equal expected, see print above")
+
+ def test_inspect_classify_class_attrs(self):
+ # indirectly test __objclass__
+ from inspect import Attribute
+ values = [
+ Attribute(name='__class__', kind='data',
+ defining_class=object, object=EnumMeta),
+ Attribute(name='__doc__', kind='data',
+ defining_class=self.Color, object=None),
+ Attribute(name='__members__', kind='property',
+ defining_class=EnumMeta, object=EnumMeta.__members__),
+ Attribute(name='__module__', kind='data',
+ defining_class=self.Color, object=__name__),
+ Attribute(name='blue', kind='data',
+ defining_class=self.Color, object=self.Color.blue),
+ Attribute(name='green', kind='data',
+ defining_class=self.Color, object=self.Color.green),
+ Attribute(name='red', kind='data',
+ defining_class=self.Color, object=self.Color.red),
+ Attribute(name='name', kind='data',
+ defining_class=Enum, object=Enum.__dict__['name']),
+ Attribute(name='value', kind='data',
+ defining_class=Enum, object=Enum.__dict__['value']),
+ ]
+ values.sort(key=lambda item: item.name)
+ result = list(inspect.classify_class_attrs(self.Color))
+ result.sort(key=lambda item: item.name)
+ failed = False
+ for v, r in zip(values, result):
+ if r != v:
+ print('\n%s\n%s\n%s\n%s\n' % ('=' * 75, r, v, '=' * 75), sep='')
+ failed = True
+ if failed:
+ self.fail("result does not equal expected, see print above")
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py
index 4af217b..8742afc 100644
--- a/Lib/test/test_enumerate.py
+++ b/Lib/test/test_enumerate.py
@@ -1,4 +1,5 @@
import unittest
+import operator
import sys
import pickle
@@ -168,15 +169,12 @@ class TestReversed(unittest.TestCase, PickleTest):
x = range(1)
self.assertEqual(type(reversed(x)), type(iter(x)))
- @support.cpython_only
def test_len(self):
- # This is an implementation detail, not an interface requirement
- from test.test_iterlen import len
for s in ('hello', tuple('hello'), list('hello'), range(5)):
- self.assertEqual(len(reversed(s)), len(s))
+ self.assertEqual(operator.length_hint(reversed(s)), len(s))
r = reversed(s)
list(r)
- self.assertEqual(len(r), 0)
+ self.assertEqual(operator.length_hint(r), 0)
class SeqWithWeirdLen:
called = False
def __len__(self):
@@ -187,7 +185,7 @@ class TestReversed(unittest.TestCase, PickleTest):
def __getitem__(self, index):
return index
r = reversed(SeqWithWeirdLen())
- self.assertRaises(ZeroDivisionError, len, r)
+ self.assertRaises(ZeroDivisionError, operator.length_hint, r)
def test_gc(self):
diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py
index 871efb2..b37f033 100644
--- a/Lib/test/test_epoll.py
+++ b/Lib/test/test_epoll.py
@@ -21,10 +21,11 @@
"""
Tests for epoll wrapper.
"""
-import socket
import errno
-import time
+import os
import select
+import socket
+import time
import unittest
from test import support
@@ -33,7 +34,7 @@ if not hasattr(select, "epoll"):
try:
select.epoll()
-except IOError as e:
+except OSError as e:
if e.errno == errno.ENOSYS:
raise unittest.SkipTest("kernel doesn't support epoll()")
raise
@@ -55,7 +56,7 @@ class TestEPoll(unittest.TestCase):
client.setblocking(False)
try:
client.connect(('127.0.0.1', self.serverSocket.getsockname()[1]))
- except socket.error as e:
+ except OSError as e:
self.assertEqual(e.args[0], errno.EINPROGRESS)
else:
raise AssertionError("Connect should have raised EINPROGRESS")
@@ -86,6 +87,13 @@ class TestEPoll(unittest.TestCase):
self.assertRaises(TypeError, select.epoll, ['foo'])
self.assertRaises(TypeError, select.epoll, {})
+ def test_context_manager(self):
+ with select.epoll(16) as ep:
+ self.assertGreater(ep.fileno(), 0)
+ self.assertFalse(ep.closed)
+ self.assertTrue(ep.closed)
+ self.assertRaises(ValueError, ep.fileno)
+
def test_add(self):
server, client = self._connected_pair()
@@ -114,12 +122,12 @@ class TestEPoll(unittest.TestCase):
# ValueError: file descriptor cannot be a negative integer (-1)
self.assertRaises(ValueError, ep.register, -1,
select.EPOLLIN | select.EPOLLOUT)
- # IOError: [Errno 9] Bad file descriptor
- self.assertRaises(IOError, ep.register, 10000,
+ # OSError: [Errno 9] Bad file descriptor
+ self.assertRaises(OSError, ep.register, 10000,
select.EPOLLIN | select.EPOLLOUT)
# registering twice also raises an exception
ep.register(server, select.EPOLLIN | select.EPOLLOUT)
- self.assertRaises(IOError, ep.register, server,
+ self.assertRaises(OSError, ep.register, server,
select.EPOLLIN | select.EPOLLOUT)
finally:
ep.close()
@@ -141,7 +149,7 @@ class TestEPoll(unittest.TestCase):
ep.close()
try:
ep2.poll(1, 4)
- except IOError as e:
+ except OSError as e:
self.assertEqual(e.args[0], errno.EBADF, e)
else:
self.fail("epoll on closed fd didn't raise EBADF")
@@ -155,9 +163,9 @@ class TestEPoll(unittest.TestCase):
ep.register(client.fileno(),
select.EPOLLIN | select.EPOLLOUT | select.EPOLLET)
- now = time.time()
+ now = time.monotonic()
events = ep.poll(1, 4)
- then = time.time()
+ then = time.monotonic()
self.assertFalse(then - now > 0.1, then - now)
events.sort()
@@ -166,19 +174,16 @@ class TestEPoll(unittest.TestCase):
expected.sort()
self.assertEqual(events, expected)
- self.assertFalse(then - now > 0.01, then - now)
- now = time.time()
events = ep.poll(timeout=2.1, maxevents=4)
- then = time.time()
self.assertFalse(events)
client.send(b"Hello!")
server.send(b"world!!!")
- now = time.time()
+ now = time.monotonic()
events = ep.poll(1, 4)
- then = time.time()
+ then = time.monotonic()
self.assertFalse(then - now > 0.01)
events.sort()
@@ -190,9 +195,9 @@ class TestEPoll(unittest.TestCase):
ep.unregister(client.fileno())
ep.modify(server.fileno(), select.EPOLLOUT)
- now = time.time()
+ now = time.monotonic()
events = ep.poll(1, 4)
- then = time.time()
+ then = time.monotonic()
self.assertFalse(then - now > 0.01)
expected = [(server.fileno(), select.EPOLLOUT)]
@@ -209,14 +214,43 @@ class TestEPoll(unittest.TestCase):
ep = select.epoll(16)
ep.register(server)
- now = time.time()
+ now = time.monotonic()
events = ep.poll(1, 4)
- then = time.time()
+ then = time.monotonic()
self.assertFalse(then - now > 0.01)
server.close()
ep.unregister(fd)
+ def test_close(self):
+ open_file = open(__file__, "rb")
+ self.addCleanup(open_file.close)
+ fd = open_file.fileno()
+ epoll = select.epoll()
+
+ # test fileno() method and closed attribute
+ self.assertIsInstance(epoll.fileno(), int)
+ self.assertFalse(epoll.closed)
+
+ # test close()
+ epoll.close()
+ self.assertTrue(epoll.closed)
+ self.assertRaises(ValueError, epoll.fileno)
+
+ # close() can be called more than once
+ epoll.close()
+
+ # operations must fail with ValueError("I/O operation on closed ...")
+ self.assertRaises(ValueError, epoll.modify, fd, select.EPOLLIN)
+ self.assertRaises(ValueError, epoll.poll, 1.0)
+ self.assertRaises(ValueError, epoll.register, fd, select.EPOLLIN)
+ self.assertRaises(ValueError, epoll.unregister, fd)
+
+ def test_fd_non_inheritable(self):
+ epoll = select.epoll()
+ self.addCleanup(epoll.close)
+ self.assertEqual(os.get_inheritable(epoll.fileno()), False)
+
def test_main():
support.run_unittest(TestEPoll)
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 8d11d90..62a2e32 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -8,8 +8,8 @@ import weakref
import errno
from test.support import (TESTFN, captured_output, check_impl_detail,
- check_warnings, cpython_only, gc_collect,
- no_tracing, run_unittest, unlink)
+ check_warnings, cpython_only, gc_collect, run_unittest,
+ no_tracing, unlink)
class NaiveException(Exception):
def __init__(self, x):
@@ -257,22 +257,22 @@ class ExceptionTests(unittest.TestCase):
{'args' : ('foo', 1)}),
(SystemExit, ('foo',),
{'args' : ('foo',), 'code' : 'foo'}),
- (IOError, ('foo',),
+ (OSError, ('foo',),
{'args' : ('foo',), 'filename' : None,
'errno' : None, 'strerror' : None}),
- (IOError, ('foo', 'bar'),
+ (OSError, ('foo', 'bar'),
{'args' : ('foo', 'bar'), 'filename' : None,
'errno' : 'foo', 'strerror' : 'bar'}),
- (IOError, ('foo', 'bar', 'baz'),
+ (OSError, ('foo', 'bar', 'baz'),
{'args' : ('foo', 'bar'), 'filename' : 'baz',
'errno' : 'foo', 'strerror' : 'bar'}),
- (IOError, ('foo', 'bar', 'baz', 'quux'),
- {'args' : ('foo', 'bar', 'baz', 'quux')}),
- (EnvironmentError, ('errnoStr', 'strErrorStr', 'filenameStr'),
+ (OSError, ('foo', 'bar', 'baz', None, 'quux'),
+ {'args' : ('foo', 'bar'), 'filename' : 'baz', 'filename2': 'quux'}),
+ (OSError, ('errnoStr', 'strErrorStr', 'filenameStr'),
{'args' : ('errnoStr', 'strErrorStr'),
'strerror' : 'strErrorStr', 'errno' : 'errnoStr',
'filename' : 'filenameStr'}),
- (EnvironmentError, (1, 'strErrorStr', 'filenameStr'),
+ (OSError, (1, 'strErrorStr', 'filenameStr'),
{'args' : (1, 'strErrorStr'), 'errno' : 1,
'strerror' : 'strErrorStr', 'filename' : 'filenameStr'}),
(SyntaxError, (), {'msg' : None, 'text' : None,
@@ -422,7 +422,7 @@ class ExceptionTests(unittest.TestCase):
self.assertIsNone(e.__context__)
self.assertIsNone(e.__cause__)
- class MyException(EnvironmentError):
+ class MyException(OSError):
pass
e = MyException()
@@ -968,8 +968,5 @@ class ImportErrorTests(unittest.TestCase):
self.assertEqual(str(arg), str(exc))
-def test_main():
- run_unittest(ExceptionTests, ImportErrorTests)
-
if __name__ == '__main__':
unittest.main()
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
index 770e70c..ebd99ed 100644
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -19,18 +19,6 @@ except ImportError:
TIMEOUT = 0.5
-try:
- from resource import setrlimit, RLIMIT_CORE, error as resource_error
-except ImportError:
- prepare_subprocess = None
-else:
- def prepare_subprocess():
- # don't create core file
- try:
- setrlimit(RLIMIT_CORE, (0, 0))
- except (ValueError, resource_error):
- pass
-
def expected_traceback(lineno1, lineno2, header, min_count=1):
regex = header
regex += ' File "<string>", line %s in func\n' % lineno1
@@ -59,10 +47,8 @@ class FaultHandlerTests(unittest.TestCase):
build, and replace "Current thread 0x00007f8d8fbd9700" by "Current
thread XXX".
"""
- options = {}
- if prepare_subprocess:
- options['preexec_fn'] = prepare_subprocess
- process = script_helper.spawn_python('-c', code, **options)
+ with support.SuppressCrashReport():
+ process = script_helper.spawn_python('-c', code)
stdout, stderr = process.communicate()
exitcode = process.wait()
output = support.strip_python_stderr(stdout)
@@ -86,9 +72,9 @@ class FaultHandlerTests(unittest.TestCase):
Raise an error if the output doesn't match the expected format.
"""
if all_threads:
- header = 'Current thread XXX'
+ header = 'Current thread XXX (most recent call first)'
else:
- header = 'Traceback (most recent call first)'
+ header = 'Stack (most recent call first)'
regex = """
^Fatal Python error: {name}
@@ -101,8 +87,7 @@ class FaultHandlerTests(unittest.TestCase):
header=re.escape(header))
if other_regex:
regex += '|' + other_regex
- with support.suppress_crash_popup():
- output, exitcode = self.get_output(code, filename)
+ output, exitcode = self.get_output(code, filename)
output = '\n'.join(output)
self.assertRegex(output, regex)
self.assertNotEqual(exitcode, 0)
@@ -232,8 +217,7 @@ faulthandler.disable()
faulthandler._sigsegv()
""".strip()
not_expected = 'Fatal Python error'
- with support.suppress_crash_popup():
- stderr, exitcode = self.get_output(code)
+ stderr, exitcode = self.get_output(code)
stder = '\n'.join(stderr)
self.assertTrue(not_expected not in stderr,
"%r is present in %r" % (not_expected, stderr))
@@ -264,16 +248,34 @@ faulthandler._sigsegv()
def test_disabled_by_default(self):
# By default, the module should be disabled
code = "import faulthandler; print(faulthandler.is_enabled())"
- rc, stdout, stderr = assert_python_ok("-c", code)
- stdout = (stdout + stderr).strip()
- self.assertEqual(stdout, b"False")
+ args = (sys.executable, '-E', '-c', code)
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args)
+ self.assertEqual(output.rstrip(), b"False")
def test_sys_xoptions(self):
# Test python -X faulthandler
code = "import faulthandler; print(faulthandler.is_enabled())"
- rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code)
- stdout = (stdout + stderr).strip()
- self.assertEqual(stdout, b"True")
+ args = (sys.executable, "-E", "-X", "faulthandler", "-c", code)
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args)
+ self.assertEqual(output.rstrip(), b"True")
+
+ def test_env_var(self):
+ # empty env var
+ code = "import faulthandler; print(faulthandler.is_enabled())"
+ args = (sys.executable, "-c", code)
+ env = os.environ.copy()
+ env['PYTHONFAULTHANDLER'] = ''
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args, env=env)
+ self.assertEqual(output.rstrip(), b"False")
+
+ # non-empty env var
+ env = os.environ.copy()
+ env['PYTHONFAULTHANDLER'] = '1'
+ output = subprocess.check_output(args, env=env)
+ self.assertEqual(output.rstrip(), b"True")
def check_dump_traceback(self, filename):
"""
@@ -304,7 +306,7 @@ funcA()
else:
lineno = 8
expected = [
- 'Traceback (most recent call first):',
+ 'Stack (most recent call first):',
' File "<string>", line %s in funcB' % lineno,
' File "<string>", line 11 in funcA',
' File "<string>", line 13 in <module>'
@@ -336,7 +338,7 @@ def {func_name}():
func_name=func_name,
)
expected = [
- 'Traceback (most recent call first):',
+ 'Stack (most recent call first):',
' File "<string>", line 4 in %s' % truncated,
' File "<string>", line 6 in <module>'
]
@@ -390,13 +392,13 @@ waiter.join()
else:
lineno = 10
regex = """
-^Thread 0x[0-9a-f]+:
+^Thread 0x[0-9a-f]+ \(most recent call first\):
(?: File ".*threading.py", line [0-9]+ in [_a-z]+
){{1,3}} File "<string>", line 23 in run
File ".*threading.py", line [0-9]+ in _bootstrap_inner
File ".*threading.py", line [0-9]+ in _bootstrap
-Current thread XXX:
+Current thread XXX \(most recent call first\):
File "<string>", line {lineno} in dump
File "<string>", line 28 in <module>$
""".strip()
@@ -459,7 +461,7 @@ if file is not None:
count = loops
if repeat:
count *= 2
- header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+:\n' % timeout_str
+ header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+ \(most recent call first\):\n' % timeout_str
regex = expected_traceback(9, 20, header, min_count=count)
self.assertRegex(trace, regex)
else:
@@ -561,9 +563,9 @@ sys.exit(exitcode)
trace = '\n'.join(trace)
if not unregister:
if all_threads:
- regex = 'Current thread XXX:\n'
+ regex = 'Current thread XXX \(most recent call first\):\n'
else:
- regex = 'Traceback \(most recent call first\):\n'
+ regex = 'Stack \(most recent call first\):\n'
regex = expected_traceback(7, 28, regex)
self.assertRegex(trace, regex)
else:
@@ -590,8 +592,5 @@ sys.exit(exitcode)
self.check_register(chain=True)
-def test_main():
- support.run_unittest(FaultHandlerTests)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py
index 1810c4e..e3b7ed2 100644
--- a/Lib/test/test_fcntl.py
+++ b/Lib/test/test_fcntl.py
@@ -1,7 +1,4 @@
"""Test program for the fcntl C module.
-
-OS/2+EMX doesn't support the file locking operations.
-
"""
import platform
import os
@@ -39,8 +36,6 @@ def get_lockdata():
lockdata = struct.pack('qqihhi', 0, 0, 0, fcntl.F_WRLCK, 0, 0)
elif sys.platform in ['aix3', 'aix4', 'hp-uxB', 'unixware7']:
lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0)
- elif sys.platform in ['os2emx']:
- lockdata = None
else:
lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
if lockdata:
@@ -72,18 +67,20 @@ class TestFcntl(unittest.TestCase):
rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
if verbose:
print('Status from fcntl with O_NONBLOCK: ', rv)
- if sys.platform not in ['os2emx']:
- rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
- if verbose:
- print('String from fcntl with F_SETLKW: ', repr(rv))
+ rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
+ if verbose:
+ print('String from fcntl with F_SETLKW: ', repr(rv))
self.f.close()
def test_fcntl_file_descriptor(self):
# again, but pass the file rather than numeric descriptor
self.f = open(TESTFN, 'wb')
rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK)
- if sys.platform not in ['os2emx']:
- rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
+ if verbose:
+ print('Status from fcntl with O_NONBLOCK: ', rv)
+ rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
+ if verbose:
+ print('String from fcntl with F_SETLKW: ', repr(rv))
self.f.close()
def test_fcntl_bad_file(self):
@@ -127,6 +124,26 @@ class TestFcntl(unittest.TestCase):
finally:
os.close(fd)
+ def test_flock(self):
+ # Solaris needs readable file for shared lock
+ self.f = open(TESTFN, 'wb+')
+ fileno = self.f.fileno()
+ fcntl.flock(fileno, fcntl.LOCK_SH)
+ fcntl.flock(fileno, fcntl.LOCK_UN)
+ fcntl.flock(self.f, fcntl.LOCK_SH | fcntl.LOCK_NB)
+ fcntl.flock(self.f, fcntl.LOCK_UN)
+ fcntl.flock(fileno, fcntl.LOCK_EX)
+ fcntl.flock(fileno, fcntl.LOCK_UN)
+
+ self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH)
+ self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH)
+
+ @cpython_only
+ def test_flock_overflow(self):
+ import _testcapi
+ self.assertRaises(OverflowError, fcntl.flock, _testcapi.INT_MAX+1,
+ fcntl.LOCK_SH)
+
def test_main():
run_unittest(TestFcntl)
diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py
index 76b1694..d54e976 100644
--- a/Lib/test/test_file.py
+++ b/Lib/test/test_file.py
@@ -87,7 +87,7 @@ class AutoFileTests:
self.assertTrue(not f.closed)
if hasattr(f, "readinto"):
- self.assertRaises((IOError, TypeError), f.readinto, "")
+ self.assertRaises((OSError, TypeError), f.readinto, "")
f.close()
self.assertTrue(f.closed)
@@ -126,7 +126,7 @@ class AutoFileTests:
self.assertEqual(self.f.__exit__(*sys.exc_info()), None)
def testReadWhenWriting(self):
- self.assertRaises(IOError, self.f.read)
+ self.assertRaises(OSError, self.f.read)
class CAutoFileTests(AutoFileTests, unittest.TestCase):
open = io.open
@@ -177,7 +177,7 @@ class OtherFileTests:
d = int(f.read().decode("ascii"))
f.close()
f.close()
- except IOError as msg:
+ except OSError as msg:
self.fail('error setting buffer size %d: %s' % (s, str(msg)))
self.assertEqual(d, s)
diff --git a/Lib/test/test_filecmp.py b/Lib/test/test_filecmp.py
index 0959979..c7cb3fc 100644
--- a/Lib/test/test_filecmp.py
+++ b/Lib/test/test_filecmp.py
@@ -1,4 +1,3 @@
-
import os, filecmp, shutil, tempfile
import unittest
from test import support
@@ -40,15 +39,27 @@ class FileCompareTestCase(unittest.TestCase):
self.assertFalse(filecmp.cmp(self.name, self.dir),
"File and directory compare as equal")
+ def test_cache_clear(self):
+ first_compare = filecmp.cmp(self.name, self.name_same, shallow=False)
+ second_compare = filecmp.cmp(self.name, self.name_diff, shallow=False)
+ filecmp.clear_cache()
+ self.assertTrue(len(filecmp._cache) == 0,
+ "Cache not cleared after calling clear_cache")
+
class DirCompareTestCase(unittest.TestCase):
def setUp(self):
tmpdir = tempfile.gettempdir()
self.dir = os.path.join(tmpdir, 'dir')
self.dir_same = os.path.join(tmpdir, 'dir-same')
self.dir_diff = os.path.join(tmpdir, 'dir-diff')
+
+ # Another dir is created under dir_same, but it has a name from the
+ # ignored list so it should not affect testing results.
+ self.dir_ignored = os.path.join(self.dir_same, '.hg')
+
self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a')
data = 'Contents of file go here.\n'
- for dir in [self.dir, self.dir_same, self.dir_diff]:
+ for dir in (self.dir, self.dir_same, self.dir_diff, self.dir_ignored):
shutil.rmtree(dir, True)
os.mkdir(dir)
if self.caseinsensitive and dir is self.dir_same:
@@ -64,9 +75,11 @@ class DirCompareTestCase(unittest.TestCase):
output.close()
def tearDown(self):
- shutil.rmtree(self.dir)
- shutil.rmtree(self.dir_same)
- shutil.rmtree(self.dir_diff)
+ for dir in (self.dir, self.dir_same, self.dir_diff):
+ shutil.rmtree(dir)
+
+ def test_default_ignores(self):
+ self.assertIn('.hg', filecmp.DEFAULT_IGNORES)
def test_cmpfiles(self):
self.assertTrue(filecmp.cmpfiles(self.dir, self.dir, ['file']) ==
diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py
index b9523cc..eba55c9 100644
--- a/Lib/test/test_fileinput.py
+++ b/Lib/test/test_fileinput.py
@@ -22,7 +22,7 @@ except ImportError:
from io import StringIO
from fileinput import FileInput, hook_encoded
-from test.support import verbose, TESTFN, run_unittest
+from test.support import verbose, TESTFN, run_unittest, check_warnings
from test.support import unlink as safe_unlink
@@ -224,8 +224,10 @@ class FileInputTests(unittest.TestCase):
try:
# try opening in universal newline mode
t1 = writeTmp(1, [b"A\nB\r\nC\rD"], mode="wb")
- fi = FileInput(files=t1, mode="U")
- lines = list(fi)
+ with check_warnings(('', DeprecationWarning)):
+ fi = FileInput(files=t1, mode="U")
+ with check_warnings(('', DeprecationWarning)):
+ lines = list(fi)
self.assertEqual(lines, ["A\n", "B\n", "C\n", "D"])
finally:
remove_tempfiles(t1)
@@ -296,8 +298,8 @@ class FileInputTests(unittest.TestCase):
try:
t1 = writeTmp(1, [""])
with FileInput(files=t1) as fi:
- raise IOError
- except IOError:
+ raise OSError
+ except OSError:
self.assertEqual(fi._files, ())
finally:
remove_tempfiles(t1)
@@ -869,27 +871,13 @@ class Test_hook_encoded(unittest.TestCase):
self.assertEqual(lines, expected_lines)
check('r', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
- check('rU', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
- check('U', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
+ with self.assertWarns(DeprecationWarning):
+ check('rU', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
+ with self.assertWarns(DeprecationWarning):
+ check('U', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertRaises(ValueError):
check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac'])
-def test_main():
- run_unittest(
- BufferSizesTests,
- FileInputTests,
- Test_fileinput_input,
- Test_fileinput_close,
- Test_fileinput_nextfile,
- Test_fileinput_filename,
- Test_fileinput_lineno,
- Test_fileinput_filelineno,
- Test_fileinput_fileno,
- Test_fileinput_isfirstline,
- Test_fileinput_isstdin,
- Test_hook_compressed,
- Test_hook_encoded,
- )
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py
index 444be91..c37482e 100644
--- a/Lib/test/test_fileio.py
+++ b/Lib/test/test_fileio.py
@@ -144,16 +144,16 @@ class AutoFileTests(unittest.TestCase):
# Unix calls dircheck() and returns "[Errno 21]: Is a directory"
try:
_FileIO('.', 'r')
- except IOError as e:
+ except OSError as e:
self.assertNotEqual(e.errno, 0)
self.assertEqual(e.filename, ".")
else:
- self.fail("Should have raised IOError")
+ self.fail("Should have raised OSError")
@unittest.skipIf(os.name == 'nt', "test only works on a POSIX-like system")
def testOpenDirFD(self):
fd = os.open('.', os.O_RDONLY)
- with self.assertRaises(IOError) as cm:
+ with self.assertRaises(OSError) as cm:
_FileIO(fd, 'r')
os.close(fd)
self.assertEqual(cm.exception.errno, errno.EISDIR)
@@ -171,7 +171,7 @@ class AutoFileTests(unittest.TestCase):
finally:
try:
self.f.close()
- except IOError:
+ except OSError:
pass
return wrapper
@@ -183,14 +183,14 @@ class AutoFileTests(unittest.TestCase):
os.close(f.fileno())
try:
func(self, f)
- except IOError as e:
+ except OSError as e:
self.assertEqual(e.errno, errno.EBADF)
else:
- self.fail("Should have raised IOError")
+ self.fail("Should have raised OSError")
finally:
try:
self.f.close()
- except IOError:
+ except OSError:
pass
return wrapper
@@ -237,7 +237,7 @@ class AutoFileTests(unittest.TestCase):
def ReopenForRead(self):
try:
self.f.close()
- except IOError:
+ except OSError:
pass
self.f = _FileIO(TESTFN, 'r')
os.close(self.f.fileno())
@@ -285,7 +285,7 @@ class OtherFileTests(unittest.TestCase):
if sys.platform != "win32":
try:
f = _FileIO("/dev/tty", "a")
- except EnvironmentError:
+ except OSError:
# When run in a cron job there just aren't any
# ttys, so skip the test. This also handles other
# OS'es that don't support /dev/tty.
@@ -360,7 +360,7 @@ class OtherFileTests(unittest.TestCase):
self.assertRaises(OSError, _FileIO, make_bad_fd())
if sys.platform == 'win32':
import msvcrt
- self.assertRaises(IOError, msvcrt.get_osfhandle, make_bad_fd())
+ self.assertRaises(OSError, msvcrt.get_osfhandle, make_bad_fd())
@cpython_only
def testInvalidFd_overflow(self):
diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py
new file mode 100644
index 0000000..03ac1aa
--- /dev/null
+++ b/Lib/test/test_finalization.py
@@ -0,0 +1,522 @@
+"""
+Tests for object finalization semantics, as outlined in PEP 442.
+"""
+
+import contextlib
+import gc
+import unittest
+import weakref
+
+try:
+ from _testcapi import with_tp_del
+except ImportError:
+ def with_tp_del(cls):
+ class C(object):
+ def __new__(cls, *args, **kwargs):
+ raise TypeError('requires _testcapi.with_tp_del')
+ return C
+
+from test import support
+
+
+class NonGCSimpleBase:
+ """
+ The base class for all the objects under test, equipped with various
+ testing features.
+ """
+
+ survivors = []
+ del_calls = []
+ tp_del_calls = []
+ errors = []
+
+ _cleaning = False
+
+ __slots__ = ()
+
+ @classmethod
+ def _cleanup(cls):
+ cls.survivors.clear()
+ cls.errors.clear()
+ gc.garbage.clear()
+ gc.collect()
+ cls.del_calls.clear()
+ cls.tp_del_calls.clear()
+
+ @classmethod
+ @contextlib.contextmanager
+ def test(cls):
+ """
+ A context manager to use around all finalization tests.
+ """
+ with support.disable_gc():
+ cls.del_calls.clear()
+ cls.tp_del_calls.clear()
+ NonGCSimpleBase._cleaning = False
+ try:
+ yield
+ if cls.errors:
+ raise cls.errors[0]
+ finally:
+ NonGCSimpleBase._cleaning = True
+ cls._cleanup()
+
+ def check_sanity(self):
+ """
+ Check the object is sane (non-broken).
+ """
+
+ def __del__(self):
+ """
+ PEP 442 finalizer. Record that this was called, check the
+ object is in a sane state, and invoke a side effect.
+ """
+ try:
+ if not self._cleaning:
+ self.del_calls.append(id(self))
+ self.check_sanity()
+ self.side_effect()
+ except Exception as e:
+ self.errors.append(e)
+
+ def side_effect(self):
+ """
+ A side effect called on destruction.
+ """
+
+
+class SimpleBase(NonGCSimpleBase):
+
+ def __init__(self):
+ self.id_ = id(self)
+
+ def check_sanity(self):
+ assert self.id_ == id(self)
+
+
+class NonGC(NonGCSimpleBase):
+ __slots__ = ()
+
+class NonGCResurrector(NonGCSimpleBase):
+ __slots__ = ()
+
+ def side_effect(self):
+ """
+ Resurrect self by storing self in a class-wide list.
+ """
+ self.survivors.append(self)
+
+class Simple(SimpleBase):
+ pass
+
+class SimpleResurrector(NonGCResurrector, SimpleBase):
+ pass
+
+
+class TestBase:
+
+ def setUp(self):
+ self.old_garbage = gc.garbage[:]
+ gc.garbage[:] = []
+
+ def tearDown(self):
+ # None of the tests here should put anything in gc.garbage
+ try:
+ self.assertEqual(gc.garbage, [])
+ finally:
+ del self.old_garbage
+ gc.collect()
+
+ def assert_del_calls(self, ids):
+ self.assertEqual(sorted(SimpleBase.del_calls), sorted(ids))
+
+ def assert_tp_del_calls(self, ids):
+ self.assertEqual(sorted(SimpleBase.tp_del_calls), sorted(ids))
+
+ def assert_survivors(self, ids):
+ self.assertEqual(sorted(id(x) for x in SimpleBase.survivors), sorted(ids))
+
+ def assert_garbage(self, ids):
+ self.assertEqual(sorted(id(x) for x in gc.garbage), sorted(ids))
+
+ def clear_survivors(self):
+ SimpleBase.survivors.clear()
+
+
+class SimpleFinalizationTest(TestBase, unittest.TestCase):
+ """
+ Test finalization without refcycles.
+ """
+
+ def test_simple(self):
+ with SimpleBase.test():
+ s = Simple()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+
+ def test_simple_resurrect(self):
+ with SimpleBase.test():
+ s = SimpleResurrector()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors(ids)
+ self.assertIsNot(wr(), None)
+ self.clear_survivors()
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+
+ def test_non_gc(self):
+ with SimpleBase.test():
+ s = NonGC()
+ self.assertFalse(gc.is_tracked(s))
+ ids = [id(s)]
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+
+ def test_non_gc_resurrect(self):
+ with SimpleBase.test():
+ s = NonGCResurrector()
+ self.assertFalse(gc.is_tracked(s))
+ ids = [id(s)]
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors(ids)
+ self.clear_survivors()
+ gc.collect()
+ self.assert_del_calls(ids * 2)
+ self.assert_survivors(ids)
+
+
+class SelfCycleBase:
+
+ def __init__(self):
+ super().__init__()
+ self.ref = self
+
+ def check_sanity(self):
+ super().check_sanity()
+ assert self.ref is self
+
+class SimpleSelfCycle(SelfCycleBase, Simple):
+ pass
+
+class SelfCycleResurrector(SelfCycleBase, SimpleResurrector):
+ pass
+
+class SuicidalSelfCycle(SelfCycleBase, Simple):
+
+ def side_effect(self):
+ """
+ Explicitly break the reference cycle.
+ """
+ self.ref = None
+
+
+class SelfCycleFinalizationTest(TestBase, unittest.TestCase):
+ """
+ Test finalization of an object having a single cyclic reference to
+ itself.
+ """
+
+ def test_simple(self):
+ with SimpleBase.test():
+ s = SimpleSelfCycle()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+
+ def test_simple_resurrect(self):
+ # Test that __del__ can resurrect the object being finalized.
+ with SimpleBase.test():
+ s = SelfCycleResurrector()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors(ids)
+ # XXX is this desirable?
+ self.assertIs(wr(), None)
+ # When trying to destroy the object a second time, __del__
+ # isn't called anymore (and the object isn't resurrected).
+ self.clear_survivors()
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+
+ def test_simple_suicide(self):
+ # Test the GC is able to deal with an object that kills its last
+ # reference during __del__.
+ with SimpleBase.test():
+ s = SuicidalSelfCycle()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+
+
+class ChainedBase:
+
+ def chain(self, left):
+ self.suicided = False
+ self.left = left
+ left.right = self
+
+ def check_sanity(self):
+ super().check_sanity()
+ if self.suicided:
+ assert self.left is None
+ assert self.right is None
+ else:
+ left = self.left
+ if left.suicided:
+ assert left.right is None
+ else:
+ assert left.right is self
+ right = self.right
+ if right.suicided:
+ assert right.left is None
+ else:
+ assert right.left is self
+
+class SimpleChained(ChainedBase, Simple):
+ pass
+
+class ChainedResurrector(ChainedBase, SimpleResurrector):
+ pass
+
+class SuicidalChained(ChainedBase, Simple):
+
+ def side_effect(self):
+ """
+ Explicitly break the reference cycle.
+ """
+ self.suicided = True
+ self.left = None
+ self.right = None
+
+
+class CycleChainFinalizationTest(TestBase, unittest.TestCase):
+ """
+ Test finalization of a cyclic chain. These tests are similar in
+ spirit to the self-cycle tests above, but the collectable object
+ graph isn't trivial anymore.
+ """
+
+ def build_chain(self, classes):
+ nodes = [cls() for cls in classes]
+ for i in range(len(nodes)):
+ nodes[i].chain(nodes[i-1])
+ return nodes
+
+ def check_non_resurrecting_chain(self, classes):
+ N = len(classes)
+ with SimpleBase.test():
+ nodes = self.build_chain(classes)
+ ids = [id(s) for s in nodes]
+ wrs = [weakref.ref(s) for s in nodes]
+ del nodes
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+ self.assertEqual([wr() for wr in wrs], [None] * N)
+ gc.collect()
+ self.assert_del_calls(ids)
+
+ def check_resurrecting_chain(self, classes):
+ N = len(classes)
+ with SimpleBase.test():
+ nodes = self.build_chain(classes)
+ N = len(nodes)
+ ids = [id(s) for s in nodes]
+ survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)]
+ wrs = [weakref.ref(s) for s in nodes]
+ del nodes
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors(survivor_ids)
+ # XXX desirable?
+ self.assertEqual([wr() for wr in wrs], [None] * N)
+ self.clear_survivors()
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_survivors([])
+
+ def test_homogenous(self):
+ self.check_non_resurrecting_chain([SimpleChained] * 3)
+
+ def test_homogenous_resurrect(self):
+ self.check_resurrecting_chain([ChainedResurrector] * 3)
+
+ def test_homogenous_suicidal(self):
+ self.check_non_resurrecting_chain([SuicidalChained] * 3)
+
+ def test_heterogenous_suicidal_one(self):
+ self.check_non_resurrecting_chain([SuicidalChained, SimpleChained] * 2)
+
+ def test_heterogenous_suicidal_two(self):
+ self.check_non_resurrecting_chain(
+ [SuicidalChained] * 2 + [SimpleChained] * 2)
+
+ def test_heterogenous_resurrect_one(self):
+ self.check_resurrecting_chain([ChainedResurrector, SimpleChained] * 2)
+
+ def test_heterogenous_resurrect_two(self):
+ self.check_resurrecting_chain(
+ [ChainedResurrector, SimpleChained, SuicidalChained] * 2)
+
+ def test_heterogenous_resurrect_three(self):
+ self.check_resurrecting_chain(
+ [ChainedResurrector] * 2 + [SimpleChained] * 2 + [SuicidalChained] * 2)
+
+
+# NOTE: the tp_del slot isn't automatically inherited, so we have to call
+# with_tp_del() for each instantiated class.
+
+class LegacyBase(SimpleBase):
+
+ def __del__(self):
+ try:
+ # Do not invoke side_effect here, since we are now exercising
+ # the tp_del slot.
+ if not self._cleaning:
+ self.del_calls.append(id(self))
+ self.check_sanity()
+ except Exception as e:
+ self.errors.append(e)
+
+ def __tp_del__(self):
+ """
+ Legacy (pre-PEP 442) finalizer, mapped to a tp_del slot.
+ """
+ try:
+ if not self._cleaning:
+ self.tp_del_calls.append(id(self))
+ self.check_sanity()
+ self.side_effect()
+ except Exception as e:
+ self.errors.append(e)
+
+@with_tp_del
+class Legacy(LegacyBase):
+ pass
+
+@with_tp_del
+class LegacyResurrector(LegacyBase):
+
+ def side_effect(self):
+ """
+ Resurrect self by storing self in a class-wide list.
+ """
+ self.survivors.append(self)
+
+@with_tp_del
+class LegacySelfCycle(SelfCycleBase, LegacyBase):
+ pass
+
+
+@support.cpython_only
+class LegacyFinalizationTest(TestBase, unittest.TestCase):
+ """
+ Test finalization of objects with a tp_del.
+ """
+
+ def tearDown(self):
+ # These tests need to clean up a bit more, since they create
+ # uncollectable objects.
+ gc.garbage.clear()
+ gc.collect()
+ super().tearDown()
+
+ def test_legacy(self):
+ with SimpleBase.test():
+ s = Legacy()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_tp_del_calls(ids)
+ self.assert_survivors([])
+ self.assertIs(wr(), None)
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_tp_del_calls(ids)
+
+ def test_legacy_resurrect(self):
+ with SimpleBase.test():
+ s = LegacyResurrector()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_tp_del_calls(ids)
+ self.assert_survivors(ids)
+ # weakrefs are cleared before tp_del is called.
+ self.assertIs(wr(), None)
+ self.clear_survivors()
+ gc.collect()
+ self.assert_del_calls(ids)
+ self.assert_tp_del_calls(ids * 2)
+ self.assert_survivors(ids)
+ self.assertIs(wr(), None)
+
+ def test_legacy_self_cycle(self):
+ # Self-cycles with legacy finalizers end up in gc.garbage.
+ with SimpleBase.test():
+ s = LegacySelfCycle()
+ ids = [id(s)]
+ wr = weakref.ref(s)
+ del s
+ gc.collect()
+ self.assert_del_calls([])
+ self.assert_tp_del_calls([])
+ self.assert_survivors([])
+ self.assert_garbage(ids)
+ self.assertIsNot(wr(), None)
+ # Break the cycle to allow collection
+ gc.garbage[0].ref = None
+ self.assert_garbage([])
+ self.assertIs(wr(), None)
+
+
+def test_main():
+ support.run_unittest(__name__)
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py
index 96d907a..e1e1f04 100644
--- a/Lib/test/test_float.py
+++ b/Lib/test/test_float.py
@@ -41,6 +41,7 @@ class GeneralFloatCases(unittest.TestCase):
self.assertRaises(ValueError, float, "-.")
self.assertRaises(ValueError, float, b"-")
self.assertRaises(TypeError, float, {})
+ self.assertRaisesRegex(TypeError, "not 'dict'", float, {})
# Lone surrogate
self.assertRaises(UnicodeEncodeError, float, '\uD8F0')
# check that we don't accept alternate exponent markers
diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py
index 8192c38..e0626df 100644
--- a/Lib/test/test_fork1.py
+++ b/Lib/test/test_fork1.py
@@ -1,7 +1,7 @@
"""This test checks for correct fork() behavior.
"""
-import imp
+import _imp as imp
import os
import signal
import sys
diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py
index ea5a731..fc71e48 100644
--- a/Lib/test/test_format.py
+++ b/Lib/test/test_format.py
@@ -142,7 +142,8 @@ class FormatTest(unittest.TestCase):
testformat("%#+027.23X", big, "+0X0001234567890ABCDEF12345")
# same, except no 0 flag
testformat("%#+27.23X", big, " +0X001234567890ABCDEF12345")
- testformat("%x", float(big), "123456_______________", 6)
+ with self.assertWarns(DeprecationWarning):
+ testformat("%x", float(big), "123456_______________", 6)
big = 0o12345670123456701234567012345670 # 32 octal digits
testformat("%o", big, "12345670123456701234567012345670")
testformat("%o", -big, "-12345670123456701234567012345670")
@@ -182,7 +183,8 @@ class FormatTest(unittest.TestCase):
testformat("%034.33o", big, "0012345670123456701234567012345670")
# base marker shouldn't change that
testformat("%0#34.33o", big, "0o012345670123456701234567012345670")
- testformat("%o", float(big), "123456__________________________", 6)
+ with self.assertWarns(DeprecationWarning):
+ testformat("%o", float(big), "123456__________________________", 6)
# Some small ints, in both Python int and flavors).
testformat("%d", 42, "42")
testformat("%d", -42, "-42")
@@ -193,7 +195,8 @@ class FormatTest(unittest.TestCase):
testformat("%#x", 1, "0x1")
testformat("%#X", 1, "0X1")
testformat("%#X", 1, "0X1")
- testformat("%#x", 1.0, "0x1")
+ with self.assertWarns(DeprecationWarning):
+ testformat("%#x", 1.0, "0x1")
testformat("%#o", 1, "0o1")
testformat("%#o", 1, "0o1")
testformat("%#o", 0, "0o0")
@@ -210,12 +213,14 @@ class FormatTest(unittest.TestCase):
testformat("%x", -0x42, "-42")
testformat("%x", 0x42, "42")
testformat("%x", -0x42, "-42")
- testformat("%x", float(0x42), "42")
+ with self.assertWarns(DeprecationWarning):
+ testformat("%x", float(0x42), "42")
testformat("%o", 0o42, "42")
testformat("%o", -0o42, "-42")
testformat("%o", 0o42, "42")
testformat("%o", -0o42, "-42")
- testformat("%o", float(0o42), "42")
+ with self.assertWarns(DeprecationWarning):
+ testformat("%o", float(0o42), "42")
testformat("%r", "\u0378", "'\\u0378'") # non printable
testformat("%a", "\u0378", "'\\u0378'") # non printable
testformat("%r", "\u0374", "'\u0374'") # printable
@@ -307,10 +312,25 @@ class FormatTest(unittest.TestCase):
finally:
locale.setlocale(locale.LC_ALL, oldloc)
+ @support.cpython_only
+ def test_optimisations(self):
+ text = "abcde" # 5 characters
+
+ self.assertIs("%s" % text, text)
+ self.assertIs("%.5s" % text, text)
+ self.assertIs("%.10s" % text, text)
+ self.assertIs("%1s" % text, text)
+ self.assertIs("%5s" % text, text)
+ self.assertIs("{0}".format(text), text)
+ self.assertIs("{0:s}".format(text), text)
+ self.assertIs("{0:.5s}".format(text), text)
+ self.assertIs("{0:.10s}".format(text), text)
+ self.assertIs("{0:1s}".format(text), text)
+ self.assertIs("{0:5s}".format(text), text)
-def test_main():
- support.run_unittest(FormatTest)
+ self.assertIs(text % (), text)
+ self.assertIs(text.format(), text)
def test_precision(self):
f = 1.2
@@ -318,14 +338,12 @@ def test_main():
self.assertEqual(format(f, ".3f"), "1.200")
with self.assertRaises(ValueError) as cm:
format(f, ".%sf" % (sys.maxsize + 1))
- self.assertEqual(str(cm.exception), "precision too big")
c = complex(f)
self.assertEqual(format(c, ".0f"), "1+0j")
self.assertEqual(format(c, ".3f"), "1.200+0.000j")
with self.assertRaises(ValueError) as cm:
format(c, ".%sf" % (sys.maxsize + 1))
- self.assertEqual(str(cm.exception), "precision too big")
@support.cpython_only
def test_precision_c_limits(self):
diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py
index 1fad921..3336532 100644
--- a/Lib/test/test_fractions.py
+++ b/Lib/test/test_fractions.py
@@ -146,9 +146,10 @@ class FractionTest(unittest.TestCase):
self.assertEqual((0, 1), _components(F(-0.0)))
self.assertEqual((3602879701896397, 36028797018963968),
_components(F(0.1)))
- self.assertRaises(TypeError, F, float('nan'))
- self.assertRaises(TypeError, F, float('inf'))
- self.assertRaises(TypeError, F, float('-inf'))
+ # bug 16469: error types should be consistent with float -> int
+ self.assertRaises(ValueError, F, float('nan'))
+ self.assertRaises(OverflowError, F, float('inf'))
+ self.assertRaises(OverflowError, F, float('-inf'))
def testInitFromDecimal(self):
self.assertEqual((11, 10),
@@ -157,10 +158,11 @@ class FractionTest(unittest.TestCase):
_components(F(Decimal('3.5e-2'))))
self.assertEqual((0, 1),
_components(F(Decimal('.000e20'))))
- self.assertRaises(TypeError, F, Decimal('nan'))
- self.assertRaises(TypeError, F, Decimal('snan'))
- self.assertRaises(TypeError, F, Decimal('inf'))
- self.assertRaises(TypeError, F, Decimal('-inf'))
+ # bug 16469: error types should be consistent with decimal -> int
+ self.assertRaises(ValueError, F, Decimal('nan'))
+ self.assertRaises(ValueError, F, Decimal('snan'))
+ self.assertRaises(OverflowError, F, Decimal('inf'))
+ self.assertRaises(OverflowError, F, Decimal('-inf'))
def testFromString(self):
self.assertEqual((5, 1), _components(F("5")))
@@ -248,14 +250,15 @@ class FractionTest(unittest.TestCase):
inf = 1e1000
nan = inf - inf
+ # bug 16469: error types should be consistent with float -> int
self.assertRaisesMessage(
- TypeError, "Cannot convert inf to Fraction.",
+ OverflowError, "Cannot convert inf to Fraction.",
F.from_float, inf)
self.assertRaisesMessage(
- TypeError, "Cannot convert -inf to Fraction.",
+ OverflowError, "Cannot convert -inf to Fraction.",
F.from_float, -inf)
self.assertRaisesMessage(
- TypeError, "Cannot convert nan to Fraction.",
+ ValueError, "Cannot convert nan to Fraction.",
F.from_float, nan)
def testFromDecimal(self):
@@ -268,17 +271,18 @@ class FractionTest(unittest.TestCase):
self.assertEqual(1 - F(1, 10**30),
F.from_decimal(Decimal("0." + "9" * 30)))
+ # bug 16469: error types should be consistent with decimal -> int
self.assertRaisesMessage(
- TypeError, "Cannot convert Infinity to Fraction.",
+ OverflowError, "Cannot convert Infinity to Fraction.",
F.from_decimal, Decimal("inf"))
self.assertRaisesMessage(
- TypeError, "Cannot convert -Infinity to Fraction.",
+ OverflowError, "Cannot convert -Infinity to Fraction.",
F.from_decimal, Decimal("-inf"))
self.assertRaisesMessage(
- TypeError, "Cannot convert NaN to Fraction.",
+ ValueError, "Cannot convert NaN to Fraction.",
F.from_decimal, Decimal("nan"))
self.assertRaisesMessage(
- TypeError, "Cannot convert sNaN to Fraction.",
+ ValueError, "Cannot convert sNaN to Fraction.",
F.from_decimal, Decimal("snan"))
def testLimitDenominator(self):
diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py
new file mode 100644
index 0000000..2dd5780
--- /dev/null
+++ b/Lib/test/test_frame.py
@@ -0,0 +1,116 @@
+import gc
+import sys
+import unittest
+import weakref
+
+from test import support
+
+
+class ClearTest(unittest.TestCase):
+ """
+ Tests for frame.clear().
+ """
+
+ def inner(self, x=5, **kwargs):
+ 1/0
+
+ def outer(self, **kwargs):
+ try:
+ self.inner(**kwargs)
+ except ZeroDivisionError as e:
+ exc = e
+ return exc
+
+ def clear_traceback_frames(self, tb):
+ """
+ Clear all frames in a traceback.
+ """
+ while tb is not None:
+ tb.tb_frame.clear()
+ tb = tb.tb_next
+
+ def test_clear_locals(self):
+ class C:
+ pass
+ c = C()
+ wr = weakref.ref(c)
+ exc = self.outer(c=c)
+ del c
+ support.gc_collect()
+ # A reference to c is held through the frames
+ self.assertIsNot(None, wr())
+ self.clear_traceback_frames(exc.__traceback__)
+ support.gc_collect()
+ # The reference was released by .clear()
+ self.assertIs(None, wr())
+
+ def test_clear_generator(self):
+ endly = False
+ def g():
+ nonlocal endly
+ try:
+ yield
+ inner()
+ finally:
+ endly = True
+ gen = g()
+ next(gen)
+ self.assertFalse(endly)
+ # Clearing the frame closes the generator
+ gen.gi_frame.clear()
+ self.assertTrue(endly)
+
+ def test_clear_executing(self):
+ # Attempting to clear an executing frame is forbidden.
+ try:
+ 1/0
+ except ZeroDivisionError as e:
+ f = e.__traceback__.tb_frame
+ with self.assertRaises(RuntimeError):
+ f.clear()
+ with self.assertRaises(RuntimeError):
+ f.f_back.clear()
+
+ def test_clear_executing_generator(self):
+ # Attempting to clear an executing generator frame is forbidden.
+ endly = False
+ def g():
+ nonlocal endly
+ try:
+ 1/0
+ except ZeroDivisionError as e:
+ f = e.__traceback__.tb_frame
+ with self.assertRaises(RuntimeError):
+ f.clear()
+ with self.assertRaises(RuntimeError):
+ f.f_back.clear()
+ yield f
+ finally:
+ endly = True
+ gen = g()
+ f = next(gen)
+ self.assertFalse(endly)
+ # Clearing the frame closes the generator
+ f.clear()
+ self.assertTrue(endly)
+
+ @support.cpython_only
+ def test_clear_refcycles(self):
+ # .clear() doesn't leave any refcycle behind
+ with support.disable_gc():
+ class C:
+ pass
+ c = C()
+ wr = weakref.ref(c)
+ exc = self.outer(c=c)
+ del c
+ self.assertIsNot(None, wr())
+ self.clear_traceback_frames(exc.__traceback__)
+ self.assertIs(None, wr())
+
+
+def test_main():
+ support.run_unittest(__name__)
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py
deleted file mode 100644
index fd6761c..0000000
--- a/Lib/test/test_frozen.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Test the frozen module defined in frozen.c.
-
-from test.support import captured_stdout, run_unittest
-import unittest
-import sys
-
-class FrozenTests(unittest.TestCase):
-
- module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
- '__loader__', '__name__',
- '__package__'])
- package_attrs = frozenset(list(module_attrs) + ['__path__'])
-
- def test_frozen(self):
- with captured_stdout() as stdout:
- try:
- import __hello__
- except ImportError as x:
- self.fail("import __hello__ failed:" + str(x))
- self.assertEqual(__hello__.initialized, True)
- expect = set(self.module_attrs)
- expect.add('initialized')
- self.assertEqual(set(dir(__hello__)), expect)
- self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-
- with captured_stdout() as stdout:
- try:
- import __phello__
- except ImportError as x:
- self.fail("import __phello__ failed:" + str(x))
- self.assertEqual(__phello__.initialized, True)
- expect = set(self.package_attrs)
- expect.add('initialized')
- if not "__phello__.spam" in sys.modules:
- self.assertEqual(set(dir(__phello__)), expect)
- else:
- expect.add('spam')
- self.assertEqual(set(dir(__phello__)), expect)
- self.assertEqual(__phello__.__path__, [__phello__.__name__])
- self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-
- with captured_stdout() as stdout:
- try:
- import __phello__.spam
- except ImportError as x:
- self.fail("import __phello__.spam failed:" + str(x))
- self.assertEqual(__phello__.spam.initialized, True)
- spam_expect = set(self.module_attrs)
- spam_expect.add('initialized')
- self.assertEqual(set(dir(__phello__.spam)), spam_expect)
- phello_expect = set(self.package_attrs)
- phello_expect.add('initialized')
- phello_expect.add('spam')
- self.assertEqual(set(dir(__phello__)), phello_expect)
- self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-
- try:
- import __phello__.foo
- except ImportError:
- pass
- else:
- self.fail("import __phello__.foo should have failed")
-
- try:
- import __phello__.foo
- except ImportError:
- pass
- else:
- self.fail("import __phello__.foo should have failed")
-
- del sys.modules['__hello__']
- del sys.modules['__phello__']
- del sys.modules['__phello__.spam']
-
-def test_main():
- run_unittest(FrozenTests)
-
-if __name__ == "__main__":
- test_main()
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
index 6c95c49..a9bf30b 100644
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -15,12 +15,16 @@ try:
import ssl
except ImportError:
ssl = None
+ HAS_SNI = False
+else:
+ from ssl import HAS_SNI
from unittest import TestCase, skipUnless
from test import support
from test.support import HOST, HOSTv6
threading = support.import_module('threading')
+TIMEOUT = 3
# the dummy data returned by server over the data channel when
# RETR, LIST, NLST, MLSD commands are issued
RETR_DATA = 'abcde12345\r\n' * 1000
@@ -126,7 +130,7 @@ class DummyFTPHandler(asynchat.async_chat):
addr = list(map(int, arg.split(',')))
ip = '%d.%d.%d.%d' %tuple(addr[:4])
port = (addr[4] * 256) + addr[5]
- s = socket.create_connection((ip, port), timeout=2)
+ s = socket.create_connection((ip, port), timeout=TIMEOUT)
self.dtp = self.dtp_handler(s, baseclass=self)
self.push('200 active data connection established')
@@ -134,7 +138,7 @@ class DummyFTPHandler(asynchat.async_chat):
with socket.socket() as sock:
sock.bind((self.socket.getsockname()[0], 0))
sock.listen(5)
- sock.settimeout(10)
+ sock.settimeout(TIMEOUT)
ip, port = sock.getsockname()[:2]
ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256
self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2))
@@ -144,7 +148,7 @@ class DummyFTPHandler(asynchat.async_chat):
def cmd_eprt(self, arg):
af, ip, port = arg.split(arg[0])[1:-1]
port = int(port)
- s = socket.create_connection((ip, port), timeout=2)
+ s = socket.create_connection((ip, port), timeout=TIMEOUT)
self.dtp = self.dtp_handler(s, baseclass=self)
self.push('200 active data connection established')
@@ -152,7 +156,7 @@ class DummyFTPHandler(asynchat.async_chat):
with socket.socket(socket.AF_INET6) as sock:
sock.bind((self.socket.getsockname()[0], 0))
sock.listen(5)
- sock.settimeout(10)
+ sock.settimeout(TIMEOUT)
port = sock.getsockname()[1]
self.push('229 entering extended passive mode (|||%d|)' %port)
conn, addr = sock.accept()
@@ -300,7 +304,8 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread):
if ssl is not None:
- CERTFILE = os.path.join(os.path.dirname(__file__), "keycert.pem")
+ CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem")
+ CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem")
class SSLConnection(asyncore.dispatcher):
"""An asyncore.dispatcher subclass supporting TLS/SSL."""
@@ -327,7 +332,7 @@ if ssl is not None:
elif err.args[0] == ssl.SSL_ERROR_EOF:
return self.handle_close()
raise
- except socket.error as err:
+ except OSError as err:
if err.args[0] == errno.ECONNABORTED:
return self.handle_close()
else:
@@ -341,7 +346,7 @@ if ssl is not None:
if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
ssl.SSL_ERROR_WANT_WRITE):
return
- except socket.error as err:
+ except OSError as err:
# Any "socket error" corresponds to a SSL_ERROR_SYSCALL return
# from OpenSSL's SSL_shutdown(), corresponding to a
# closed socket condition. See also:
@@ -460,7 +465,7 @@ class TestFTPClass(TestCase):
def setUp(self):
self.server = DummyFTPServer((HOST, 0))
self.server.start()
- self.client = ftplib.FTP(timeout=2)
+ self.client = ftplib.FTP(timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
def tearDown(self):
@@ -488,7 +493,7 @@ class TestFTPClass(TestCase):
def test_all_errors(self):
exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm,
- ftplib.error_proto, ftplib.Error, IOError, EOFError)
+ ftplib.error_proto, ftplib.Error, OSError, EOFError)
for x in exceptions:
try:
raise x('exception not included in all_errors set')
@@ -678,7 +683,7 @@ class TestFTPClass(TestCase):
def test_makepasv(self):
host, port = self.client.makepasv()
- conn = socket.create_connection((host, port), 10)
+ conn = socket.create_connection((host, port), timeout=TIMEOUT)
conn.close()
# IPv4 is in use, just make sure send_epsv has not been used
self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv')
@@ -691,12 +696,12 @@ class TestFTPClass(TestCase):
return False
try:
self.client.sendcmd('noop')
- except (socket.error, EOFError):
+ except (OSError, EOFError):
return False
return True
# base test
- with ftplib.FTP(timeout=2) as self.client:
+ with ftplib.FTP(timeout=TIMEOUT) as self.client:
self.client.connect(self.server.host, self.server.port)
self.client.sendcmd('noop')
self.assertTrue(is_client_connected())
@@ -704,7 +709,7 @@ class TestFTPClass(TestCase):
self.assertFalse(is_client_connected())
# QUIT sent inside the with block
- with ftplib.FTP(timeout=2) as self.client:
+ with ftplib.FTP(timeout=TIMEOUT) as self.client:
self.client.connect(self.server.host, self.server.port)
self.client.sendcmd('noop')
self.client.quit()
@@ -714,7 +719,7 @@ class TestFTPClass(TestCase):
# force a wrong response code to be sent on QUIT: error_perm
# is expected and the connection is supposed to be closed
try:
- with ftplib.FTP(timeout=2) as self.client:
+ with ftplib.FTP(timeout=TIMEOUT) as self.client:
self.client.connect(self.server.host, self.server.port)
self.client.sendcmd('noop')
self.server.handler_instance.next_response = '550 error on quit'
@@ -736,7 +741,7 @@ class TestFTPClass(TestCase):
source_address=(HOST, port))
self.assertEqual(self.client.sock.getsockname()[1], port)
self.client.quit()
- except IOError as e:
+ except OSError as e:
if e.errno == errno.EADDRINUSE:
self.skipTest("couldn't bind to port %d" % port)
raise
@@ -747,7 +752,7 @@ class TestFTPClass(TestCase):
try:
with self.client.transfercmd('list') as sock:
self.assertEqual(sock.getsockname()[1], port)
- except IOError as e:
+ except OSError as e:
if e.errno == errno.EADDRINUSE:
self.skipTest("couldn't bind to port %d" % port)
raise
@@ -785,7 +790,7 @@ class TestIPv6Environment(TestCase):
def setUp(self):
self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6)
self.server.start()
- self.client = ftplib.FTP()
+ self.client = ftplib.FTP(timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
def tearDown(self):
@@ -802,7 +807,7 @@ class TestIPv6Environment(TestCase):
def test_makepasv(self):
host, port = self.client.makepasv()
- conn = socket.create_connection((host, port), 10)
+ conn = socket.create_connection((host, port), timeout=TIMEOUT)
conn.close()
self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv')
@@ -829,7 +834,7 @@ class TestTLS_FTPClassMixin(TestFTPClass):
def setUp(self):
self.server = DummyTLS_FTPServer((HOST, 0))
self.server.start()
- self.client = ftplib.FTP_TLS(timeout=2)
+ self.client = ftplib.FTP_TLS(timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
# enable TLS
self.client.auth()
@@ -843,7 +848,7 @@ class TestTLS_FTPClass(TestCase):
def setUp(self):
self.server = DummyTLS_FTPServer((HOST, 0))
self.server.start()
- self.client = ftplib.FTP_TLS(timeout=2)
+ self.client = ftplib.FTP_TLS(timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
def tearDown(self):
@@ -903,7 +908,7 @@ class TestTLS_FTPClass(TestCase):
self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE,
keyfile=CERTFILE, context=ctx)
- self.client = ftplib.FTP_TLS(context=ctx, timeout=2)
+ self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT)
self.client.connect(self.server.host, self.server.port)
self.assertNotIsInstance(self.client.sock, ssl.SSLSocket)
self.client.auth()
@@ -922,6 +927,37 @@ class TestTLS_FTPClass(TestCase):
self.client.ccc()
self.assertRaises(ValueError, self.client.sock.unwrap)
+ @skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_check_hostname(self):
+ self.client.quit()
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+ ctx.verify_mode = ssl.CERT_REQUIRED
+ ctx.check_hostname = True
+ ctx.load_verify_locations(CAFILE)
+ self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT)
+
+ # 127.0.0.1 doesn't match SAN
+ self.client.connect(self.server.host, self.server.port)
+ with self.assertRaises(ssl.CertificateError):
+ self.client.auth()
+ # exception quits connection
+
+ self.client.connect(self.server.host, self.server.port)
+ self.client.prot_p()
+ with self.assertRaises(ssl.CertificateError):
+ with self.client.transfercmd("list") as sock:
+ pass
+ self.client.quit()
+
+ self.client.connect("localhost", self.server.port)
+ self.client.auth()
+ self.client.quit()
+
+ self.client.connect("localhost", self.server.port)
+ self.client.prot_p()
+ with self.client.transfercmd("list") as sock:
+ pass
+
class TestTimeouts(TestCase):
@@ -1017,8 +1053,19 @@ class TestTimeouts(TestCase):
ftp.close()
+class TestNetrcDeprecation(TestCase):
+
+ def test_deprecation(self):
+ with support.temp_cwd(), support.EnvironmentVarGuard() as env:
+ env['HOME'] = os.getcwd()
+ open('.netrc', 'w').close()
+ with self.assertWarns(DeprecationWarning):
+ ftplib.Netrc()
+
+
+
def test_main():
- tests = [TestFTPClass, TestTimeouts,
+ tests = [TestFTPClass, TestTimeouts, TestNetrcDeprecation,
TestIPv6Environment,
TestTLS_FTPClassMixin, TestTLS_FTPClass]
diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py
index c8ed830..5094f7b 100644
--- a/Lib/test/test_funcattrs.py
+++ b/Lib/test/test_funcattrs.py
@@ -7,6 +7,11 @@ def global_function():
def inner_function():
class LocalClass:
pass
+ global inner_global_function
+ def inner_global_function():
+ def inner_function2():
+ pass
+ return inner_function2
return LocalClass
return lambda: inner_function
@@ -116,6 +121,8 @@ class FunctionPropertiesTest(FuncAttrsTest):
'global_function.<locals>.inner_function')
self.assertEqual(global_function()()().__qualname__,
'global_function.<locals>.inner_function.<locals>.LocalClass')
+ self.assertEqual(inner_global_function.__qualname__, 'inner_global_function')
+ self.assertEqual(inner_global_function().__qualname__, 'inner_global_function.<locals>.inner_function2')
self.b.__qualname__ = 'c'
self.assertEqual(self.b.__qualname__, 'c')
self.b.__qualname__ = 'd'
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
index d1ce2a9..75ae7f3 100644
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -1,66 +1,52 @@
-import functools
+import abc
import collections
+from itertools import permutations
+import pickle
+from random import choice
import sys
-import unittest
from test import support
+import unittest
from weakref import proxy
-import pickle
-from random import choice
-@staticmethod
-def PythonPartial(func, *args, **keywords):
- 'Pure Python approximation of partial()'
- def newfunc(*fargs, **fkeywords):
- newkeywords = keywords.copy()
- newkeywords.update(fkeywords)
- return func(*(args + fargs), **newkeywords)
- newfunc.func = func
- newfunc.args = args
- newfunc.keywords = keywords
- return newfunc
+import functools
+
+py_functools = support.import_fresh_module('functools', blocked=['_functools'])
+c_functools = support.import_fresh_module('functools', fresh=['_functools'])
+
+decimal = support.import_fresh_module('decimal', fresh=['_decimal'])
+
def capture(*args, **kw):
"""capture all positional and keyword arguments"""
return args, kw
+
def signature(part):
""" return the signature of a partial object """
return (part.func, part.args, part.keywords, part.__dict__)
-class TestPartial(unittest.TestCase):
- thetype = functools.partial
+class TestPartial:
def test_basic_examples(self):
- p = self.thetype(capture, 1, 2, a=10, b=20)
+ p = self.partial(capture, 1, 2, a=10, b=20)
+ self.assertTrue(callable(p))
self.assertEqual(p(3, 4, b=30, c=40),
((1, 2, 3, 4), dict(a=10, b=30, c=40)))
- p = self.thetype(map, lambda x: x*10)
+ p = self.partial(map, lambda x: x*10)
self.assertEqual(list(p([1,2,3,4])), [10, 20, 30, 40])
def test_attributes(self):
- p = self.thetype(capture, 1, 2, a=10, b=20)
+ p = self.partial(capture, 1, 2, a=10, b=20)
# attributes should be readable
self.assertEqual(p.func, capture)
self.assertEqual(p.args, (1, 2))
self.assertEqual(p.keywords, dict(a=10, b=20))
- # attributes should not be writable
- self.assertRaises(AttributeError, setattr, p, 'func', map)
- self.assertRaises(AttributeError, setattr, p, 'args', (1, 2))
- self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2))
-
- p = self.thetype(hex)
- try:
- del p.__dict__
- except TypeError:
- pass
- else:
- self.fail('partial object allowed __dict__ to be deleted')
def test_argument_checking(self):
- self.assertRaises(TypeError, self.thetype) # need at least a func arg
+ self.assertRaises(TypeError, self.partial) # need at least a func arg
try:
- self.thetype(2)()
+ self.partial(2)()
except TypeError:
pass
else:
@@ -71,7 +57,7 @@ class TestPartial(unittest.TestCase):
def func(a=10, b=20):
return a
d = {'a':3}
- p = self.thetype(func, a=5)
+ p = self.partial(func, a=5)
self.assertEqual(p(**d), 3)
self.assertEqual(d, {'a':3})
p(b=7)
@@ -80,20 +66,20 @@ class TestPartial(unittest.TestCase):
def test_arg_combinations(self):
# exercise special code paths for zero args in either partial
# object or the caller
- p = self.thetype(capture)
+ p = self.partial(capture)
self.assertEqual(p(), ((), {}))
self.assertEqual(p(1,2), ((1,2), {}))
- p = self.thetype(capture, 1, 2)
+ p = self.partial(capture, 1, 2)
self.assertEqual(p(), ((1,2), {}))
self.assertEqual(p(3,4), ((1,2,3,4), {}))
def test_kw_combinations(self):
# exercise special code paths for no keyword args in
# either the partial object or the caller
- p = self.thetype(capture)
+ p = self.partial(capture)
self.assertEqual(p(), ((), {}))
self.assertEqual(p(a=1), ((), {'a':1}))
- p = self.thetype(capture, a=1)
+ p = self.partial(capture, a=1)
self.assertEqual(p(), ((), {'a':1}))
self.assertEqual(p(b=2), ((), {'a':1, 'b':2}))
# keyword args in the call override those in the partial object
@@ -102,7 +88,7 @@ class TestPartial(unittest.TestCase):
def test_positional(self):
# make sure positional arguments are captured correctly
for args in [(), (0,), (0,1), (0,1,2), (0,1,2,3)]:
- p = self.thetype(capture, *args)
+ p = self.partial(capture, *args)
expected = args + ('x',)
got, empty = p('x')
self.assertTrue(expected == got and empty == {})
@@ -110,14 +96,14 @@ class TestPartial(unittest.TestCase):
def test_keyword(self):
# make sure keyword arguments are captured correctly
for a in ['a', 0, None, 3.5]:
- p = self.thetype(capture, a=a)
+ p = self.partial(capture, a=a)
expected = {'a':a,'x':None}
empty, got = p(x=None)
self.assertTrue(expected == got and empty == ())
def test_no_side_effects(self):
# make sure there are no side effects that affect subsequent calls
- p = self.thetype(capture, 0, a=1)
+ p = self.partial(capture, 0, a=1)
args1, kw1 = p(1, b=2)
self.assertTrue(args1 == (0,1) and kw1 == {'a':1,'b':2})
args2, kw2 = p()
@@ -126,13 +112,13 @@ class TestPartial(unittest.TestCase):
def test_error_propagation(self):
def f(x, y):
x / y
- self.assertRaises(ZeroDivisionError, self.thetype(f, 1, 0))
- self.assertRaises(ZeroDivisionError, self.thetype(f, 1), 0)
- self.assertRaises(ZeroDivisionError, self.thetype(f), 1, 0)
- self.assertRaises(ZeroDivisionError, self.thetype(f, y=0), 1)
+ self.assertRaises(ZeroDivisionError, self.partial(f, 1, 0))
+ self.assertRaises(ZeroDivisionError, self.partial(f, 1), 0)
+ self.assertRaises(ZeroDivisionError, self.partial(f), 1, 0)
+ self.assertRaises(ZeroDivisionError, self.partial(f, y=0), 1)
def test_weakref(self):
- f = self.thetype(int, base=16)
+ f = self.partial(int, base=16)
p = proxy(f)
self.assertEqual(f.func, p.func)
f = None
@@ -140,39 +126,61 @@ class TestPartial(unittest.TestCase):
def test_with_bound_and_unbound_methods(self):
data = list(map(str, range(10)))
- join = self.thetype(str.join, '')
+ join = self.partial(str.join, '')
self.assertEqual(join(data), '0123456789')
- join = self.thetype(''.join)
+ join = self.partial(''.join)
self.assertEqual(join(data), '0123456789')
+
+@unittest.skipUnless(c_functools, 'requires the C _functools module')
+class TestPartialC(TestPartial, unittest.TestCase):
+ if c_functools:
+ partial = c_functools.partial
+
+ def test_attributes_unwritable(self):
+ # attributes should not be writable
+ p = self.partial(capture, 1, 2, a=10, b=20)
+ self.assertRaises(AttributeError, setattr, p, 'func', map)
+ self.assertRaises(AttributeError, setattr, p, 'args', (1, 2))
+ self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2))
+
+ p = self.partial(hex)
+ try:
+ del p.__dict__
+ except TypeError:
+ pass
+ else:
+ self.fail('partial object allowed __dict__ to be deleted')
+
def test_repr(self):
args = (object(), object())
args_repr = ', '.join(repr(a) for a in args)
- kwargs = {'a': object(), 'b': object()}
+ #kwargs = {'a': object(), 'b': object()}
+ kwargs = {'a': object()}
kwargs_repr = ', '.join("%s=%r" % (k, v) for k, v in kwargs.items())
- if self.thetype is functools.partial:
+ if self.partial is c_functools.partial:
name = 'functools.partial'
else:
- name = self.thetype.__name__
+ name = self.partial.__name__
- f = self.thetype(capture)
+ f = self.partial(capture)
self.assertEqual('{}({!r})'.format(name, capture),
repr(f))
- f = self.thetype(capture, *args)
+ f = self.partial(capture, *args)
self.assertEqual('{}({!r}, {})'.format(name, capture, args_repr),
repr(f))
- f = self.thetype(capture, **kwargs)
+ f = self.partial(capture, **kwargs)
self.assertEqual('{}({!r}, {})'.format(name, capture, kwargs_repr),
repr(f))
- f = self.thetype(capture, *args, **kwargs)
+ f = self.partial(capture, *args, **kwargs)
self.assertEqual('{}({!r}, {}, {})'.format(name, capture, args_repr, kwargs_repr),
repr(f))
def test_pickle(self):
- f = self.thetype(signature, 'asdf', bar=True)
+ f = self.partial(signature, 'asdf', bar=True)
f.add_something_to__dict__ = True
f_copy = pickle.loads(pickle.dumps(f))
self.assertEqual(signature(f), signature(f_copy))
@@ -191,30 +199,140 @@ class TestPartial(unittest.TestCase):
return {}
raise IndexError
- f = self.thetype(object)
+ f = self.partial(object)
self.assertRaisesRegex(SystemError,
"new style getargs format but argument is not a tuple",
f.__setstate__, BadSequence())
-class PartialSubclass(functools.partial):
- pass
-class TestPartialSubclass(TestPartial):
+class TestPartialPy(TestPartial, unittest.TestCase):
+ partial = staticmethod(py_functools.partial)
+
+
+if c_functools:
+ class PartialSubclass(c_functools.partial):
+ pass
- thetype = PartialSubclass
-class TestPythonPartial(TestPartial):
+@unittest.skipUnless(c_functools, 'requires the C _functools module')
+class TestPartialCSubclass(TestPartialC):
+ if c_functools:
+ partial = PartialSubclass
- thetype = PythonPartial
- # the python version hasn't a nice repr
- test_repr = None
+class TestPartialMethod(unittest.TestCase):
- # the python version isn't picklable
- test_pickle = test_setstate_refcount = None
+ class A(object):
+ nothing = functools.partialmethod(capture)
+ positional = functools.partialmethod(capture, 1)
+ keywords = functools.partialmethod(capture, a=2)
+ both = functools.partialmethod(capture, 3, b=4)
+
+ nested = functools.partialmethod(positional, 5)
+
+ over_partial = functools.partialmethod(functools.partial(capture, c=6), 7)
+
+ static = functools.partialmethod(staticmethod(capture), 8)
+ cls = functools.partialmethod(classmethod(capture), d=9)
+
+ a = A()
+
+ def test_arg_combinations(self):
+ self.assertEqual(self.a.nothing(), ((self.a,), {}))
+ self.assertEqual(self.a.nothing(5), ((self.a, 5), {}))
+ self.assertEqual(self.a.nothing(c=6), ((self.a,), {'c': 6}))
+ self.assertEqual(self.a.nothing(5, c=6), ((self.a, 5), {'c': 6}))
+
+ self.assertEqual(self.a.positional(), ((self.a, 1), {}))
+ self.assertEqual(self.a.positional(5), ((self.a, 1, 5), {}))
+ self.assertEqual(self.a.positional(c=6), ((self.a, 1), {'c': 6}))
+ self.assertEqual(self.a.positional(5, c=6), ((self.a, 1, 5), {'c': 6}))
+
+ self.assertEqual(self.a.keywords(), ((self.a,), {'a': 2}))
+ self.assertEqual(self.a.keywords(5), ((self.a, 5), {'a': 2}))
+ self.assertEqual(self.a.keywords(c=6), ((self.a,), {'a': 2, 'c': 6}))
+ self.assertEqual(self.a.keywords(5, c=6), ((self.a, 5), {'a': 2, 'c': 6}))
+
+ self.assertEqual(self.a.both(), ((self.a, 3), {'b': 4}))
+ self.assertEqual(self.a.both(5), ((self.a, 3, 5), {'b': 4}))
+ self.assertEqual(self.a.both(c=6), ((self.a, 3), {'b': 4, 'c': 6}))
+ self.assertEqual(self.a.both(5, c=6), ((self.a, 3, 5), {'b': 4, 'c': 6}))
+
+ self.assertEqual(self.A.both(self.a, 5, c=6), ((self.a, 3, 5), {'b': 4, 'c': 6}))
+
+ def test_nested(self):
+ self.assertEqual(self.a.nested(), ((self.a, 1, 5), {}))
+ self.assertEqual(self.a.nested(6), ((self.a, 1, 5, 6), {}))
+ self.assertEqual(self.a.nested(d=7), ((self.a, 1, 5), {'d': 7}))
+ self.assertEqual(self.a.nested(6, d=7), ((self.a, 1, 5, 6), {'d': 7}))
+
+ self.assertEqual(self.A.nested(self.a, 6, d=7), ((self.a, 1, 5, 6), {'d': 7}))
+
+ def test_over_partial(self):
+ self.assertEqual(self.a.over_partial(), ((self.a, 7), {'c': 6}))
+ self.assertEqual(self.a.over_partial(5), ((self.a, 7, 5), {'c': 6}))
+ self.assertEqual(self.a.over_partial(d=8), ((self.a, 7), {'c': 6, 'd': 8}))
+ self.assertEqual(self.a.over_partial(5, d=8), ((self.a, 7, 5), {'c': 6, 'd': 8}))
+
+ self.assertEqual(self.A.over_partial(self.a, 5, d=8), ((self.a, 7, 5), {'c': 6, 'd': 8}))
+
+ def test_bound_method_introspection(self):
+ obj = self.a
+ self.assertIs(obj.both.__self__, obj)
+ self.assertIs(obj.nested.__self__, obj)
+ self.assertIs(obj.over_partial.__self__, obj)
+ self.assertIs(obj.cls.__self__, self.A)
+ self.assertIs(self.A.cls.__self__, self.A)
+
+ def test_unbound_method_retrieval(self):
+ obj = self.A
+ self.assertFalse(hasattr(obj.both, "__self__"))
+ self.assertFalse(hasattr(obj.nested, "__self__"))
+ self.assertFalse(hasattr(obj.over_partial, "__self__"))
+ self.assertFalse(hasattr(obj.static, "__self__"))
+ self.assertFalse(hasattr(self.a.static, "__self__"))
+
+ def test_descriptors(self):
+ for obj in [self.A, self.a]:
+ with self.subTest(obj=obj):
+ self.assertEqual(obj.static(), ((8,), {}))
+ self.assertEqual(obj.static(5), ((8, 5), {}))
+ self.assertEqual(obj.static(d=8), ((8,), {'d': 8}))
+ self.assertEqual(obj.static(5, d=8), ((8, 5), {'d': 8}))
+
+ self.assertEqual(obj.cls(), ((self.A,), {'d': 9}))
+ self.assertEqual(obj.cls(5), ((self.A, 5), {'d': 9}))
+ self.assertEqual(obj.cls(c=8), ((self.A,), {'c': 8, 'd': 9}))
+ self.assertEqual(obj.cls(5, c=8), ((self.A, 5), {'c': 8, 'd': 9}))
+
+ def test_overriding_keywords(self):
+ self.assertEqual(self.a.keywords(a=3), ((self.a,), {'a': 3}))
+ self.assertEqual(self.A.keywords(self.a, a=3), ((self.a,), {'a': 3}))
+
+ def test_invalid_args(self):
+ with self.assertRaises(TypeError):
+ class B(object):
+ method = functools.partialmethod(None, 1)
+
+ def test_repr(self):
+ self.assertEqual(repr(vars(self.A)['both']),
+ 'functools.partialmethod({}, 3, b=4)'.format(capture))
+
+ def test_abstract(self):
+ class Abstract(abc.ABCMeta):
+
+ @abc.abstractmethod
+ def add(self, x, y):
+ pass
+
+ add5 = functools.partialmethod(add, 5)
+
+ self.assertTrue(Abstract.add.__isabstractmethod__)
+ self.assertTrue(Abstract.add5.__isabstractmethod__)
+
+ for func in [self.A.static, self.A.cls, self.A.over_partial, self.A.nested, self.A.both]:
+ self.assertFalse(getattr(func, '__isabstractmethod__', False))
- # the python version isn't a type
- test_attributes = None
class TestUpdateWrapper(unittest.TestCase):
@@ -223,19 +341,26 @@ class TestUpdateWrapper(unittest.TestCase):
updated=functools.WRAPPER_UPDATES):
# Check attributes were assigned
for name in assigned:
- self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name))
+ self.assertIs(getattr(wrapper, name), getattr(wrapped, name))
# Check attributes were updated
for name in updated:
wrapper_attr = getattr(wrapper, name)
wrapped_attr = getattr(wrapped, name)
for key in wrapped_attr:
- self.assertTrue(wrapped_attr[key] is wrapper_attr[key])
+ if name == "__dict__" and key == "__wrapped__":
+ # __wrapped__ is overwritten by the update code
+ continue
+ self.assertIs(wrapped_attr[key], wrapper_attr[key])
+ # Check __wrapped__
+ self.assertIs(wrapper.__wrapped__, wrapped)
+
def _default_update(self):
def f(a:'This is a new annotation'):
"""This is a test"""
pass
f.attr = 'This is also a test'
+ f.__wrapped__ = "This is a bald faced lie"
def wrapper(b:'This is the prior annotation'):
pass
functools.update_wrapper(wrapper, f)
@@ -322,6 +447,7 @@ class TestUpdateWrapper(unittest.TestCase):
self.assertTrue(wrapper.__doc__.startswith('max('))
self.assertEqual(wrapper.__annotations__, {})
+
class TestWraps(TestUpdateWrapper):
def _default_update(self):
@@ -329,14 +455,15 @@ class TestWraps(TestUpdateWrapper):
"""This is a test"""
pass
f.attr = 'This is also a test'
+ f.__wrapped__ = "This is still a bald faced lie"
@functools.wraps(f)
def wrapper():
pass
- self.check_wrapper(wrapper, f)
return wrapper, f
def test_default_update(self):
wrapper, f = self._default_update()
+ self.check_wrapper(wrapper, f)
self.assertEqual(wrapper.__name__, 'f')
self.assertEqual(wrapper.__qualname__, f.__qualname__)
self.assertEqual(wrapper.attr, 'This is also a test')
@@ -382,6 +509,7 @@ class TestWraps(TestUpdateWrapper):
self.assertEqual(wrapper.attr, 'This is a different test')
self.assertEqual(wrapper.dict_attr, f.dict_attr)
+
class TestReduce(unittest.TestCase):
func = functools.reduce
@@ -462,24 +590,29 @@ class TestReduce(unittest.TestCase):
d = {"one": 1, "two": 2, "three": 3}
self.assertEqual(self.func(add, d), "".join(d.keys()))
-class TestCmpToKey(unittest.TestCase):
+
+class TestCmpToKey:
def test_cmp_to_key(self):
def cmp1(x, y):
return (x > y) - (x < y)
- key = functools.cmp_to_key(cmp1)
+ key = self.cmp_to_key(cmp1)
self.assertEqual(key(3), key(3))
self.assertGreater(key(3), key(1))
+ self.assertGreaterEqual(key(3), key(3))
+
def cmp2(x, y):
return int(x) - int(y)
- key = functools.cmp_to_key(cmp2)
+ key = self.cmp_to_key(cmp2)
self.assertEqual(key(4.0), key('4'))
self.assertLess(key(2), key('35'))
+ self.assertLessEqual(key(2), key('35'))
+ self.assertNotEqual(key(2), key('35'))
def test_cmp_to_key_arguments(self):
def cmp1(x, y):
return (x > y) - (x < y)
- key = functools.cmp_to_key(mycmp=cmp1)
+ key = self.cmp_to_key(mycmp=cmp1)
self.assertEqual(key(obj=3), key(obj=3))
self.assertGreater(key(obj=3), key(obj=1))
with self.assertRaises((TypeError, AttributeError)):
@@ -487,10 +620,10 @@ class TestCmpToKey(unittest.TestCase):
with self.assertRaises((TypeError, AttributeError)):
1 < key(3) # lhs is not a K object
with self.assertRaises(TypeError):
- key = functools.cmp_to_key() # too few args
+ key = self.cmp_to_key() # too few args
with self.assertRaises(TypeError):
- key = functools.cmp_to_key(cmp1, None) # too many args
- key = functools.cmp_to_key(cmp1)
+ key = self.cmp_to_key(cmp1, None) # too many args
+ key = self.cmp_to_key(cmp1)
with self.assertRaises(TypeError):
key() # too few args
with self.assertRaises(TypeError):
@@ -499,7 +632,7 @@ class TestCmpToKey(unittest.TestCase):
def test_bad_cmp(self):
def cmp1(x, y):
raise ZeroDivisionError
- key = functools.cmp_to_key(cmp1)
+ key = self.cmp_to_key(cmp1)
with self.assertRaises(ZeroDivisionError):
key(3) > key(1)
@@ -514,13 +647,13 @@ class TestCmpToKey(unittest.TestCase):
def test_obj_field(self):
def cmp1(x, y):
return (x > y) - (x < y)
- key = functools.cmp_to_key(mycmp=cmp1)
+ key = self.cmp_to_key(mycmp=cmp1)
self.assertEqual(key(50).obj, 50)
def test_sort_int(self):
def mycmp(x, y):
return y - x
- self.assertEqual(sorted(range(5), key=functools.cmp_to_key(mycmp)),
+ self.assertEqual(sorted(range(5), key=self.cmp_to_key(mycmp)),
[4, 3, 2, 1, 0])
def test_sort_int_str(self):
@@ -528,18 +661,29 @@ class TestCmpToKey(unittest.TestCase):
x, y = int(x), int(y)
return (x > y) - (x < y)
values = [5, '3', 7, 2, '0', '1', 4, '10', 1]
- values = sorted(values, key=functools.cmp_to_key(mycmp))
+ values = sorted(values, key=self.cmp_to_key(mycmp))
self.assertEqual([int(value) for value in values],
[0, 1, 1, 2, 3, 4, 5, 7, 10])
def test_hash(self):
def mycmp(x, y):
return y - x
- key = functools.cmp_to_key(mycmp)
+ key = self.cmp_to_key(mycmp)
k = key(10)
self.assertRaises(TypeError, hash, k)
self.assertNotIsInstance(k, collections.Hashable)
+
+@unittest.skipUnless(c_functools, 'requires the C _functools module')
+class TestCmpToKeyC(TestCmpToKey, unittest.TestCase):
+ if c_functools:
+ cmp_to_key = c_functools.cmp_to_key
+
+
+class TestCmpToKeyPy(TestCmpToKey, unittest.TestCase):
+ cmp_to_key = staticmethod(py_functools.cmp_to_key)
+
+
class TestTotalOrdering(unittest.TestCase):
def test_total_ordering_lt(self):
@@ -557,6 +701,7 @@ class TestTotalOrdering(unittest.TestCase):
self.assertTrue(A(2) >= A(1))
self.assertTrue(A(2) <= A(2))
self.assertTrue(A(2) >= A(2))
+ self.assertFalse(A(1) > A(2))
def test_total_ordering_le(self):
@functools.total_ordering
@@ -573,6 +718,7 @@ class TestTotalOrdering(unittest.TestCase):
self.assertTrue(A(2) >= A(1))
self.assertTrue(A(2) <= A(2))
self.assertTrue(A(2) >= A(2))
+ self.assertFalse(A(1) >= A(2))
def test_total_ordering_gt(self):
@functools.total_ordering
@@ -589,6 +735,7 @@ class TestTotalOrdering(unittest.TestCase):
self.assertTrue(A(2) >= A(1))
self.assertTrue(A(2) <= A(2))
self.assertTrue(A(2) >= A(2))
+ self.assertFalse(A(2) < A(1))
def test_total_ordering_ge(self):
@functools.total_ordering
@@ -605,6 +752,7 @@ class TestTotalOrdering(unittest.TestCase):
self.assertTrue(A(2) >= A(1))
self.assertTrue(A(2) <= A(2))
self.assertTrue(A(2) >= A(2))
+ self.assertFalse(A(2) <= A(1))
def test_total_ordering_no_overwrite(self):
# new methods should not overwrite existing
@@ -624,27 +772,118 @@ class TestTotalOrdering(unittest.TestCase):
class A:
pass
- def test_bug_10042(self):
+ def test_type_error_when_not_implemented(self):
+ # bug 10042; ensure stack overflow does not occur
+ # when decorated types return NotImplemented
@functools.total_ordering
- class TestTO:
+ class ImplementsLessThan:
def __init__(self, value):
self.value = value
def __eq__(self, other):
- if isinstance(other, TestTO):
+ if isinstance(other, ImplementsLessThan):
return self.value == other.value
return False
def __lt__(self, other):
- if isinstance(other, TestTO):
+ if isinstance(other, ImplementsLessThan):
return self.value < other.value
- raise TypeError
- with self.assertRaises(TypeError):
- TestTO(8) <= ()
+ return NotImplemented
+
+ @functools.total_ordering
+ class ImplementsGreaterThan:
+ def __init__(self, value):
+ self.value = value
+ def __eq__(self, other):
+ if isinstance(other, ImplementsGreaterThan):
+ return self.value == other.value
+ return False
+ def __gt__(self, other):
+ if isinstance(other, ImplementsGreaterThan):
+ return self.value > other.value
+ return NotImplemented
+
+ @functools.total_ordering
+ class ImplementsLessThanEqualTo:
+ def __init__(self, value):
+ self.value = value
+ def __eq__(self, other):
+ if isinstance(other, ImplementsLessThanEqualTo):
+ return self.value == other.value
+ return False
+ def __le__(self, other):
+ if isinstance(other, ImplementsLessThanEqualTo):
+ return self.value <= other.value
+ return NotImplemented
+
+ @functools.total_ordering
+ class ImplementsGreaterThanEqualTo:
+ def __init__(self, value):
+ self.value = value
+ def __eq__(self, other):
+ if isinstance(other, ImplementsGreaterThanEqualTo):
+ return self.value == other.value
+ return False
+ def __ge__(self, other):
+ if isinstance(other, ImplementsGreaterThanEqualTo):
+ return self.value >= other.value
+ return NotImplemented
+
+ @functools.total_ordering
+ class ComparatorNotImplemented:
+ def __init__(self, value):
+ self.value = value
+ def __eq__(self, other):
+ if isinstance(other, ComparatorNotImplemented):
+ return self.value == other.value
+ return False
+ def __lt__(self, other):
+ return NotImplemented
+
+ with self.subTest("LT < 1"), self.assertRaises(TypeError):
+ ImplementsLessThan(-1) < 1
+
+ with self.subTest("LT < LE"), self.assertRaises(TypeError):
+ ImplementsLessThan(0) < ImplementsLessThanEqualTo(0)
+
+ with self.subTest("LT < GT"), self.assertRaises(TypeError):
+ ImplementsLessThan(1) < ImplementsGreaterThan(1)
+
+ with self.subTest("LE <= LT"), self.assertRaises(TypeError):
+ ImplementsLessThanEqualTo(2) <= ImplementsLessThan(2)
+
+ with self.subTest("LE <= GE"), self.assertRaises(TypeError):
+ ImplementsLessThanEqualTo(3) <= ImplementsGreaterThanEqualTo(3)
+
+ with self.subTest("GT > GE"), self.assertRaises(TypeError):
+ ImplementsGreaterThan(4) > ImplementsGreaterThanEqualTo(4)
+
+ with self.subTest("GT > LT"), self.assertRaises(TypeError):
+ ImplementsGreaterThan(5) > ImplementsLessThan(5)
+
+ with self.subTest("GE >= GT"), self.assertRaises(TypeError):
+ ImplementsGreaterThanEqualTo(6) >= ImplementsGreaterThan(6)
+
+ with self.subTest("GE >= LE"), self.assertRaises(TypeError):
+ ImplementsGreaterThanEqualTo(7) >= ImplementsLessThanEqualTo(7)
+
+ with self.subTest("GE when equal"):
+ a = ComparatorNotImplemented(8)
+ b = ComparatorNotImplemented(8)
+ self.assertEqual(a, b)
+ with self.assertRaises(TypeError):
+ a >= b
+
+ with self.subTest("LE when equal"):
+ a = ComparatorNotImplemented(9)
+ b = ComparatorNotImplemented(9)
+ self.assertEqual(a, b)
+ with self.assertRaises(TypeError):
+ a <= b
class TestLRU(unittest.TestCase):
def test_lru(self):
def orig(x, y):
- return 3*x+y
+ return 3 * x + y
f = functools.lru_cache(maxsize=20)(orig)
hits, misses, maxsize, currsize = f.cache_info()
self.assertEqual(maxsize, 20)
@@ -749,7 +988,7 @@ class TestLRU(unittest.TestCase):
# Verify that user_function exceptions get passed through without
# creating a hard-to-read chained exception.
# http://bugs.python.org/issue13177
- for maxsize in (None, 100):
+ for maxsize in (None, 128):
@functools.lru_cache(maxsize)
def func(i):
return 'abc'[i]
@@ -762,7 +1001,7 @@ class TestLRU(unittest.TestCase):
func(15)
def test_lru_with_types(self):
- for maxsize in (None, 100):
+ for maxsize in (None, 128):
@functools.lru_cache(maxsize=maxsize, typed=True)
def square(x):
return x * x
@@ -777,6 +1016,36 @@ class TestLRU(unittest.TestCase):
self.assertEqual(square.cache_info().hits, 4)
self.assertEqual(square.cache_info().misses, 4)
+ def test_lru_with_keyword_args(self):
+ @functools.lru_cache()
+ def fib(n):
+ if n < 2:
+ return n
+ return fib(n=n-1) + fib(n=n-2)
+ self.assertEqual(
+ [fib(n=number) for number in range(16)],
+ [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610]
+ )
+ self.assertEqual(fib.cache_info(),
+ functools._CacheInfo(hits=28, misses=16, maxsize=128, currsize=16))
+ fib.cache_clear()
+ self.assertEqual(fib.cache_info(),
+ functools._CacheInfo(hits=0, misses=0, maxsize=128, currsize=0))
+
+ def test_lru_with_keyword_args_maxsize_none(self):
+ @functools.lru_cache(maxsize=None)
+ def fib(n):
+ if n < 2:
+ return n
+ return fib(n=n-1) + fib(n=n-2)
+ self.assertEqual([fib(n=number) for number in range(16)],
+ [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610])
+ self.assertEqual(fib.cache_info(),
+ functools._CacheInfo(hits=28, misses=16, maxsize=None, currsize=16))
+ fib.cache_clear()
+ self.assertEqual(fib.cache_info(),
+ functools._CacheInfo(hits=0, misses=0, maxsize=None, currsize=0))
+
def test_need_for_rlock(self):
# This will deadlock on an LRU cache that uses a regular lock
@@ -802,17 +1071,495 @@ class TestLRU(unittest.TestCase):
DoubleEq(2)) # Verify the correct return value
+class TestSingleDispatch(unittest.TestCase):
+ def test_simple_overloads(self):
+ @functools.singledispatch
+ def g(obj):
+ return "base"
+ def g_int(i):
+ return "integer"
+ g.register(int, g_int)
+ self.assertEqual(g("str"), "base")
+ self.assertEqual(g(1), "integer")
+ self.assertEqual(g([1,2,3]), "base")
+
+ def test_mro(self):
+ @functools.singledispatch
+ def g(obj):
+ return "base"
+ class A:
+ pass
+ class C(A):
+ pass
+ class B(A):
+ pass
+ class D(C, B):
+ pass
+ def g_A(a):
+ return "A"
+ def g_B(b):
+ return "B"
+ g.register(A, g_A)
+ g.register(B, g_B)
+ self.assertEqual(g(A()), "A")
+ self.assertEqual(g(B()), "B")
+ self.assertEqual(g(C()), "A")
+ self.assertEqual(g(D()), "B")
+
+ def test_register_decorator(self):
+ @functools.singledispatch
+ def g(obj):
+ return "base"
+ @g.register(int)
+ def g_int(i):
+ return "int %s" % (i,)
+ self.assertEqual(g(""), "base")
+ self.assertEqual(g(12), "int 12")
+ self.assertIs(g.dispatch(int), g_int)
+ self.assertIs(g.dispatch(object), g.dispatch(str))
+ # Note: in the assert above this is not g.
+ # @singledispatch returns the wrapper.
+
+ def test_wrapping_attributes(self):
+ @functools.singledispatch
+ def g(obj):
+ "Simple test"
+ return "Test"
+ self.assertEqual(g.__name__, "g")
+ if sys.flags.optimize < 2:
+ self.assertEqual(g.__doc__, "Simple test")
+
+ @unittest.skipUnless(decimal, 'requires _decimal')
+ @support.cpython_only
+ def test_c_classes(self):
+ @functools.singledispatch
+ def g(obj):
+ return "base"
+ @g.register(decimal.DecimalException)
+ def _(obj):
+ return obj.args
+ subn = decimal.Subnormal("Exponent < Emin")
+ rnd = decimal.Rounded("Number got rounded")
+ self.assertEqual(g(subn), ("Exponent < Emin",))
+ self.assertEqual(g(rnd), ("Number got rounded",))
+ @g.register(decimal.Subnormal)
+ def _(obj):
+ return "Too small to care."
+ self.assertEqual(g(subn), "Too small to care.")
+ self.assertEqual(g(rnd), ("Number got rounded",))
+
+ def test_compose_mro(self):
+ # None of the examples in this test depend on haystack ordering.
+ c = collections
+ mro = functools._compose_mro
+ bases = [c.Sequence, c.MutableMapping, c.Mapping, c.Set]
+ for haystack in permutations(bases):
+ m = mro(dict, haystack)
+ self.assertEqual(m, [dict, c.MutableMapping, c.Mapping, c.Sized,
+ c.Iterable, c.Container, object])
+ bases = [c.Container, c.Mapping, c.MutableMapping, c.OrderedDict]
+ for haystack in permutations(bases):
+ m = mro(c.ChainMap, haystack)
+ self.assertEqual(m, [c.ChainMap, c.MutableMapping, c.Mapping,
+ c.Sized, c.Iterable, c.Container, object])
+
+ # If there's a generic function with implementations registered for
+ # both Sized and Container, passing a defaultdict to it results in an
+ # ambiguous dispatch which will cause a RuntimeError (see
+ # test_mro_conflicts).
+ bases = [c.Container, c.Sized, str]
+ for haystack in permutations(bases):
+ m = mro(c.defaultdict, [c.Sized, c.Container, str])
+ self.assertEqual(m, [c.defaultdict, dict, c.Sized, c.Container,
+ object])
+
+ # MutableSequence below is registered directly on D. In other words, it
+ # preceeds MutableMapping which means single dispatch will always
+ # choose MutableSequence here.
+ class D(c.defaultdict):
+ pass
+ c.MutableSequence.register(D)
+ bases = [c.MutableSequence, c.MutableMapping]
+ for haystack in permutations(bases):
+ m = mro(D, bases)
+ self.assertEqual(m, [D, c.MutableSequence, c.Sequence,
+ c.defaultdict, dict, c.MutableMapping,
+ c.Mapping, c.Sized, c.Iterable, c.Container,
+ object])
+
+ # Container and Callable are registered on different base classes and
+ # a generic function supporting both should always pick the Callable
+ # implementation if a C instance is passed.
+ class C(c.defaultdict):
+ def __call__(self):
+ pass
+ bases = [c.Sized, c.Callable, c.Container, c.Mapping]
+ for haystack in permutations(bases):
+ m = mro(C, haystack)
+ self.assertEqual(m, [C, c.Callable, c.defaultdict, dict, c.Mapping,
+ c.Sized, c.Iterable, c.Container, object])
+
+ def test_register_abc(self):
+ c = collections
+ d = {"a": "b"}
+ l = [1, 2, 3]
+ s = {object(), None}
+ f = frozenset(s)
+ t = (1, 2, 3)
+ @functools.singledispatch
+ def g(obj):
+ return "base"
+ self.assertEqual(g(d), "base")
+ self.assertEqual(g(l), "base")
+ self.assertEqual(g(s), "base")
+ self.assertEqual(g(f), "base")
+ self.assertEqual(g(t), "base")
+ g.register(c.Sized, lambda obj: "sized")
+ self.assertEqual(g(d), "sized")
+ self.assertEqual(g(l), "sized")
+ self.assertEqual(g(s), "sized")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.MutableMapping, lambda obj: "mutablemapping")
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(g(l), "sized")
+ self.assertEqual(g(s), "sized")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.ChainMap, lambda obj: "chainmap")
+ self.assertEqual(g(d), "mutablemapping") # irrelevant ABCs registered
+ self.assertEqual(g(l), "sized")
+ self.assertEqual(g(s), "sized")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.MutableSequence, lambda obj: "mutablesequence")
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "sized")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.MutableSet, lambda obj: "mutableset")
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.Mapping, lambda obj: "mapping")
+ self.assertEqual(g(d), "mutablemapping") # not specific enough
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sized")
+ g.register(c.Sequence, lambda obj: "sequence")
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "sized")
+ self.assertEqual(g(t), "sequence")
+ g.register(c.Set, lambda obj: "set")
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "set")
+ self.assertEqual(g(t), "sequence")
+ g.register(dict, lambda obj: "dict")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "mutablesequence")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "set")
+ self.assertEqual(g(t), "sequence")
+ g.register(list, lambda obj: "list")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "list")
+ self.assertEqual(g(s), "mutableset")
+ self.assertEqual(g(f), "set")
+ self.assertEqual(g(t), "sequence")
+ g.register(set, lambda obj: "concrete-set")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "list")
+ self.assertEqual(g(s), "concrete-set")
+ self.assertEqual(g(f), "set")
+ self.assertEqual(g(t), "sequence")
+ g.register(frozenset, lambda obj: "frozen-set")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "list")
+ self.assertEqual(g(s), "concrete-set")
+ self.assertEqual(g(f), "frozen-set")
+ self.assertEqual(g(t), "sequence")
+ g.register(tuple, lambda obj: "tuple")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "list")
+ self.assertEqual(g(s), "concrete-set")
+ self.assertEqual(g(f), "frozen-set")
+ self.assertEqual(g(t), "tuple")
+
+ def test_c3_abc(self):
+ c = collections
+ mro = functools._c3_mro
+ class A(object):
+ pass
+ class B(A):
+ def __len__(self):
+ return 0 # implies Sized
+ @c.Container.register
+ class C(object):
+ pass
+ class D(object):
+ pass # unrelated
+ class X(D, C, B):
+ def __call__(self):
+ pass # implies Callable
+ expected = [X, c.Callable, D, C, c.Container, B, c.Sized, A, object]
+ for abcs in permutations([c.Sized, c.Callable, c.Container]):
+ self.assertEqual(mro(X, abcs=abcs), expected)
+ # unrelated ABCs don't appear in the resulting MRO
+ many_abcs = [c.Mapping, c.Sized, c.Callable, c.Container, c.Iterable]
+ self.assertEqual(mro(X, abcs=many_abcs), expected)
+
+ def test_mro_conflicts(self):
+ c = collections
+ @functools.singledispatch
+ def g(arg):
+ return "base"
+ class O(c.Sized):
+ def __len__(self):
+ return 0
+ o = O()
+ self.assertEqual(g(o), "base")
+ g.register(c.Iterable, lambda arg: "iterable")
+ g.register(c.Container, lambda arg: "container")
+ g.register(c.Sized, lambda arg: "sized")
+ g.register(c.Set, lambda arg: "set")
+ self.assertEqual(g(o), "sized")
+ c.Iterable.register(O)
+ self.assertEqual(g(o), "sized") # because it's explicitly in __mro__
+ c.Container.register(O)
+ self.assertEqual(g(o), "sized") # see above: Sized is in __mro__
+ c.Set.register(O)
+ self.assertEqual(g(o), "set") # because c.Set is a subclass of
+ # c.Sized and c.Container
+ class P:
+ pass
+ p = P()
+ self.assertEqual(g(p), "base")
+ c.Iterable.register(P)
+ self.assertEqual(g(p), "iterable")
+ c.Container.register(P)
+ with self.assertRaises(RuntimeError) as re_one:
+ g(p)
+ self.assertIn(
+ str(re_one.exception),
+ (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+ "or <class 'collections.abc.Iterable'>"),
+ ("Ambiguous dispatch: <class 'collections.abc.Iterable'> "
+ "or <class 'collections.abc.Container'>")),
+ )
+ class Q(c.Sized):
+ def __len__(self):
+ return 0
+ q = Q()
+ self.assertEqual(g(q), "sized")
+ c.Iterable.register(Q)
+ self.assertEqual(g(q), "sized") # because it's explicitly in __mro__
+ c.Set.register(Q)
+ self.assertEqual(g(q), "set") # because c.Set is a subclass of
+ # c.Sized and c.Iterable
+ @functools.singledispatch
+ def h(arg):
+ return "base"
+ @h.register(c.Sized)
+ def _(arg):
+ return "sized"
+ @h.register(c.Container)
+ def _(arg):
+ return "container"
+ # Even though Sized and Container are explicit bases of MutableMapping,
+ # this ABC is implicitly registered on defaultdict which makes all of
+ # MutableMapping's bases implicit as well from defaultdict's
+ # perspective.
+ with self.assertRaises(RuntimeError) as re_two:
+ h(c.defaultdict(lambda: 0))
+ self.assertIn(
+ str(re_two.exception),
+ (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+ "or <class 'collections.abc.Sized'>"),
+ ("Ambiguous dispatch: <class 'collections.abc.Sized'> "
+ "or <class 'collections.abc.Container'>")),
+ )
+ class R(c.defaultdict):
+ pass
+ c.MutableSequence.register(R)
+ @functools.singledispatch
+ def i(arg):
+ return "base"
+ @i.register(c.MutableMapping)
+ def _(arg):
+ return "mapping"
+ @i.register(c.MutableSequence)
+ def _(arg):
+ return "sequence"
+ r = R()
+ self.assertEqual(i(r), "sequence")
+ class S:
+ pass
+ class T(S, c.Sized):
+ def __len__(self):
+ return 0
+ t = T()
+ self.assertEqual(h(t), "sized")
+ c.Container.register(T)
+ self.assertEqual(h(t), "sized") # because it's explicitly in the MRO
+ class U:
+ def __len__(self):
+ return 0
+ u = U()
+ self.assertEqual(h(u), "sized") # implicit Sized subclass inferred
+ # from the existence of __len__()
+ c.Container.register(U)
+ # There is no preference for registered versus inferred ABCs.
+ with self.assertRaises(RuntimeError) as re_three:
+ h(u)
+ self.assertIn(
+ str(re_three.exception),
+ (("Ambiguous dispatch: <class 'collections.abc.Container'> "
+ "or <class 'collections.abc.Sized'>"),
+ ("Ambiguous dispatch: <class 'collections.abc.Sized'> "
+ "or <class 'collections.abc.Container'>")),
+ )
+ class V(c.Sized, S):
+ def __len__(self):
+ return 0
+ @functools.singledispatch
+ def j(arg):
+ return "base"
+ @j.register(S)
+ def _(arg):
+ return "s"
+ @j.register(c.Container)
+ def _(arg):
+ return "container"
+ v = V()
+ self.assertEqual(j(v), "s")
+ c.Container.register(V)
+ self.assertEqual(j(v), "container") # because it ends up right after
+ # Sized in the MRO
+
+ def test_cache_invalidation(self):
+ from collections import UserDict
+ class TracingDict(UserDict):
+ def __init__(self, *args, **kwargs):
+ super(TracingDict, self).__init__(*args, **kwargs)
+ self.set_ops = []
+ self.get_ops = []
+ def __getitem__(self, key):
+ result = self.data[key]
+ self.get_ops.append(key)
+ return result
+ def __setitem__(self, key, value):
+ self.set_ops.append(key)
+ self.data[key] = value
+ def clear(self):
+ self.data.clear()
+ _orig_wkd = functools.WeakKeyDictionary
+ td = TracingDict()
+ functools.WeakKeyDictionary = lambda: td
+ c = collections
+ @functools.singledispatch
+ def g(arg):
+ return "base"
+ d = {}
+ l = []
+ self.assertEqual(len(td), 0)
+ self.assertEqual(g(d), "base")
+ self.assertEqual(len(td), 1)
+ self.assertEqual(td.get_ops, [])
+ self.assertEqual(td.set_ops, [dict])
+ self.assertEqual(td.data[dict], g.registry[object])
+ self.assertEqual(g(l), "base")
+ self.assertEqual(len(td), 2)
+ self.assertEqual(td.get_ops, [])
+ self.assertEqual(td.set_ops, [dict, list])
+ self.assertEqual(td.data[dict], g.registry[object])
+ self.assertEqual(td.data[list], g.registry[object])
+ self.assertEqual(td.data[dict], td.data[list])
+ self.assertEqual(g(l), "base")
+ self.assertEqual(g(d), "base")
+ self.assertEqual(td.get_ops, [list, dict])
+ self.assertEqual(td.set_ops, [dict, list])
+ g.register(list, lambda arg: "list")
+ self.assertEqual(td.get_ops, [list, dict])
+ self.assertEqual(len(td), 0)
+ self.assertEqual(g(d), "base")
+ self.assertEqual(len(td), 1)
+ self.assertEqual(td.get_ops, [list, dict])
+ self.assertEqual(td.set_ops, [dict, list, dict])
+ self.assertEqual(td.data[dict],
+ functools._find_impl(dict, g.registry))
+ self.assertEqual(g(l), "list")
+ self.assertEqual(len(td), 2)
+ self.assertEqual(td.get_ops, [list, dict])
+ self.assertEqual(td.set_ops, [dict, list, dict, list])
+ self.assertEqual(td.data[list],
+ functools._find_impl(list, g.registry))
+ class X:
+ pass
+ c.MutableMapping.register(X) # Will not invalidate the cache,
+ # not using ABCs yet.
+ self.assertEqual(g(d), "base")
+ self.assertEqual(g(l), "list")
+ self.assertEqual(td.get_ops, [list, dict, dict, list])
+ self.assertEqual(td.set_ops, [dict, list, dict, list])
+ g.register(c.Sized, lambda arg: "sized")
+ self.assertEqual(len(td), 0)
+ self.assertEqual(g(d), "sized")
+ self.assertEqual(len(td), 1)
+ self.assertEqual(td.get_ops, [list, dict, dict, list])
+ self.assertEqual(td.set_ops, [dict, list, dict, list, dict])
+ self.assertEqual(g(l), "list")
+ self.assertEqual(len(td), 2)
+ self.assertEqual(td.get_ops, [list, dict, dict, list])
+ self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list])
+ self.assertEqual(g(l), "list")
+ self.assertEqual(g(d), "sized")
+ self.assertEqual(td.get_ops, [list, dict, dict, list, list, dict])
+ self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list])
+ g.dispatch(list)
+ g.dispatch(dict)
+ self.assertEqual(td.get_ops, [list, dict, dict, list, list, dict,
+ list, dict])
+ self.assertEqual(td.set_ops, [dict, list, dict, list, dict, list])
+ c.MutableSet.register(X) # Will invalidate the cache.
+ self.assertEqual(len(td), 2) # Stale cache.
+ self.assertEqual(g(l), "list")
+ self.assertEqual(len(td), 1)
+ g.register(c.MutableMapping, lambda arg: "mutablemapping")
+ self.assertEqual(len(td), 0)
+ self.assertEqual(g(d), "mutablemapping")
+ self.assertEqual(len(td), 1)
+ self.assertEqual(g(l), "list")
+ self.assertEqual(len(td), 2)
+ g.register(dict, lambda arg: "dict")
+ self.assertEqual(g(d), "dict")
+ self.assertEqual(g(l), "list")
+ g._clear_cache()
+ self.assertEqual(len(td), 0)
+ functools.WeakKeyDictionary = _orig_wkd
+
+
def test_main(verbose=None):
test_classes = (
- TestPartial,
- TestPartialSubclass,
- TestPythonPartial,
+ TestPartialC,
+ TestPartialPy,
+ TestPartialCSubclass,
+ TestPartialMethod,
TestUpdateWrapper,
TestTotalOrdering,
- TestCmpToKey,
+ TestCmpToKeyC,
+ TestCmpToKeyPy,
TestWraps,
TestReduce,
TestLRU,
+ TestSingleDispatch,
)
support.run_unittest(*test_classes)
diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py
index a0c156f..beac993 100644
--- a/Lib/test/test_future.py
+++ b/Lib/test/test_future.py
@@ -82,6 +82,14 @@ class FutureTest(unittest.TestCase):
else:
self.fail("expected exception didn't occur")
+ def test_badfuture10(self):
+ try:
+ from test import badsyntax_future10
+ except SyntaxError as msg:
+ self.assertEqual(get_error_location(msg), ("badsyntax_future10", '3'))
+ else:
+ self.fail("expected exception didn't occur")
+
def test_parserhack(self):
# test that the parser.c::future_hack function works as expected
# Note: although this test must pass, it's not testing the original
diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
index c59b72e..7eb104a 100644
--- a/Lib/test/test_gc.py
+++ b/Lib/test/test_gc.py
@@ -1,6 +1,8 @@
import unittest
from test.support import (verbose, refcount_test, run_unittest,
- strip_python_stderr)
+ strip_python_stderr, cpython_only)
+from test.script_helper import assert_python_ok, make_script, temp_dir
+
import sys
import time
import gc
@@ -11,6 +13,15 @@ try:
except ImportError:
threading = None
+try:
+ from _testcapi import with_tp_del
+except ImportError:
+ def with_tp_del(cls):
+ class C(object):
+ def __new__(cls, *args, **kwargs):
+ raise TypeError('requires _testcapi.with_tp_del')
+ return C
+
### Support code
###############################################################################
@@ -38,6 +49,7 @@ class GC_Detector(object):
# gc collects it.
self.wr = weakref.ref(C1055820(666), it_happened)
+@with_tp_del
class Uncollectable(object):
"""Create a reference cycle with multiple __del__ methods.
@@ -50,7 +62,7 @@ class Uncollectable(object):
self.partner = Uncollectable(partner=self)
else:
self.partner = partner
- def __del__(self):
+ def __tp_del__(self):
pass
### Tests
@@ -139,11 +151,13 @@ class GCTests(unittest.TestCase):
del a
self.assertNotEqual(gc.collect(), 0)
- def test_finalizer(self):
+ @cpython_only
+ def test_legacy_finalizer(self):
# A() is uncollectable if it is part of a cycle, make sure it shows up
# in gc.garbage.
+ @with_tp_del
class A:
- def __del__(self): pass
+ def __tp_del__(self): pass
class B:
pass
a = A()
@@ -163,11 +177,13 @@ class GCTests(unittest.TestCase):
self.fail("didn't find obj in garbage (finalizer)")
gc.garbage.remove(obj)
- def test_finalizer_newclass(self):
+ @cpython_only
+ def test_legacy_finalizer_newclass(self):
# A() is uncollectable if it is part of a cycle, make sure it shows up
# in gc.garbage.
+ @with_tp_del
class A(object):
- def __del__(self): pass
+ def __tp_del__(self): pass
class B(object):
pass
a = A()
@@ -564,16 +580,19 @@ class GCTests(unittest.TestCase):
# would be damaged, with an empty __dict__.
self.assertEqual(x, None)
+ @cpython_only
def test_garbage_at_shutdown(self):
import subprocess
code = """if 1:
import gc
+ import _testcapi
+ @_testcapi.with_tp_del
class X:
def __init__(self, name):
self.name = name
def __repr__(self):
return "<X %%r>" %% self.name
- def __del__(self):
+ def __tp_del__(self):
pass
x = X('first')
@@ -610,6 +629,66 @@ class GCTests(unittest.TestCase):
stderr = run_command(code % "gc.DEBUG_SAVEALL")
self.assertNotIn(b"uncollectable objects at shutdown", stderr)
+ def test_gc_main_module_at_shutdown(self):
+ # Create a reference cycle through the __main__ module and check
+ # it gets collected at interpreter shutdown.
+ code = """if 1:
+ import weakref
+ class C:
+ def __del__(self):
+ print('__del__ called')
+ l = [C()]
+ l.append(l)
+ """
+ rc, out, err = assert_python_ok('-c', code)
+ self.assertEqual(out.strip(), b'__del__ called')
+
+ def test_gc_ordinary_module_at_shutdown(self):
+ # Same as above, but with a non-__main__ module.
+ with temp_dir() as script_dir:
+ module = """if 1:
+ import weakref
+ class C:
+ def __del__(self):
+ print('__del__ called')
+ l = [C()]
+ l.append(l)
+ """
+ code = """if 1:
+ import sys
+ sys.path.insert(0, %r)
+ import gctest
+ """ % (script_dir,)
+ make_script(script_dir, 'gctest', module)
+ rc, out, err = assert_python_ok('-c', code)
+ self.assertEqual(out.strip(), b'__del__ called')
+
+ def test_get_stats(self):
+ stats = gc.get_stats()
+ self.assertEqual(len(stats), 3)
+ for st in stats:
+ self.assertIsInstance(st, dict)
+ self.assertEqual(set(st),
+ {"collected", "collections", "uncollectable"})
+ self.assertGreaterEqual(st["collected"], 0)
+ self.assertGreaterEqual(st["collections"], 0)
+ self.assertGreaterEqual(st["uncollectable"], 0)
+ # Check that collection counts are incremented correctly
+ if gc.isenabled():
+ self.addCleanup(gc.enable)
+ gc.disable()
+ old = gc.get_stats()
+ gc.collect(0)
+ new = gc.get_stats()
+ self.assertEqual(new[0]["collections"], old[0]["collections"] + 1)
+ self.assertEqual(new[1]["collections"], old[1]["collections"])
+ self.assertEqual(new[2]["collections"], old[2]["collections"])
+ gc.collect(2)
+ new = gc.get_stats()
+ self.assertEqual(new[0]["collections"], old[0]["collections"] + 1)
+ self.assertEqual(new[1]["collections"], old[1]["collections"])
+ self.assertEqual(new[2]["collections"], old[2]["collections"] + 1)
+
class GCCallbackTests(unittest.TestCase):
def setUp(self):
@@ -696,6 +775,7 @@ class GCCallbackTests(unittest.TestCase):
info = v[2]
self.assertEqual(info["generation"], 2)
+ @cpython_only
def test_collect_garbage(self):
self.preclean()
# Each of these cause four objects to be garbage: Two
diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py
index 846422b..0e254a2 100644
--- a/Lib/test/test_gdb.py
+++ b/Lib/test/test_gdb.py
@@ -5,6 +5,7 @@
import os
import re
+import pprint
import subprocess
import sys
import sysconfig
@@ -17,6 +18,7 @@ try:
except ImportError:
_thread = None
+from test import support
from test.support import run_unittest, findfile, python_is_optimized
try:
@@ -40,6 +42,8 @@ if not sysconfig.is_python_build():
checkout_hook_path = os.path.join(os.path.dirname(sys.executable),
'python-gdb.py')
+PYTHONHASHSEED = '123'
+
def run_gdb(*args, **env_vars):
"""Runs gdb in --batch mode with the additional arguments given by *args.
@@ -144,7 +148,7 @@ class DebuggerTests(unittest.TestCase):
# print (' '.join(args))
# Use "args" to invoke gdb, capturing stdout, stderr:
- out, err = run_gdb(*args, PYTHONHASHSEED='0')
+ out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED)
errlines = err.splitlines()
unexpected_errlines = []
@@ -165,6 +169,10 @@ class DebuggerTests(unittest.TestCase):
'linux-gate.so',
'Do you need "set solib-search-path" or '
'"set sysroot"?',
+ 'warning: Source file is more recent than executable.',
+ # Issue #19753: missing symbols on System Z
+ 'Missing separate debuginfo for ',
+ 'Try: zypper install -C ',
)
for line in errlines:
if not line.startswith(ignore_patterns):
@@ -248,9 +256,8 @@ class PrettyPrintTests(DebuggerTests):
def test_dicts(self):
'Verify the pretty-printing of dictionaries'
self.assertGdbRepr({})
- self.assertGdbRepr({'foo': 'bar'})
- self.assertGdbRepr({'foo': 'bar', 'douglas': 42},
- "{'foo': 'bar', 'douglas': 42}")
+ self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}")
+ self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}")
def test_lists(self):
'Verify the pretty-printing of lists'
@@ -305,26 +312,30 @@ class PrettyPrintTests(DebuggerTests):
def test_tuples(self):
'Verify the pretty-printing of tuples'
- self.assertGdbRepr(tuple())
+ self.assertGdbRepr(tuple(), '()')
self.assertGdbRepr((1,), '(1,)')
self.assertGdbRepr(('foo', 'bar', 'baz'))
def test_sets(self):
'Verify the pretty-printing of sets'
- self.assertGdbRepr(set())
+ if (gdb_major_version, gdb_minor_version) < (7, 3):
+ self.skipTest("pretty-printing of sets needs gdb 7.3 or later")
+ self.assertGdbRepr(set(), 'set()')
self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}")
self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}")
# Ensure that we handle sets containing the "dummy" key value,
# which happens on deletion:
gdb_repr, gdb_output = self.get_gdb_repr('''s = set(['a','b'])
-s.pop()
+s.remove('a')
id(s)''')
self.assertEqual(gdb_repr, "{'b'}")
def test_frozensets(self):
'Verify the pretty-printing of frozensets'
- self.assertGdbRepr(frozenset())
+ if (gdb_major_version, gdb_minor_version) < (7, 3):
+ self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later")
+ self.assertGdbRepr(frozenset(), 'frozenset()')
self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})")
self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})")
@@ -841,6 +852,10 @@ class PyLocalsTests(DebuggerTests):
r".*\na = 1\nb = 2\nc = 3\n.*")
def test_main():
+ if support.verbose:
+ print("GDB version:")
+ for line in os.fsdecode(gdb_version).splitlines():
+ print(" " * 4 + line)
run_unittest(PrettyPrintTests,
PyListTests,
StackNavigationTests,
diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index 958054a..91afe47 100644
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -1,3 +1,55 @@
+import gc
+import sys
+import unittest
+import weakref
+
+from test import support
+
+
+class FinalizationTest(unittest.TestCase):
+
+ def test_frame_resurrect(self):
+ # A generator frame can be resurrected by a generator's finalization.
+ def gen():
+ nonlocal frame
+ try:
+ yield
+ finally:
+ frame = sys._getframe()
+
+ g = gen()
+ wr = weakref.ref(g)
+ next(g)
+ del g
+ support.gc_collect()
+ self.assertIs(wr(), None)
+ self.assertTrue(frame)
+ del frame
+ support.gc_collect()
+
+ def test_refcycle(self):
+ # A generator caught in a refcycle gets finalized anyway.
+ old_garbage = gc.garbage[:]
+ finalized = False
+ def gen():
+ nonlocal finalized
+ try:
+ g = yield
+ yield 1
+ finally:
+ finalized = True
+
+ g = gen()
+ next(g)
+ g.send(g)
+ self.assertGreater(sys.getrefcount(g), 2)
+ self.assertFalse(finalized)
+ del g
+ support.gc_collect()
+ self.assertTrue(finalized)
+ self.assertEqual(gc.garbage, old_garbage)
+
+
tutorial_tests = """
Let's try a simple generator:
@@ -384,8 +436,8 @@ From the Iterators list, about the types of these things.
>>> [s for s in dir(i) if not s.startswith('_')]
['close', 'gi_code', 'gi_frame', 'gi_running', 'send', 'throw']
>>> from test.support import HAVE_DOCSTRINGS
->>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'x.__next__() <==> next(x)')
-x.__next__() <==> next(x)
+>>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).')
+Implement next(self).
>>> iter(i) is i
True
>>> import types
@@ -1729,9 +1781,7 @@ Our ill-behaved code should be invoked during GC:
>>> g = f()
>>> next(g)
>>> del g
->>> sys.stderr.getvalue().startswith(
-... "Exception RuntimeError: 'generator ignored GeneratorExit' in "
-... )
+>>> "RuntimeError: generator ignored GeneratorExit" in sys.stderr.getvalue()
True
>>> sys.stderr = old
@@ -1841,22 +1891,23 @@ to test.
... sys.stderr = io.StringIO()
... class Leaker:
... def __del__(self):
-... raise RuntimeError
+... def invoke(message):
+... raise RuntimeError(message)
+... invoke("test")
...
... l = Leaker()
... del l
... err = sys.stderr.getvalue().strip()
-... err.startswith(
-... "Exception RuntimeError: RuntimeError() in <"
-... )
-... err.endswith("> ignored")
-... len(err.splitlines())
+... "Exception ignored in" in err
+... "RuntimeError: test" in err
+... "Traceback" in err
+... "in invoke" in err
... finally:
... sys.stderr = old
True
True
-1
-
+True
+True
These refleak tests should perhaps be in a testfile of their own,
@@ -1881,6 +1932,7 @@ __test__ = {"tut": tutorial_tests,
# so this works as expected in both ways of running regrtest.
def test_main(verbose=None):
from test import support, test_generators
+ support.run_unittest(__name__)
support.run_doctest(test_generators, verbose)
# This part isn't needed for regrtest, but for running the test directly.
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index b5068e4..e59ed4d 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -188,12 +188,93 @@ class GenericTest:
support.unlink(support.TESTFN)
safe_rmdir(support.TESTFN)
+ @staticmethod
+ def _create_file(filename):
+ with open(filename, 'wb') as f:
+ f.write(b'foo')
+
+ def test_samefile(self):
+ try:
+ test_fn = support.TESTFN + "1"
+ self._create_file(test_fn)
+ self.assertTrue(self.pathmodule.samefile(test_fn, test_fn))
+ self.assertRaises(TypeError, self.pathmodule.samefile)
+ finally:
+ os.remove(test_fn)
+
+ @support.skip_unless_symlink
+ def test_samefile_on_symlink(self):
+ self._test_samefile_on_link_func(os.symlink)
+
+ def test_samefile_on_link(self):
+ self._test_samefile_on_link_func(os.link)
+
+ def _test_samefile_on_link_func(self, func):
+ try:
+ test_fn1 = support.TESTFN + "1"
+ test_fn2 = support.TESTFN + "2"
+ self._create_file(test_fn1)
+
+ func(test_fn1, test_fn2)
+ self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2))
+ os.remove(test_fn2)
+
+ self._create_file(test_fn2)
+ self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2))
+ finally:
+ os.remove(test_fn1)
+ os.remove(test_fn2)
+
+ def test_samestat(self):
+ try:
+ test_fn = support.TESTFN + "1"
+ self._create_file(test_fn)
+ test_fns = [test_fn]*2
+ stats = map(os.stat, test_fns)
+ self.assertTrue(self.pathmodule.samestat(*stats))
+ finally:
+ os.remove(test_fn)
+
+ @support.skip_unless_symlink
+ def test_samestat_on_symlink(self):
+ self._test_samestat_on_link_func(os.symlink)
+
+ def test_samestat_on_link(self):
+ self._test_samestat_on_link_func(os.link)
+
+ def _test_samestat_on_link_func(self, func):
+ try:
+ test_fn1 = support.TESTFN + "1"
+ test_fn2 = support.TESTFN + "2"
+ self._create_file(test_fn1)
+ test_fns = (test_fn1, test_fn2)
+ func(*test_fns)
+ stats = map(os.stat, test_fns)
+ self.assertTrue(self.pathmodule.samestat(*stats))
+ os.remove(test_fn2)
+
+ self._create_file(test_fn2)
+ stats = map(os.stat, test_fns)
+ self.assertFalse(self.pathmodule.samestat(*stats))
+
+ self.assertRaises(TypeError, self.pathmodule.samestat)
+ finally:
+ os.remove(test_fn1)
+ os.remove(test_fn2)
+
+ def test_sameopenfile(self):
+ fname = support.TESTFN + "1"
+ with open(fname, "wb") as a, open(fname, "wb") as b:
+ self.assertTrue(self.pathmodule.sameopenfile(
+ a.fileno(), b.fileno()))
+
class TestGenericTest(GenericTest, unittest.TestCase):
# Issue 16852: GenericTest can't inherit from unittest.TestCase
# for test discovery purposes; CommonTest inherits from GenericTest
# and is only meant to be inherited by others.
pathmodule = genericpath
+
# Following TestCase is not supposed to be run from test_genericpath.
# It is inherited by other test modules (macpath, ntpath, posixpath).
@@ -348,7 +429,6 @@ class CommonTest(GenericTest):
else:
self.skipTest("need support.TESTFN_NONASCII")
- # Test non-ASCII, non-UTF8 bytes in the path.
with warnings.catch_warnings():
warnings.simplefilter("ignore", DeprecationWarning)
with support.temp_cwd(name):
diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py
index 203b336..fb531d6 100644
--- a/Lib/test/test_genexps.py
+++ b/Lib/test/test_genexps.py
@@ -222,8 +222,8 @@ Check that generator attributes are present
True
>>> from test.support import HAVE_DOCSTRINGS
- >>> print(g.__next__.__doc__ if HAVE_DOCSTRINGS else 'x.__next__() <==> next(x)')
- x.__next__() <==> next(x)
+ >>> print(g.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).')
+ Implement next(self).
>>> import types
>>> isinstance(g, types.GeneratorType)
True
diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py
index fe64ee1..1853a2d 100644
--- a/Lib/test/test_getargs2.py
+++ b/Lib/test/test_getargs2.py
@@ -3,38 +3,40 @@ from test import support
# Skip this test if the _testcapi module isn't available.
support.import_module('_testcapi')
from _testcapi import getargs_keywords, getargs_keyword_only
-
-"""
-> How about the following counterproposal. This also changes some of
-> the other format codes to be a little more regular.
->
-> Code C type Range check
->
-> b unsigned char 0..UCHAR_MAX
-> h signed short SHRT_MIN..SHRT_MAX
-> B unsigned char none **
-> H unsigned short none **
-> k * unsigned long none
-> I * unsigned int 0..UINT_MAX
-
-
-> i int INT_MIN..INT_MAX
-> l long LONG_MIN..LONG_MAX
-
-> K * unsigned long long none
-> L long long LLONG_MIN..LLONG_MAX
-
-> Notes:
->
-> * New format codes.
->
-> ** Changed from previous "range-and-a-half" to "none"; the
-> range-and-a-half checking wasn't particularly useful.
-
-Plus a C API or two, e.g. PyInt_AsLongMask() ->
-unsigned long and PyInt_AsLongLongMask() -> unsigned
-long long (if that exists).
-"""
+try:
+ from _testcapi import getargs_L, getargs_K
+except ImportError:
+ getargs_L = None # PY_LONG_LONG not available
+
+# > How about the following counterproposal. This also changes some of
+# > the other format codes to be a little more regular.
+# >
+# > Code C type Range check
+# >
+# > b unsigned char 0..UCHAR_MAX
+# > h signed short SHRT_MIN..SHRT_MAX
+# > B unsigned char none **
+# > H unsigned short none **
+# > k * unsigned long none
+# > I * unsigned int 0..UINT_MAX
+#
+#
+# > i int INT_MIN..INT_MAX
+# > l long LONG_MIN..LONG_MAX
+#
+# > K * unsigned long long none
+# > L long long LLONG_MIN..LLONG_MAX
+#
+# > Notes:
+# >
+# > * New format codes.
+# >
+# > ** Changed from previous "range-and-a-half" to "none"; the
+# > range-and-a-half checking wasn't particularly useful.
+#
+# Plus a C API or two, e.g. PyInt_AsLongMask() ->
+# unsigned long and PyInt_AsLongLongMask() -> unsigned
+# long long (if that exists).
LARGE = 0x7FFFFFFF
VERY_LARGE = 0xFF0000121212121212121242
@@ -77,7 +79,8 @@ class Unsigned_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_b(Int()))
self.assertEqual(0, getargs_b(IntSubclass()))
self.assertRaises(TypeError, getargs_b, BadInt())
- self.assertEqual(1, getargs_b(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_b(BadInt2()))
self.assertEqual(0, getargs_b(BadInt3()))
self.assertRaises(OverflowError, getargs_b, -1)
@@ -95,7 +98,8 @@ class Unsigned_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_B(Int()))
self.assertEqual(0, getargs_B(IntSubclass()))
self.assertRaises(TypeError, getargs_B, BadInt())
- self.assertEqual(1, getargs_B(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_B(BadInt2()))
self.assertEqual(0, getargs_B(BadInt3()))
self.assertEqual(UCHAR_MAX, getargs_B(-1))
@@ -113,7 +117,8 @@ class Unsigned_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_H(Int()))
self.assertEqual(0, getargs_H(IntSubclass()))
self.assertRaises(TypeError, getargs_H, BadInt())
- self.assertEqual(1, getargs_H(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_H(BadInt2()))
self.assertEqual(0, getargs_H(BadInt3()))
self.assertEqual(USHRT_MAX, getargs_H(-1))
@@ -132,7 +137,8 @@ class Unsigned_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_I(Int()))
self.assertEqual(0, getargs_I(IntSubclass()))
self.assertRaises(TypeError, getargs_I, BadInt())
- self.assertEqual(1, getargs_I(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_I(BadInt2()))
self.assertEqual(0, getargs_I(BadInt3()))
self.assertEqual(UINT_MAX, getargs_I(-1))
@@ -172,7 +178,8 @@ class Signed_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_h(Int()))
self.assertEqual(0, getargs_h(IntSubclass()))
self.assertRaises(TypeError, getargs_h, BadInt())
- self.assertEqual(1, getargs_h(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_h(BadInt2()))
self.assertEqual(0, getargs_h(BadInt3()))
self.assertRaises(OverflowError, getargs_h, SHRT_MIN-1)
@@ -190,7 +197,8 @@ class Signed_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_i(Int()))
self.assertEqual(0, getargs_i(IntSubclass()))
self.assertRaises(TypeError, getargs_i, BadInt())
- self.assertEqual(1, getargs_i(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_i(BadInt2()))
self.assertEqual(0, getargs_i(BadInt3()))
self.assertRaises(OverflowError, getargs_i, INT_MIN-1)
@@ -208,7 +216,8 @@ class Signed_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_l(Int()))
self.assertEqual(0, getargs_l(IntSubclass()))
self.assertRaises(TypeError, getargs_l, BadInt())
- self.assertEqual(1, getargs_l(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_l(BadInt2()))
self.assertEqual(0, getargs_l(BadInt3()))
self.assertRaises(OverflowError, getargs_l, LONG_MIN-1)
@@ -239,6 +248,7 @@ class Signed_TestCase(unittest.TestCase):
self.assertRaises(OverflowError, getargs_n, VERY_LARGE)
+@unittest.skipIf(getargs_L is None, 'PY_LONG_LONG is not available')
class LongLong_TestCase(unittest.TestCase):
def test_L(self):
from _testcapi import getargs_L
@@ -249,7 +259,8 @@ class LongLong_TestCase(unittest.TestCase):
self.assertEqual(99, getargs_L(Int()))
self.assertEqual(0, getargs_L(IntSubclass()))
self.assertRaises(TypeError, getargs_L, BadInt())
- self.assertEqual(1, getargs_L(BadInt2()))
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(1, getargs_L(BadInt2()))
self.assertEqual(0, getargs_L(BadInt3()))
self.assertRaises(OverflowError, getargs_L, LLONG_MIN-1)
@@ -600,24 +611,5 @@ class Unicode_TestCase(unittest.TestCase):
self.assertIsNone(getargs_Z_hash(None))
-def test_main():
- tests = [
- Signed_TestCase,
- Unsigned_TestCase,
- Boolean_TestCase,
- Tuple_TestCase,
- Keywords_TestCase,
- KeywordOnly_TestCase,
- Bytes_TestCase,
- Unicode_TestCase,
- ]
- try:
- from _testcapi import getargs_L, getargs_K
- except ImportError:
- pass # PY_LONG_LONG not available
- else:
- tests.append(LongLong_TestCase)
- support.run_unittest(*tests)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index eb9aeb5..a5ab8d6 100644
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -169,6 +169,28 @@ class GlobTests(unittest.TestCase):
eq(glob.glob('\\\\*\\*\\'), [])
eq(glob.glob(b'\\\\*\\*\\'), [])
+ def check_escape(self, arg, expected):
+ self.assertEqual(glob.escape(arg), expected)
+ self.assertEqual(glob.escape(os.fsencode(arg)), os.fsencode(expected))
+
+ def test_escape(self):
+ check = self.check_escape
+ check('abc', 'abc')
+ check('[', '[[]')
+ check('?', '[?]')
+ check('*', '[*]')
+ check('[[_/*?*/_]]', '[[][[]_/[*][?][*]/_]]')
+ check('/[[_/*?*/_]]/', '/[[][[]_/[*][?][*]/_]]/')
+
+ @unittest.skipUnless(sys.platform == "win32", "Win32 specific test")
+ def test_escape_windows(self):
+ check = self.check_escape
+ check('?:?', '?:[?]')
+ check('*:*', '*:[*]')
+ check(r'\\?\c:\?', r'\\?\c:\[?]')
+ check(r'\\*\*\*', r'\\*\*\[*]')
+ check('//?/c:/?', '//?/c:/[?]')
+ check('//*/*/*', '//*/*/[*]')
def test_main():
run_unittest(GlobTests)
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 6b326bd..bba8820 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -314,6 +314,13 @@ class GrammarTests(unittest.TestCase):
self.assertEqual(f.__annotations__,
{'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
'k': 11, 'return': 12})
+ # Check for issue #20625 -- annotations mangling
+ class Spam:
+ def f(self, *, __kw:1):
+ pass
+ class Ham(Spam): pass
+ self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': 1})
+ self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': 1})
# Check for SF Bug #1697248 - mixing decorators and a return annotation
def null(x): return x
@null
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index 034acb0..b7a7e03 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -130,6 +130,14 @@ class TestGzip(BaseTest):
if not ztxt: break
self.assertEqual(contents, b'a'*201)
+ def test_exclusive_write(self):
+ with gzip.GzipFile(self.filename, 'xb') as f:
+ f.write(data1 * 50)
+ with gzip.GzipFile(self.filename, 'rb') as f:
+ self.assertEqual(f.read(), data1 * 50)
+ with self.assertRaises(FileExistsError):
+ gzip.GzipFile(self.filename, 'xb')
+
def test_buffered_reader(self):
# Issue #7471: a GzipFile can be wrapped in a BufferedReader for
# performance.
@@ -205,6 +213,9 @@ class TestGzip(BaseTest):
self.test_write()
with gzip.GzipFile(self.filename, 'r') as f:
self.assertEqual(f.myfileobj.mode, 'rb')
+ support.unlink(self.filename)
+ with gzip.GzipFile(self.filename, 'x') as f:
+ self.assertEqual(f.myfileobj.mode, 'xb')
def test_1647484(self):
for mode in ('wb', 'rb'):
@@ -388,6 +399,20 @@ class TestGzip(BaseTest):
datac = gzip.compress(data)
self.assertEqual(gzip.decompress(datac), data)
+ def test_read_truncated(self):
+ data = data1*50
+ # Drop the CRC (4 bytes) and file size (4 bytes).
+ truncated = gzip.compress(data)[:-8]
+ with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f:
+ self.assertRaises(EOFError, f.read)
+ with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f:
+ self.assertEqual(f.read(len(data)), data)
+ self.assertRaises(EOFError, f.read, 1)
+ # Incomplete 10-byte header.
+ for i in range(2, 10):
+ with gzip.GzipFile(fileobj=io.BytesIO(truncated[:i])) as f:
+ self.assertRaises(EOFError, f.read, 1)
+
def test_read_with_extra(self):
# Gzip data with an extra field
gzdata = (b'\x1f\x8b\x08\x04\xb2\x17cQ\x02\xff'
@@ -406,35 +431,59 @@ class TestGzip(BaseTest):
class TestOpen(BaseTest):
def test_binary_modes(self):
uncompressed = data1 * 50
+
with gzip.open(self.filename, "wb") as f:
f.write(uncompressed)
with open(self.filename, "rb") as f:
file_data = gzip.decompress(f.read())
self.assertEqual(file_data, uncompressed)
+
with gzip.open(self.filename, "rb") as f:
self.assertEqual(f.read(), uncompressed)
+
with gzip.open(self.filename, "ab") as f:
f.write(uncompressed)
with open(self.filename, "rb") as f:
file_data = gzip.decompress(f.read())
self.assertEqual(file_data, uncompressed * 2)
+ with self.assertRaises(FileExistsError):
+ gzip.open(self.filename, "xb")
+ support.unlink(self.filename)
+ with gzip.open(self.filename, "xb") as f:
+ f.write(uncompressed)
+ with open(self.filename, "rb") as f:
+ file_data = gzip.decompress(f.read())
+ self.assertEqual(file_data, uncompressed)
+
def test_implicit_binary_modes(self):
# Test implicit binary modes (no "b" or "t" in mode string).
uncompressed = data1 * 50
+
with gzip.open(self.filename, "w") as f:
f.write(uncompressed)
with open(self.filename, "rb") as f:
file_data = gzip.decompress(f.read())
self.assertEqual(file_data, uncompressed)
+
with gzip.open(self.filename, "r") as f:
self.assertEqual(f.read(), uncompressed)
+
with gzip.open(self.filename, "a") as f:
f.write(uncompressed)
with open(self.filename, "rb") as f:
file_data = gzip.decompress(f.read())
self.assertEqual(file_data, uncompressed * 2)
+ with self.assertRaises(FileExistsError):
+ gzip.open(self.filename, "x")
+ support.unlink(self.filename)
+ with gzip.open(self.filename, "x") as f:
+ f.write(uncompressed)
+ with open(self.filename, "rb") as f:
+ file_data = gzip.decompress(f.read())
+ self.assertEqual(file_data, uncompressed)
+
def test_text_modes(self):
uncompressed = data1.decode("ascii") * 50
uncompressed_raw = uncompressed.replace("\n", os.linesep)
@@ -469,6 +518,8 @@ class TestOpen(BaseTest):
with self.assertRaises(ValueError):
gzip.open(self.filename, "wbt")
with self.assertRaises(ValueError):
+ gzip.open(self.filename, "xbt")
+ with self.assertRaises(ValueError):
gzip.open(self.filename, "rb", encoding="utf-8")
with self.assertRaises(ValueError):
gzip.open(self.filename, "rb", errors="ignore")
diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py
index 22ebbcc..f647c6f 100644
--- a/Lib/test/test_hash.py
+++ b/Lib/test/test_hash.py
@@ -12,6 +12,40 @@ from collections import Hashable
IS_64BIT = sys.maxsize > 2**32
+def lcg(x, length=16):
+ """Linear congruential generator"""
+ if x == 0:
+ return bytes(length)
+ out = bytearray(length)
+ for i in range(length):
+ x = (214013 * x + 2531011) & 0x7fffffff
+ out[i] = (x >> 16) & 0xff
+ return bytes(out)
+
+def pysiphash(uint64):
+ """Convert SipHash24 output to Py_hash_t
+ """
+ assert 0 <= uint64 < (1 << 64)
+ # simple unsigned to signed int64
+ if uint64 > (1 << 63) - 1:
+ int64 = uint64 - (1 << 64)
+ else:
+ int64 = uint64
+ # mangle uint64 to uint32
+ uint32 = (uint64 ^ uint64 >> 32) & 0xffffffff
+ # simple unsigned to signed int32
+ if uint32 > (1 << 31) - 1:
+ int32 = uint32 - (1 << 32)
+ else:
+ int32 = uint32
+ return int32, int64
+
+def skip_unless_internalhash(test):
+ """Skip decorator for tests that depend on SipHash24 or FNV"""
+ ok = sys.hash_info.algorithm in {"fnv", "siphash24"}
+ msg = "Requires SipHash24 or FNV"
+ return test if ok else unittest.skip(msg)(test)
+
class HashEqualityTestCase(unittest.TestCase):
@@ -161,12 +195,64 @@ class HashRandomizationTests:
self.assertNotEqual(run1, run2)
class StringlikeHashRandomizationTests(HashRandomizationTests):
+ repr_ = None
+ repr_long = None
+
+ # 32bit little, 64bit little, 32bit big, 64bit big
+ known_hashes = {
+ 'djba33x': [ # only used for small strings
+ # seed 0, 'abc'
+ [193485960, 193485960, 193485960, 193485960],
+ # seed 42, 'abc'
+ [-678966196, 573763426263223372, -820489388, -4282905804826039665],
+ ],
+ 'siphash24': [
+ # NOTE: PyUCS2 layout depends on endianess
+ # seed 0, 'abc'
+ [1198583518, 4596069200710135518, 1198583518, 4596069200710135518],
+ # seed 42, 'abc'
+ [273876886, -4501618152524544106, 273876886, -4501618152524544106],
+ # seed 42, 'abcdefghijk'
+ [-1745215313, 4436719588892876975, -1745215313, 4436719588892876975],
+ # seed 0, 'äú∑ℇ'
+ [493570806, 5749986484189612790, -1006381564, -5915111450199468540],
+ # seed 42, 'äú∑ℇ'
+ [-1677110816, -2947981342227738144, -1860207793, -4296699217652516017],
+ ],
+ 'fnv': [
+ # seed 0, 'abc'
+ [-1600925533, 1453079729188098211, -1600925533,
+ 1453079729188098211],
+ # seed 42, 'abc'
+ [-206076799, -4410911502303878509, -1024014457,
+ -3570150969479994130],
+ # seed 42, 'abcdefghijk'
+ [811136751, -5046230049376118746, -77208053 ,
+ -4779029615281019666],
+ # seed 0, 'äú∑ℇ'
+ [44402817, 8998297579845987431, -1956240331,
+ -782697888614047887],
+ # seed 42, 'äú∑ℇ'
+ [-283066365, -4576729883824601543, -271871407,
+ -3927695501187247084],
+ ]
+ }
+
+ def get_expected_hash(self, position, length):
+ if length < sys.hash_info.cutoff:
+ algorithm = "djba33x"
+ else:
+ algorithm = sys.hash_info.algorithm
+ if sys.byteorder == 'little':
+ platform = 1 if IS_64BIT else 0
+ else:
+ assert(sys.byteorder == 'big')
+ platform = 3 if IS_64BIT else 2
+ return self.known_hashes[algorithm][position][platform]
+
def test_null_hash(self):
# PYTHONHASHSEED=0 disables the randomized hash
- if IS_64BIT:
- known_hash_of_obj = 1453079729188098211
- else:
- known_hash_of_obj = -1600925533
+ known_hash_of_obj = self.get_expected_hash(0, 3)
# Randomization is enabled by default:
self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
@@ -174,39 +260,53 @@ class StringlikeHashRandomizationTests(HashRandomizationTests):
# It can also be disabled by setting the seed to 0:
self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
+ @skip_unless_internalhash
def test_fixed_hash(self):
# test a fixed seed for the randomized hash
# Note that all types share the same values:
- if IS_64BIT:
- if sys.byteorder == 'little':
- h = -4410911502303878509
- else:
- h = -3570150969479994130
- else:
- if sys.byteorder == 'little':
- h = -206076799
- else:
- h = -1024014457
+ h = self.get_expected_hash(1, 3)
self.assertEqual(self.get_hash(self.repr_, seed=42), h)
+ @skip_unless_internalhash
+ def test_long_fixed_hash(self):
+ if self.repr_long is None:
+ return
+ h = self.get_expected_hash(2, 11)
+ self.assertEqual(self.get_hash(self.repr_long, seed=42), h)
+
+
class StrHashRandomizationTests(StringlikeHashRandomizationTests,
unittest.TestCase):
repr_ = repr('abc')
+ repr_long = repr('abcdefghijk')
+ repr_ucs2 = repr('äú∑ℇ')
+ @skip_unless_internalhash
def test_empty_string(self):
self.assertEqual(hash(""), 0)
+ @skip_unless_internalhash
+ def test_ucs2_string(self):
+ h = self.get_expected_hash(3, 6)
+ self.assertEqual(self.get_hash(self.repr_ucs2, seed=0), h)
+ h = self.get_expected_hash(4, 6)
+ self.assertEqual(self.get_hash(self.repr_ucs2, seed=42), h)
+
class BytesHashRandomizationTests(StringlikeHashRandomizationTests,
unittest.TestCase):
repr_ = repr(b'abc')
+ repr_long = repr(b'abcdefghijk')
+ @skip_unless_internalhash
def test_empty_string(self):
self.assertEqual(hash(b""), 0)
class MemoryviewHashRandomizationTests(StringlikeHashRandomizationTests,
unittest.TestCase):
repr_ = "memoryview(b'abc')"
+ repr_long = "memoryview(b'abcdefghijk')"
+ @skip_unless_internalhash
def test_empty_string(self):
self.assertEqual(hash(memoryview(b"")), 0)
@@ -224,5 +324,23 @@ class DatetimeTimeTests(DatetimeTests, unittest.TestCase):
repr_ = repr(datetime.time(0))
+class HashDistributionTestCase(unittest.TestCase):
+
+ def test_hash_distribution(self):
+ # check for hash collision
+ base = "abcdefghabcdefg"
+ for i in range(1, len(base)):
+ prefix = base[:i]
+ with self.subTest(prefix=prefix):
+ s15 = set()
+ s255 = set()
+ for c in range(256):
+ h = hash(prefix + chr(c))
+ s15.add(h & 0xf)
+ s255.add(h & 0xff)
+ # SipHash24 distribution depends on key, usually > 60%
+ self.assertGreater(len(s15), 8, prefix)
+ self.assertGreater(len(s255), 128, prefix)
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index c0470d6..36a98cb 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -18,11 +18,13 @@ except ImportError:
import unittest
import warnings
from test import support
-from test.support import _4G, bigmemtest
+from test.support import _4G, bigmemtest, import_fresh_module
# Were we compiled --with-pydebug or with #define Py_DEBUG?
COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
+c_hashlib = import_fresh_module('hashlib', fresh=['_hashlib'])
+py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
def hexstr(s):
assert isinstance(s, bytes), repr(s)
@@ -36,7 +38,7 @@ def hexstr(s):
class HashLibTestCase(unittest.TestCase):
supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1',
'sha224', 'SHA224', 'sha256', 'SHA256',
- 'sha384', 'SHA384', 'sha512', 'SHA512' )
+ 'sha384', 'SHA384', 'sha512', 'SHA512')
# Issue #14693: fallback modules are always compiled under POSIX
_warn_on_extension_import = os.name == 'posix' or COMPILED_WITH_PYDEBUG
@@ -72,27 +74,31 @@ class HashLibTestCase(unittest.TestCase):
if _hashlib:
# These two algorithms should always be present when this module
# is compiled. If not, something was compiled wrong.
- assert hasattr(_hashlib, 'openssl_md5')
- assert hasattr(_hashlib, 'openssl_sha1')
+ self.assertTrue(hasattr(_hashlib, 'openssl_md5'))
+ self.assertTrue(hasattr(_hashlib, 'openssl_sha1'))
for algorithm, constructors in self.constructors_to_test.items():
constructor = getattr(_hashlib, 'openssl_'+algorithm, None)
if constructor:
constructors.add(constructor)
+ def add_builtin_constructor(name):
+ constructor = getattr(hashlib, "__get_builtin_constructor")(name)
+ self.constructors_to_test[name].add(constructor)
+
_md5 = self._conditional_import_module('_md5')
if _md5:
- self.constructors_to_test['md5'].add(_md5.md5)
+ add_builtin_constructor('md5')
_sha1 = self._conditional_import_module('_sha1')
if _sha1:
- self.constructors_to_test['sha1'].add(_sha1.sha1)
+ add_builtin_constructor('sha1')
_sha256 = self._conditional_import_module('_sha256')
if _sha256:
- self.constructors_to_test['sha224'].add(_sha256.sha224)
- self.constructors_to_test['sha256'].add(_sha256.sha256)
+ add_builtin_constructor('sha224')
+ add_builtin_constructor('sha256')
_sha512 = self._conditional_import_module('_sha512')
if _sha512:
- self.constructors_to_test['sha384'].add(_sha512.sha384)
- self.constructors_to_test['sha512'].add(_sha512.sha512)
+ add_builtin_constructor('sha384')
+ add_builtin_constructor('sha512')
super(HashLibTestCase, self).__init__(*args, **kwargs)
@@ -121,8 +127,10 @@ class HashLibTestCase(unittest.TestCase):
self.assertRaises(TypeError, hashlib.new, 1)
def test_get_builtin_constructor(self):
- get_builtin_constructor = hashlib.__dict__[
- '__get_builtin_constructor']
+ get_builtin_constructor = getattr(hashlib,
+ '__get_builtin_constructor')
+ builtin_constructor_cache = getattr(hashlib,
+ '__builtin_constructor_cache')
self.assertRaises(ValueError, get_builtin_constructor, 'test')
try:
import _md5
@@ -130,6 +138,8 @@ class HashLibTestCase(unittest.TestCase):
pass
# This forces an ImportError for "import _md5" statements
sys.modules['_md5'] = None
+ # clear the cache
+ builtin_constructor_cache.clear()
try:
self.assertRaises(ValueError, get_builtin_constructor, 'md5')
finally:
@@ -138,13 +148,23 @@ class HashLibTestCase(unittest.TestCase):
else:
del sys.modules['_md5']
self.assertRaises(TypeError, get_builtin_constructor, 3)
+ constructor = get_builtin_constructor('md5')
+ self.assertIs(constructor, _md5.md5)
+ self.assertEqual(sorted(builtin_constructor_cache), ['MD5', 'md5'])
def test_hexdigest(self):
for cons in self.hash_constructors:
h = cons()
- assert isinstance(h.digest(), bytes), name
+ self.assertIsInstance(h.digest(), bytes)
self.assertEqual(hexstr(h.digest()), h.hexdigest())
+ def test_name_attribute(self):
+ for cons in self.hash_constructors:
+ h = cons()
+ self.assertIsInstance(h.name, str)
+ self.assertIn(h.name, self.supported_hash_names)
+ self.assertEqual(h.name, hashlib.new(h.name).name)
+
def test_large_update(self):
aas = b'a' * 128
bees = b'b' * 127
@@ -213,8 +233,9 @@ class HashLibTestCase(unittest.TestCase):
self.assertEqual(m.block_size, block_size)
self.assertEqual(m.digest_size, digest_size)
self.assertEqual(len(m.digest()), digest_size)
- self.assertEqual(m.name.lower(), name.lower())
- self.assertIn(name.split("_")[0], repr(m).lower())
+ self.assertEqual(m.name, name)
+ # split for sha3_512 / _sha3.sha3 object
+ self.assertIn(name.split("_")[0], repr(m))
def test_blocksize_name(self):
self.check_blocksize_name('md5', 64, 16)
@@ -352,6 +373,12 @@ class HashLibTestCase(unittest.TestCase):
"e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+
"de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b")
+ @unittest.skipIf(sys.maxsize < _4G + 5, 'test cannot run on 32-bit systems')
+ @bigmemtest(size=_4G + 5, memuse=1, dry_run=False)
+ def test_case_sha3_224_huge(self, size):
+ self.check('sha3_224', b'A'*size,
+ '58ef60057c9dddb6a87477e9ace5a26f0d9db01881cf9b10a9f8c224')
+
def test_gil(self):
# Check things work fine with an input larger than the size required
# for multithreaded operation (which is hardwired to 2048).
@@ -400,8 +427,8 @@ class HashLibTestCase(unittest.TestCase):
events = []
for threadnum in range(num_threads):
chunk_size = len(data) // (10**threadnum)
- assert chunk_size > 0
- assert chunk_size % len(smallest_data) == 0
+ self.assertGreater(chunk_size, 0)
+ self.assertEqual(chunk_size % len(smallest_data), 0)
event = threading.Event()
events.append(event)
threading.Thread(target=hash_in_chunks,
@@ -412,8 +439,95 @@ class HashLibTestCase(unittest.TestCase):
self.assertEqual(expected_hash, hasher.hexdigest())
-def test_main():
- support.run_unittest(HashLibTestCase)
+
+class KDFTests(unittest.TestCase):
+
+ pbkdf2_test_vectors = [
+ (b'password', b'salt', 1, None),
+ (b'password', b'salt', 2, None),
+ (b'password', b'salt', 4096, None),
+ # too slow, it takes over a minute on a fast CPU.
+ #(b'password', b'salt', 16777216, None),
+ (b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt',
+ 4096, -1),
+ (b'pass\0word', b'sa\0lt', 4096, 16),
+ ]
+
+ pbkdf2_results = {
+ "sha1": [
+ # offical test vectors from RFC 6070
+ (bytes.fromhex('0c60c80f961f0e71f3a9b524af6012062fe037a6'), None),
+ (bytes.fromhex('ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957'), None),
+ (bytes.fromhex('4b007901b765489abead49d926f721d065a429c1'), None),
+ #(bytes.fromhex('eefe3d61cd4da4e4e9945b3d6ba2158c2634e984'), None),
+ (bytes.fromhex('3d2eec4fe41c849b80c8d83662c0e44a8b291a964c'
+ 'f2f07038'), 25),
+ (bytes.fromhex('56fa6aa75548099dcc37d7f03425e0c3'), None),],
+ "sha256": [
+ (bytes.fromhex('120fb6cffcf8b32c43e7225256c4f837'
+ 'a86548c92ccc35480805987cb70be17b'), None),
+ (bytes.fromhex('ae4d0c95af6b46d32d0adff928f06dd0'
+ '2a303f8ef3c251dfd6e2d85a95474c43'), None),
+ (bytes.fromhex('c5e478d59288c841aa530db6845c4c8d'
+ '962893a001ce4e11a4963873aa98134a'), None),
+ #(bytes.fromhex('cf81c66fe8cfc04d1f31ecb65dab4089'
+ # 'f7f179e89b3b0bcb17ad10e3ac6eba46'), None),
+ (bytes.fromhex('348c89dbcbd32b2f32d814b8116e84cf2b17'
+ '347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9'), 40),
+ (bytes.fromhex('89b69d0516f829893c696226650a8687'), None),],
+ "sha512": [
+ (bytes.fromhex('867f70cf1ade02cff3752599a3a53dc4af34c7a669815ae5'
+ 'd513554e1c8cf252c02d470a285a0501bad999bfe943c08f'
+ '050235d7d68b1da55e63f73b60a57fce'), None),
+ (bytes.fromhex('e1d9c16aa681708a45f5c7c4e215ceb66e011a2e9f004071'
+ '3f18aefdb866d53cf76cab2868a39b9f7840edce4fef5a82'
+ 'be67335c77a6068e04112754f27ccf4e'), None),
+ (bytes.fromhex('d197b1b33db0143e018b12f3d1d1479e6cdebdcc97c5c0f8'
+ '7f6902e072f457b5143f30602641b3d55cd335988cb36b84'
+ '376060ecd532e039b742a239434af2d5'), None),
+ (bytes.fromhex('8c0511f4c6e597c6ac6315d8f0362e225f3c501495ba23b8'
+ '68c005174dc4ee71115b59f9e60cd9532fa33e0f75aefe30'
+ '225c583a186cd82bd4daea9724a3d3b8'), 64),
+ (bytes.fromhex('9d9e9c4cd21fe4be24d5b8244c759665'), None),],
+ }
+
+ def _test_pbkdf2_hmac(self, pbkdf2):
+ for digest_name, results in self.pbkdf2_results.items():
+ for i, vector in enumerate(self.pbkdf2_test_vectors):
+ password, salt, rounds, dklen = vector
+ expected, overwrite_dklen = results[i]
+ if overwrite_dklen:
+ dklen = overwrite_dklen
+ out = pbkdf2(digest_name, password, salt, rounds, dklen)
+ self.assertEqual(out, expected,
+ (digest_name, password, salt, rounds, dklen))
+ out = pbkdf2(digest_name, memoryview(password),
+ memoryview(salt), rounds, dklen)
+ out = pbkdf2(digest_name, bytearray(password),
+ bytearray(salt), rounds, dklen)
+ self.assertEqual(out, expected)
+ if dklen is None:
+ out = pbkdf2(digest_name, password, salt, rounds)
+ self.assertEqual(out, expected,
+ (digest_name, password, salt, rounds))
+
+ self.assertRaises(TypeError, pbkdf2, b'sha1', b'pass', b'salt', 1)
+ self.assertRaises(TypeError, pbkdf2, 'sha1', 'pass', 'salt', 1)
+ self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 0)
+ self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', -1)
+ self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, 0)
+ self.assertRaises(ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, -1)
+ with self.assertRaisesRegex(ValueError, 'unsupported hash type'):
+ pbkdf2('unknown', b'pass', b'salt', 1)
+
+ def test_pbkdf2_hmac_py(self):
+ self._test_pbkdf2_hmac(py_hashlib.pbkdf2_hmac)
+
+ @unittest.skipUnless(hasattr(c_hashlib, 'pbkdf2_hmac'),
+ ' test requires OpenSSL > 1.0')
+ def test_pbkdf2_hmac_c(self):
+ self._test_pbkdf2_hmac(c_hashlib.pbkdf2_hmac)
+
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 4ca7cec..cde56fd 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -1,17 +1,39 @@
+import functools
import hmac
import hashlib
import unittest
import warnings
from test import support
+
+def ignore_warning(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.filterwarnings("ignore",
+ category=PendingDeprecationWarning)
+ return func(*args, **kwargs)
+ return wrapper
+
+
class TestVectorsTestCase(unittest.TestCase):
def test_md5_vectors(self):
# Test the HMAC module against test vectors from the RFC.
def md5test(key, data, digest):
- h = hmac.HMAC(key, data)
+ h = hmac.HMAC(key, data, digestmod=hashlib.md5)
self.assertEqual(h.hexdigest().upper(), digest.upper())
+ self.assertEqual(h.name, "hmac-md5")
+ self.assertEqual(h.digest_size, 16)
+ self.assertEqual(h.block_size, 64)
+
+ h = hmac.HMAC(key, data, digestmod='md5')
+ self.assertEqual(h.hexdigest().upper(), digest.upper())
+ self.assertEqual(h.name, "hmac-md5")
+ self.assertEqual(h.digest_size, 16)
+ self.assertEqual(h.block_size, 64)
+
md5test(b"\x0b" * 16,
b"Hi There",
@@ -46,6 +68,16 @@ class TestVectorsTestCase(unittest.TestCase):
def shatest(key, data, digest):
h = hmac.HMAC(key, data, digestmod=hashlib.sha1)
self.assertEqual(h.hexdigest().upper(), digest.upper())
+ self.assertEqual(h.name, "hmac-sha1")
+ self.assertEqual(h.digest_size, 20)
+ self.assertEqual(h.block_size, 64)
+
+ h = hmac.HMAC(key, data, digestmod='sha1')
+ self.assertEqual(h.hexdigest().upper(), digest.upper())
+ self.assertEqual(h.name, "hmac-sha1")
+ self.assertEqual(h.digest_size, 20)
+ self.assertEqual(h.block_size, 64)
+
shatest(b"\x0b" * 20,
b"Hi There",
@@ -76,10 +108,21 @@ class TestVectorsTestCase(unittest.TestCase):
b"and Larger Than One Block-Size Data"),
"e8e99d0f45237d786d6bbaa7965c7808bbff1a91")
- def _rfc4231_test_cases(self, hashfunc):
+ def _rfc4231_test_cases(self, hashfunc, hash_name, digest_size, block_size):
def hmactest(key, data, hexdigests):
+ hmac_name = "hmac-" + hash_name
h = hmac.HMAC(key, data, digestmod=hashfunc)
self.assertEqual(h.hexdigest().lower(), hexdigests[hashfunc])
+ self.assertEqual(h.name, hmac_name)
+ self.assertEqual(h.digest_size, digest_size)
+ self.assertEqual(h.block_size, block_size)
+
+ h = hmac.HMAC(key, data, digestmod=hash_name)
+ self.assertEqual(h.hexdigest().lower(), hexdigests[hashfunc])
+ self.assertEqual(h.name, hmac_name)
+ self.assertEqual(h.digest_size, digest_size)
+ self.assertEqual(h.block_size, block_size)
+
# 4.2. Test Case 1
hmactest(key = b'\x0b'*20,
@@ -189,16 +232,16 @@ class TestVectorsTestCase(unittest.TestCase):
})
def test_sha224_rfc4231(self):
- self._rfc4231_test_cases(hashlib.sha224)
+ self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64)
def test_sha256_rfc4231(self):
- self._rfc4231_test_cases(hashlib.sha256)
+ self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64)
def test_sha384_rfc4231(self):
- self._rfc4231_test_cases(hashlib.sha384)
+ self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128)
def test_sha512_rfc4231(self):
- self._rfc4231_test_cases(hashlib.sha512)
+ self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128)
def test_legacy_block_size_warnings(self):
class MockCrazyHash(object):
@@ -222,46 +265,74 @@ class TestVectorsTestCase(unittest.TestCase):
hmac.HMAC(b'a', b'b', digestmod=MockCrazyHash)
self.fail('Expected warning about small block_size')
+ def test_with_digestmod_warning(self):
+ with self.assertWarns(PendingDeprecationWarning):
+ key = b"\x0b" * 16
+ data = b"Hi There"
+ digest = "9294727A3638BB1C13F48EF8158BFC9D"
+ h = hmac.HMAC(key, data)
+ self.assertEqual(h.hexdigest().upper(), digest)
class ConstructorTestCase(unittest.TestCase):
+ @ignore_warning
def test_normal(self):
# Standard constructor call.
failed = 0
try:
h = hmac.HMAC(b"key")
- except:
+ except Exception:
self.fail("Standard constructor call raised exception.")
+ @ignore_warning
def test_with_str_key(self):
# Pass a key of type str, which is an error, because it expects a key
# of type bytes
with self.assertRaises(TypeError):
h = hmac.HMAC("key")
+ @ignore_warning
def test_dot_new_with_str_key(self):
# Pass a key of type str, which is an error, because it expects a key
# of type bytes
with self.assertRaises(TypeError):
h = hmac.new("key")
+ @ignore_warning
def test_withtext(self):
# Constructor call with text.
try:
h = hmac.HMAC(b"key", b"hash this!")
- except:
+ except Exception:
self.fail("Constructor call with text argument raised exception.")
+ self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+
+ def test_with_bytearray(self):
+ try:
+ h = hmac.HMAC(bytearray(b"key"), bytearray(b"hash this!"),
+ digestmod="md5")
+ except Exception:
+ self.fail("Constructor call with bytearray arguments raised exception.")
+ self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
+
+ def test_with_memoryview_msg(self):
+ try:
+ h = hmac.HMAC(b"key", memoryview(b"hash this!"), digestmod="md5")
+ except Exception:
+ self.fail("Constructor call with memoryview msg raised exception.")
+ self.assertEqual(h.hexdigest(), '34325b639da4cfd95735b381e28cb864')
def test_withmodule(self):
# Constructor call with text and digest module.
try:
h = hmac.HMAC(b"key", b"", hashlib.sha1)
- except:
+ except Exception:
self.fail("Constructor call with hashlib.sha1 raised exception.")
class SanityTestCase(unittest.TestCase):
+ @ignore_warning
def test_default_is_md5(self):
# Testing if HMAC defaults to MD5 algorithm.
# NOTE: this whitebox test depends on the hmac class internals
@@ -272,19 +343,19 @@ class SanityTestCase(unittest.TestCase):
# Exercising all methods once.
# This must not raise any exceptions
try:
- h = hmac.HMAC(b"my secret key")
+ h = hmac.HMAC(b"my secret key", digestmod="md5")
h.update(b"compute the hash of this text!")
dig = h.digest()
dig = h.hexdigest()
h2 = h.copy()
- except:
+ except Exception:
self.fail("Exception raised during normal usage of HMAC class.")
class CopyTestCase(unittest.TestCase):
def test_attributes(self):
# Testing if attributes are of same type.
- h1 = hmac.HMAC(b"key")
+ h1 = hmac.HMAC(b"key", digestmod="md5")
h2 = h1.copy()
self.assertTrue(h1.digest_cons == h2.digest_cons,
"digest constructors don't match.")
@@ -295,7 +366,7 @@ class CopyTestCase(unittest.TestCase):
def test_realcopy(self):
# Testing if the copy method created a real copy.
- h1 = hmac.HMAC(b"key")
+ h1 = hmac.HMAC(b"key", digestmod="md5")
h2 = h1.copy()
# Using id() in case somebody has overridden __eq__/__ne__.
self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
@@ -306,7 +377,7 @@ class CopyTestCase(unittest.TestCase):
def test_equality(self):
# Testing if the copy has the same digests.
- h1 = hmac.HMAC(b"key")
+ h1 = hmac.HMAC(b"key", digestmod="md5")
h1.update(b"some random text")
h2 = h1.copy()
self.assertEqual(h1.digest(), h2.digest(),
diff --git a/Lib/test/test_html.py b/Lib/test/test_html.py
index 30dac58..5e9f382 100644
--- a/Lib/test/test_html.py
+++ b/Lib/test/test_html.py
@@ -16,9 +16,89 @@ class HtmlTests(unittest.TestCase):
html.escape('\'<script>"&foo;"</script>\'', False),
'\'&lt;script&gt;"&amp;foo;"&lt;/script&gt;\'')
+ def test_unescape(self):
+ numeric_formats = ['&#%d', '&#%d;', '&#x%x', '&#x%x;']
+ errmsg = 'unescape(%r) should have returned %r'
+ def check(text, expected):
+ self.assertEqual(html.unescape(text), expected,
+ msg=errmsg % (text, expected))
+ def check_num(num, expected):
+ for format in numeric_formats:
+ text = format % num
+ self.assertEqual(html.unescape(text), expected,
+ msg=errmsg % (text, expected))
+ # check text with no character references
+ check('no character references', 'no character references')
+ # check & followed by invalid chars
+ check('&\n&\t& &&', '&\n&\t& &&')
+ # check & followed by numbers and letters
+ check('&0 &9 &a &0; &9; &a;', '&0 &9 &a &0; &9; &a;')
+ # check incomplete entities at the end of the string
+ for x in ['&', '&#', '&#x', '&#X', '&#y', '&#xy', '&#Xy']:
+ check(x, x)
+ check(x+';', x+';')
+ # check several combinations of numeric character references,
+ # possibly followed by different characters
+ formats = ['&#%d', '&#%07d', '&#%d;', '&#%07d;',
+ '&#x%x', '&#x%06x', '&#x%x;', '&#x%06x;',
+ '&#x%X', '&#x%06X', '&#X%x;', '&#X%06x;']
+ for num, char in zip([65, 97, 34, 38, 0x2603, 0x101234],
+ ['A', 'a', '"', '&', '\u2603', '\U00101234']):
+ for s in formats:
+ check(s % num, char)
+ for end in [' ', 'X']:
+ check((s+end) % num, char+end)
+ # check invalid codepoints
+ for cp in [0xD800, 0xDB00, 0xDC00, 0xDFFF, 0x110000]:
+ check_num(cp, '\uFFFD')
+ # check more invalid codepoints
+ for cp in [0x1, 0xb, 0xe, 0x7f, 0xfffe, 0xffff, 0x10fffe, 0x10ffff]:
+ check_num(cp, '')
+ # check invalid numbers
+ for num, ch in zip([0x0d, 0x80, 0x95, 0x9d], '\r\u20ac\u2022\x9d'):
+ check_num(num, ch)
+ # check small numbers
+ check_num(0, '\uFFFD')
+ check_num(9, '\t')
+ # check a big number
+ check_num(1000000000000000000, '\uFFFD')
+ # check that multiple trailing semicolons are handled correctly
+ for e in ['&quot;;', '&#34;;', '&#x22;;', '&#X22;;']:
+ check(e, '";')
+ # check that semicolons in the middle don't create problems
+ for e in ['&quot;quot;', '&#34;quot;', '&#x22;quot;', '&#X22;quot;']:
+ check(e, '"quot;')
+ # check triple adjacent charrefs
+ for e in ['&quot', '&#34', '&#x22', '&#X22']:
+ check(e*3, '"""')
+ check((e+';')*3, '"""')
+ # check that the case is respected
+ for e in ['&amp', '&amp;', '&AMP', '&AMP;']:
+ check(e, '&')
+ for e in ['&Amp', '&Amp;']:
+ check(e, e)
+ # check that non-existent named entities are returned unchanged
+ check('&svadilfari;', '&svadilfari;')
+ # the following examples are in the html5 specs
+ check('&notit', '¬it')
+ check('&notit;', '¬it;')
+ check('&notin', '¬in')
+ check('&notin;', '∉')
+ # a similar example with a long name
+ check('&notReallyAnExistingNamedCharacterReference;',
+ '¬ReallyAnExistingNamedCharacterReference;')
+ # longest valid name
+ check('&CounterClockwiseContourIntegral;', '∳')
+ # check a charref that maps to two unicode chars
+ check('&acE;', '\u223E\u0333')
+ check('&acE', '&acE')
+ # see #12888
+ check('&#123; ' * 1050, '{ ' * 1050)
+ # see #15156
+ check('&Eacuteric&Eacute;ric&alphacentauri&alpha;centauri',
+ 'ÉricÉric&alphacentauriαcentauri')
+ check('&co;', '&co;')
-def test_main():
- run_unittest(HtmlTests)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py
index 11d9c9c..2d771a2 100644
--- a/Lib/test/test_htmlparser.py
+++ b/Lib/test/test_htmlparser.py
@@ -70,6 +70,18 @@ class EventCollectorExtra(EventCollector):
self.append(("starttag_text", self.get_starttag_text()))
+class EventCollectorCharrefs(EventCollector):
+
+ def get_events(self):
+ return self.events
+
+ def handle_charref(self, data):
+ self.fail('This should never be called with convert_charrefs=True')
+
+ def handle_entityref(self, data):
+ self.fail('This should never be called with convert_charrefs=True')
+
+
class TestCaseBase(unittest.TestCase):
def get_collector(self):
@@ -84,26 +96,30 @@ class TestCaseBase(unittest.TestCase):
parser.close()
events = parser.get_events()
if events != expected_events:
- self.fail("received events did not match expected events\n"
- "Expected:\n" + pprint.pformat(expected_events) +
+ self.fail("received events did not match expected events" +
+ "\nSource:\n" + repr(source) +
+ "\nExpected:\n" + pprint.pformat(expected_events) +
"\nReceived:\n" + pprint.pformat(events))
def _run_check_extra(self, source, events):
- self._run_check(source, events, EventCollectorExtra())
+ self._run_check(source, events,
+ EventCollectorExtra(convert_charrefs=False))
def _parse_error(self, source):
def parse(source=source):
parser = self.get_collector()
parser.feed(source)
parser.close()
- self.assertRaises(html.parser.HTMLParseError, parse)
+ with self.assertRaises(html.parser.HTMLParseError):
+ with self.assertWarns(DeprecationWarning):
+ parse()
class HTMLParserStrictTestCase(TestCaseBase):
def get_collector(self):
with support.check_warnings(("", DeprecationWarning), quite=False):
- return EventCollector(strict=True)
+ return EventCollector(strict=True, convert_charrefs=False)
def test_processing_instruction_only(self):
self._run_check("<?processing instruction>", [
@@ -339,7 +355,7 @@ text
self._run_check(s, [("starttag", element_lower, []),
("data", content),
("endtag", element_lower)],
- collector=Collector())
+ collector=Collector(convert_charrefs=False))
def test_comments(self):
html = ("<!-- I'm a valid comment -->"
@@ -367,11 +383,60 @@ text
('comment', '[if lte IE 7]>pretty?<![endif]')]
self._run_check(html, expected)
+ def test_convert_charrefs(self):
+ collector = lambda: EventCollectorCharrefs(convert_charrefs=True)
+ self.assertTrue(collector().convert_charrefs)
+ charrefs = ['&quot;', '&#34;', '&#x22;', '&quot', '&#34', '&#x22']
+ # check charrefs in the middle of the text/attributes
+ expected = [('starttag', 'a', [('href', 'foo"zar')]),
+ ('data', 'a"z'), ('endtag', 'a')]
+ for charref in charrefs:
+ self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref),
+ expected, collector=collector())
+ # check charrefs at the beginning/end of the text/attributes
+ expected = [('data', '"'),
+ ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]),
+ ('data', '"'), ('endtag', 'a'), ('data', '"')]
+ for charref in charrefs:
+ self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">'
+ '{0}</a>{0}'.format(charref),
+ expected, collector=collector())
+ # check charrefs in <script>/<style> elements
+ for charref in charrefs:
+ text = 'X'.join([charref]*3)
+ expected = [('data', '"'),
+ ('starttag', 'script', []), ('data', text),
+ ('endtag', 'script'), ('data', '"'),
+ ('starttag', 'style', []), ('data', text),
+ ('endtag', 'style'), ('data', '"')]
+ self._run_check('{1}<script>{0}</script>{1}'
+ '<style>{0}</style>{1}'.format(text, charref),
+ expected, collector=collector())
+ # check truncated charrefs at the end of the file
+ html = '&quo &# &#x'
+ for x in range(1, len(html)):
+ self._run_check(html[:x], [('data', html[:x])],
+ collector=collector())
+ # check a string with no charrefs
+ self._run_check('no charrefs here', [('data', 'no charrefs here')],
+ collector=collector())
+
class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
def get_collector(self):
- return EventCollector(strict=False)
+ return EventCollector(convert_charrefs=False)
+
+ def test_deprecation_warnings(self):
+ with self.assertWarns(DeprecationWarning):
+ EventCollector() # convert_charrefs not passed explicitly
+ with self.assertWarns(DeprecationWarning):
+ EventCollector(strict=True)
+ with self.assertWarns(DeprecationWarning):
+ EventCollector(strict=False)
+ with self.assertRaises(html.parser.HTMLParseError):
+ with self.assertWarns(DeprecationWarning):
+ EventCollector().error('test')
def test_tolerant_parsing(self):
self._run_check('<html <html>te>>xt&a<<bc</a></html>\n'
@@ -564,17 +629,12 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase):
for html, expected in data:
self._run_check(html, expected)
- def test_unescape_function(self):
+ def test_unescape_method(self):
+ from html import unescape
p = self.get_collector()
- self.assertEqual(p.unescape('&#bad;'),'&#bad;')
- self.assertEqual(p.unescape('&#0038;'),'&')
- # see #12888
- self.assertEqual(p.unescape('&#123; ' * 1050), '{ ' * 1050)
- # see #15156
- self.assertEqual(p.unescape('&Eacuteric&Eacute;ric'
- '&alphacentauri&alpha;centauri'),
- 'ÉricÉric&alphacentauriαcentauri')
- self.assertEqual(p.unescape('&co;'), '&co;')
+ with self.assertWarns(DeprecationWarning):
+ s = '&quot;&#34;&#x22;&quot&#34&#x22&#bad;'
+ self.assertEqual(p.unescape(s), unescape(s))
def test_broken_comments(self):
html = ('<! not really a comment >'
@@ -630,7 +690,7 @@ class AttributesStrictTestCase(TestCaseBase):
def get_collector(self):
with support.check_warnings(("", DeprecationWarning), quite=False):
- return EventCollector(strict=True)
+ return EventCollector(strict=True, convert_charrefs=False)
def test_attr_syntax(self):
output = [
@@ -691,7 +751,7 @@ class AttributesStrictTestCase(TestCaseBase):
class AttributesTolerantTestCase(AttributesStrictTestCase):
def get_collector(self):
- return EventCollector(strict=False)
+ return EventCollector(convert_charrefs=False)
def test_attr_funky_names2(self):
self._run_check(
diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py
index c8ded92..30b6c0c 100644
--- a/Lib/test/test_httplib.py
+++ b/Lib/test/test_httplib.py
@@ -27,8 +27,10 @@ class FakeSocket:
self.text = text
self.fileclass = fileclass
self.data = b''
+ self.sendall_calls = 0
def sendall(self, data):
+ self.sendall_calls += 1
self.data += data
def makefile(self, mode, bufsize=None):
@@ -45,7 +47,7 @@ class EPipeSocket(FakeSocket):
def sendall(self, data):
if self.pipe_trigger in data:
- raise socket.error(errno.EPIPE, "gotcha")
+ raise OSError(errno.EPIPE, "gotcha")
self.data += data
def close(self):
@@ -600,7 +602,7 @@ class BasicTest(TestCase):
b"Content-Length")
conn = client.HTTPConnection("example.com")
conn.sock = sock
- self.assertRaises(socket.error,
+ self.assertRaises(OSError,
lambda: conn.request("PUT", "/url", "body"))
resp = conn.getresponse()
self.assertEqual(401, resp.status)
@@ -646,6 +648,28 @@ class BasicTest(TestCase):
resp.close()
self.assertTrue(resp.closed)
+ def test_delayed_ack_opt(self):
+ # Test that Nagle/delayed_ack optimistaion works correctly.
+
+ # For small payloads, it should coalesce the body with
+ # headers, resulting in a single sendall() call
+ conn = client.HTTPConnection('example.com')
+ sock = FakeSocket(None)
+ conn.sock = sock
+ body = b'x' * (conn.mss - 1)
+ conn.request('POST', '/', body)
+ self.assertEqual(sock.sendall_calls, 1)
+
+ # For large payloads, it should send the headers and
+ # then the body, resulting in more than one sendall()
+ # call
+ conn = client.HTTPConnection('example.com')
+ sock = FakeSocket(None)
+ conn.sock = sock
+ body = b'x' * conn.mss
+ conn.request('POST', '/', body)
+ self.assertGreater(sock.sendall_calls, 1)
+
class OfflineTest(TestCase):
def test_responses(self):
self.assertEqual(client.responses[client.NOT_FOUND], "Not Found")
@@ -736,7 +760,7 @@ class HTTPSTest(TestCase):
def make_server(self, certfile):
from test.ssl_servers import make_https_server
- return make_https_server(self, certfile)
+ return make_https_server(self, certfile=certfile)
def test_attributes(self):
# simple test to check it's storing the timeout
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index 4e62963..15694af 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -92,6 +92,13 @@ class BaseHTTPServerTestCase(BaseTestCase):
def do_KEYERROR(self):
self.send_error(999)
+ def do_NOTFOUND(self):
+ self.send_error(404)
+
+ def do_EXPLAINERROR(self):
+ self.send_error(999, "Short Message",
+ "This is a long \n explaination")
+
def do_CUSTOM(self):
self.send_response(999)
self.send_header('Content-Type', 'text/html')
@@ -203,6 +210,12 @@ class BaseHTTPServerTestCase(BaseTestCase):
res = self.con.getresponse()
self.assertEqual(res.status, 999)
+ def test_return_explain_error(self):
+ self.con.request('EXPLAINERROR', '/')
+ res = self.con.getresponse()
+ self.assertEqual(res.status, 999)
+ self.assertTrue(int(res.getheader('Content-Length')))
+
def test_latin1_header(self):
self.con.request('LATINONEHEADER', '/', headers={
'X-Special-Incoming': 'Ärger mit Unicode'
@@ -211,6 +224,14 @@ class BaseHTTPServerTestCase(BaseTestCase):
self.assertEqual(res.getheader('X-Special'), 'Dängerous Mind')
self.assertEqual(res.read(), 'Ärger mit Unicode'.encode('utf-8'))
+ def test_error_content_length(self):
+ # Issue #16088: standard error responses should have a content-length
+ self.con.request('NOTFOUND', '/')
+ res = self.con.getresponse()
+ self.assertEqual(res.status, 404)
+ data = res.read()
+ self.assertEqual(int(res.getheader('Content-Length')), len(data))
+
class SimpleHTTPServerTestCase(BaseTestCase):
class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler):
diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py
index 7c9afd9..5e0eaea 100644
--- a/Lib/test/test_imaplib.py
+++ b/Lib/test/test_imaplib.py
@@ -18,8 +18,12 @@ try:
import ssl
except ImportError:
ssl = None
+ HAS_SNI = False
+else:
+ from ssl import HAS_SNI
CERTFILE = None
+CAFILE = None
class TestImaplib(unittest.TestCase):
@@ -125,7 +129,7 @@ class SimpleIMAPHandler(socketserver.StreamRequestHandler):
# Naked sockets return empty strings..
return
line += part
- except IOError:
+ except OSError:
# ..but SSLSockets raise exceptions.
return
if line.endswith(b'\r\n'):
@@ -347,6 +351,26 @@ class ThreadedNetworkedTestsSSL(BaseThreadedNetworkedTests):
server_class = SecureTCPServer
imap_class = IMAP4_SSL
+ @reap_threads
+ @unittest.skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_ssl_verified(self):
+ ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+ ssl_context.verify_mode = ssl.CERT_REQUIRED
+ ssl_context.check_hostname = True
+ ssl_context.load_verify_locations(CAFILE)
+
+ with self.assertRaisesRegex(ssl.CertificateError,
+ "hostname '127.0.0.1' doesn't match 'localhost'"):
+ with self.reaped_server(SimpleIMAPHandler) as server:
+ client = self.imap_class(*server.server_address,
+ ssl_context=ssl_context)
+ client.shutdown()
+
+ with self.reaped_server(SimpleIMAPHandler) as server:
+ client = self.imap_class("localhost", server.server_address[1],
+ ssl_context=ssl_context)
+ client.shutdown()
+
class RemoteIMAPTest(unittest.TestCase):
host = 'cyrus.andrew.cmu.edu'
@@ -459,11 +483,15 @@ def load_tests(*args):
if support.is_resource_enabled('network'):
if ssl:
- global CERTFILE
+ global CERTFILE, CAFILE
CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir,
- "keycert.pem")
+ "keycert3.pem")
if not os.path.exists(CERTFILE):
raise support.TestFailed("Can't read certificate files!")
+ CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir,
+ "pycacert.pem")
+ if not os.path.exists(CAFILE):
+ raise support.TestFailed("Can't read CA file!")
tests.extend([
ThreadedNetworkedTests, ThreadedNetworkedTestsSSL,
RemoteIMAPTest, RemoteIMAP_SSLTest, RemoteIMAP_STARTTLSTest,
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index 71c220f..024f438 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -1,14 +1,29 @@
-import imp
+try:
+ import _thread
+except ImportError:
+ _thread = None
import importlib
import os
import os.path
import shutil
import sys
from test import support
-from test.test_importlib import util
import unittest
import warnings
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore', PendingDeprecationWarning)
+ import imp
+
+def requires_load_dynamic(meth):
+ """Decorator to skip a test if not running under CPython or lacking
+ imp.load_dynamic()."""
+ meth = support.cpython_only(meth)
+ return unittest.skipIf(not hasattr(imp, 'load_dynamic'),
+ 'imp.load_dynamic() required')(meth)
+
+
+@unittest.skipIf(_thread is None, '_thread module is required')
class LockTests(unittest.TestCase):
"""Very basic test of import lock functions."""
@@ -150,7 +165,7 @@ class ImportTests(unittest.TestCase):
self.assertIsNotNone(file)
self.assertTrue(filename[:-3].endswith(temp_mod_name))
self.assertEqual(info[0], '.py')
- self.assertEqual(info[1], 'U')
+ self.assertEqual(info[1], 'r')
self.assertEqual(info[2], imp.PY_SOURCE)
mod = imp.load_module(temp_mod_name, file, filename, info)
@@ -208,9 +223,7 @@ class ImportTests(unittest.TestCase):
self.assertIs(orig_path, new_os.path)
self.assertIsNot(orig_getenv, new_os.getenv)
- @support.cpython_only
- @unittest.skipIf(not hasattr(imp, 'load_dynamic'),
- 'imp.load_dynamic() required')
+ @requires_load_dynamic
def test_issue15828_load_extensions(self):
# Issue 15828 picked up that the adapter between the old imp API
# and importlib couldn't handle C extensions
@@ -222,6 +235,22 @@ class ImportTests(unittest.TestCase):
mod = imp.load_module(example, *x)
self.assertEqual(mod.__name__, example)
+ @requires_load_dynamic
+ def test_issue16421_multiple_modules_in_one_dll(self):
+ # Issue 16421: loading several modules from the same compiled file fails
+ m = '_testimportmultiple'
+ fileobj, pathname, description = imp.find_module(m)
+ fileobj.close()
+ mod0 = imp.load_dynamic(m, pathname)
+ mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname)
+ mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname)
+ self.assertEqual(mod0.__name__, m)
+ self.assertEqual(mod1.__name__, '_testimportmultiple_foo')
+ self.assertEqual(mod2.__name__, '_testimportmultiple_bar')
+ with self.assertRaises(ImportError):
+ imp.load_dynamic('nonexistent', pathname)
+
+ @requires_load_dynamic
def test_load_dynamic_ImportError_path(self):
# Issue #1559549 added `name` and `path` attributes to ImportError
# in order to provide better detail. Issue #10854 implemented those
@@ -233,14 +262,12 @@ class ImportTests(unittest.TestCase):
self.assertIn(path, err.exception.path)
self.assertEqual(name, err.exception.name)
- @support.cpython_only
- @unittest.skipIf(not hasattr(imp, 'load_dynamic'),
- 'imp.load_dynamic() required')
+ @requires_load_dynamic
def test_load_module_extension_file_is_None(self):
# When loading an extension module and the file is None, open one
# on the behalf of imp.load_dynamic().
# Issue #15902
- name = '_heapq'
+ name = '_testimportmultiple'
found = imp.find_module(name)
if found[0] is not None:
found[0].close()
@@ -248,6 +275,15 @@ class ImportTests(unittest.TestCase):
self.skipTest("found module doesn't appear to be a C extension")
imp.load_module(name, None, *found[1:])
+ @unittest.skipIf(sys.dont_write_bytecode,
+ "test meaningful only when writing bytecode")
+ def test_bug7732(self):
+ with support.temp_cwd():
+ source = support.TESTFN + '.py'
+ os.mkdir(source)
+ self.assertRaisesRegex(ImportError, '^No module',
+ imp.find_module, support.TESTFN, ["."])
+
def test_multiple_calls_to_get_data(self):
# Issue #18755: make sure multiple calls to get_data() can succeed.
loader = imp._LoadSourceCompatibility('imp', imp.__file__,
@@ -293,22 +329,6 @@ class ReloadTests(unittest.TestCase):
with self.assertRaisesRegex(ImportError, 'html'):
imp.reload(parser)
- def test_module_replaced(self):
- # see #18698
- def code():
- module = type(sys)('top_level')
- module.spam = 3
- sys.modules['top_level'] = module
- mock = util.mock_modules('top_level',
- module_code={'top_level': code})
- with mock:
- with util.import_state(meta_path=[mock]):
- module = importlib.import_module('top_level')
- reloaded = imp.reload(module)
- actual = sys.modules['top_level']
- self.assertEqual(actual.spam, 3)
- self.assertEqual(reloaded.spam, 3)
-
class PEP3147Tests(unittest.TestCase):
"""Tests of PEP 3147."""
@@ -461,20 +481,5 @@ class NullImporterTests(unittest.TestCase):
os.rmdir(name)
-def test_main():
- tests = [
- ImportTests,
- PEP3147Tests,
- ReloadTests,
- NullImporterTests,
- ]
- try:
- import _thread
- except ImportError:
- pass
- else:
- tests.append(LockTests)
- support.run_unittest(*tests)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index df08e6a..bda1541 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,9 +1,8 @@
# We import importlib *ASAP* in order to test #15386
import importlib
+import importlib.util
from importlib._bootstrap import _get_sourcefile
import builtins
-import imp
-from test.test_importlib.import_ import util as importlib_util
import marshal
import os
import platform
@@ -22,7 +21,7 @@ import test.support
from test.support import (
EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython,
make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask,
- unlink, unload, create_empty_file, cpython_only)
+ unlink, unload, create_empty_file, cpython_only, TESTFN_UNENCODABLE)
from test import script_helper
@@ -70,8 +69,6 @@ class ImportTests(unittest.TestCase):
def tearDown(self):
unload(TESTFN)
- setUp = tearDown
-
def test_case_sensitivity(self):
# Brief digression to test that import is case-sensitive: if we got
# this far, we know for sure that "random" exists.
@@ -129,16 +126,6 @@ class ImportTests(unittest.TestCase):
finally:
del sys.path[0]
- @skip_if_dont_write_bytecode
- def test_bug7732(self):
- source = TESTFN + '.py'
- os.mkdir(source)
- try:
- self.assertRaisesRegex(ImportError, '^No module',
- imp.find_module, TESTFN, ["."])
- finally:
- os.rmdir(source)
-
def test_module_with_large_stack(self, module='longlist'):
# Regression test for http://bugs.python.org/issue561858.
filename = module + '.py'
@@ -161,16 +148,24 @@ class ImportTests(unittest.TestCase):
sys.path.append('')
importlib.invalidate_caches()
+ namespace = {}
try:
make_legacy_pyc(filename)
# This used to crash.
- exec('import ' + module)
+ exec('import ' + module, None, namespace)
finally:
# Cleanup.
del sys.path[-1]
unlink(filename + 'c')
unlink(filename + 'o')
+ # Remove references to the module (unload the module)
+ namespace.clear()
+ try:
+ del sys.modules[module]
+ except KeyError:
+ pass
+
def test_failing_import_sticks(self):
source = TESTFN + ".py"
with open(source, "w") as f:
@@ -225,7 +220,7 @@ class ImportTests(unittest.TestCase):
with open(source, "w") as f:
f.write("a = 10\nb=20//0\n")
- self.assertRaises(ZeroDivisionError, imp.reload, mod)
+ self.assertRaises(ZeroDivisionError, importlib.reload, mod)
# But we still expect the module to be in sys.modules.
mod = sys.modules.get(TESTFN)
self.assertIsNot(mod, None, "expected module to be in sys.modules")
@@ -280,7 +275,7 @@ class ImportTests(unittest.TestCase):
import sys
class C:
def __del__(self):
- import imp
+ import importlib
sys.argv.insert(0, C())
"""))
script_helper.assert_python_ok(testfn)
@@ -291,7 +286,7 @@ class ImportTests(unittest.TestCase):
sys.path.insert(0, os.curdir)
try:
source = TESTFN + ".py"
- compiled = imp.cache_from_source(source)
+ compiled = importlib.util.cache_from_source(source)
with open(source, 'w') as f:
pass
try:
@@ -322,6 +317,14 @@ class ImportTests(unittest.TestCase):
stdout, stderr = popen.communicate()
self.assertIn(b"ImportError", stdout)
+ def test_from_import_message_for_nonexistent_module(self):
+ with self.assertRaisesRegex(ImportError, "^No module named 'bogus'"):
+ from bogus import foo
+
+ def test_from_import_message_for_existing_module(self):
+ with self.assertRaisesRegex(ImportError, "^cannot import name 'bogus'"):
+ from re import bogus
+
@skip_if_dont_write_bytecode
class FilePermissionTests(unittest.TestCase):
@@ -332,7 +335,7 @@ class FilePermissionTests(unittest.TestCase):
def test_creation_mode(self):
mask = 0o022
with temp_umask(mask), _ready_to_import() as (name, path):
- cached_path = imp.cache_from_source(path)
+ cached_path = importlib.util.cache_from_source(path)
module = __import__(name)
if not os.path.exists(cached_path):
self.fail("__import__ did not result in creation of "
@@ -350,7 +353,7 @@ class FilePermissionTests(unittest.TestCase):
# permissions of .pyc should match those of .py, regardless of mask
mode = 0o600
with temp_umask(0o022), _ready_to_import() as (name, path):
- cached_path = imp.cache_from_source(path)
+ cached_path = importlib.util.cache_from_source(path)
os.chmod(path, mode)
__import__(name)
if not os.path.exists(cached_path):
@@ -365,7 +368,7 @@ class FilePermissionTests(unittest.TestCase):
def test_cached_readonly(self):
mode = 0o400
with temp_umask(0o022), _ready_to_import() as (name, path):
- cached_path = imp.cache_from_source(path)
+ cached_path = importlib.util.cache_from_source(path)
os.chmod(path, mode)
__import__(name)
if not os.path.exists(cached_path):
@@ -405,7 +408,7 @@ class FilePermissionTests(unittest.TestCase):
bytecode_only = path + "c"
else:
bytecode_only = path + "o"
- os.rename(imp.cache_from_source(path), bytecode_only)
+ os.rename(importlib.util.cache_from_source(path), bytecode_only)
m = __import__(name)
self.assertEqual(m.x, 'rewritten')
@@ -427,7 +430,7 @@ func_filename = func.__code__.co_filename
"""
dir_name = os.path.abspath(TESTFN)
file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
- compiled_name = imp.cache_from_source(file_name)
+ compiled_name = importlib.util.cache_from_source(file_name)
def setUp(self):
self.sys_path = sys.path[:]
@@ -488,7 +491,7 @@ func_filename = func.__code__.co_filename
header = f.read(12)
code = marshal.load(f)
constants = list(code.co_consts)
- foreign_code = test_main.__code__
+ foreign_code = importlib.import_module.__code__
pos = constants.index(1)
constants[pos] = foreign_code
code = type(code)(code.co_argcount, code.co_kwonlyargcount,
@@ -630,7 +633,7 @@ class OverridingImportBuiltinTests(unittest.TestCase):
class PycacheTests(unittest.TestCase):
# Test the various PEP 3147 related behaviors.
- tag = imp.get_tag()
+ tag = sys.implementation.cache_tag
def _clean(self):
forget(TESTFN)
@@ -678,10 +681,11 @@ class PycacheTests(unittest.TestCase):
# With PEP 3147 cache layout, removing the source but leaving the pyc
# file does not satisfy the import.
__import__(TESTFN)
- pyc_file = imp.cache_from_source(self.source)
+ pyc_file = importlib.util.cache_from_source(self.source)
self.assertTrue(os.path.exists(pyc_file))
os.remove(self.source)
forget(TESTFN)
+ importlib.invalidate_caches()
self.assertRaises(ImportError, __import__, TESTFN)
@skip_if_dont_write_bytecode
@@ -703,7 +707,7 @@ class PycacheTests(unittest.TestCase):
def test___cached__(self):
# Modules now also have an __cached__ that points to the pyc file.
m = __import__(TESTFN)
- pyc_file = imp.cache_from_source(TESTFN + '.py')
+ pyc_file = importlib.util.cache_from_source(TESTFN + '.py')
self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file))
@skip_if_dont_write_bytecode
@@ -738,10 +742,10 @@ class PycacheTests(unittest.TestCase):
pass
importlib.invalidate_caches()
m = __import__('pep3147.foo')
- init_pyc = imp.cache_from_source(
+ init_pyc = importlib.util.cache_from_source(
os.path.join('pep3147', '__init__.py'))
self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
- foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+ foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py'))
self.assertEqual(sys.modules['pep3147.foo'].__cached__,
os.path.join(os.curdir, foo_pyc))
@@ -765,10 +769,10 @@ class PycacheTests(unittest.TestCase):
unload('pep3147')
importlib.invalidate_caches()
m = __import__('pep3147.foo')
- init_pyc = imp.cache_from_source(
+ init_pyc = importlib.util.cache_from_source(
os.path.join('pep3147', '__init__.py'))
self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
- foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+ foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py'))
self.assertEqual(sys.modules['pep3147.foo'].__cached__,
os.path.join(os.curdir, foo_pyc))
@@ -852,7 +856,6 @@ class ImportlibBootstrapTests(unittest.TestCase):
from importlib import machinery
mod = sys.modules['_frozen_importlib']
self.assertIs(machinery.FileFinder, mod.FileFinder)
- self.assertIs(imp.new_module, mod.new_module)
@cpython_only
@@ -1033,11 +1036,14 @@ class ImportTracebackTests(unittest.TestCase):
# away from the traceback.
self.create_module("foo", "")
importlib = sys.modules['_frozen_importlib']
- old_load_module = importlib.SourceLoader.load_module
+ if 'load_module' in vars(importlib.SourceLoader):
+ old_exec_module = importlib.SourceLoader.exec_module
+ else:
+ old_exec_module = None
try:
- def load_module(*args):
+ def exec_module(*args):
1/0
- importlib.SourceLoader.load_module = load_module
+ importlib.SourceLoader.exec_module = exec_module
try:
import foo
except ZeroDivisionError as e:
@@ -1046,19 +1052,21 @@ class ImportTracebackTests(unittest.TestCase):
self.fail("ZeroDivisionError should have been raised")
self.assert_traceback(tb, [__file__, '<frozen importlib', __file__])
finally:
- importlib.SourceLoader.load_module = old_load_module
-
+ if old_exec_module is None:
+ del importlib.SourceLoader.exec_module
+ else:
+ importlib.SourceLoader.exec_module = old_exec_module
-def test_main(verbose=None):
- run_unittest(ImportTests, PycacheTests, FilePermissionTests,
- PycRewritingTests, PathsTests, RelativeImportTests,
- OverridingImportBuiltinTests,
- ImportlibBootstrapTests, GetSourcefileTests,
- TestSymbolicallyLinkedPackage,
- ImportTracebackTests)
+ @unittest.skipUnless(TESTFN_UNENCODABLE, 'need TESTFN_UNENCODABLE')
+ def test_unencodable_filename(self):
+ # Issue #11619: The Python parser and the import machinery must not
+ # encode filenames, especially on Windows
+ pyname = script_helper.make_script('', TESTFN_UNENCODABLE, 'pass')
+ name = pyname[:-3]
+ script_helper.assert_python_ok("-c", "mod = __import__(%a)" % name,
+ __isolated=False)
if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports.
- from test.test_import import test_main
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py
deleted file mode 100644
index 2a22d1a..0000000
--- a/Lib/test/test_importhooks.py
+++ /dev/null
@@ -1,250 +0,0 @@
-import sys
-import imp
-import os
-import unittest
-from test import support
-
-
-test_src = """\
-def get_name():
- return __name__
-def get_file():
- return __file__
-"""
-
-absimp = "import sub\n"
-relimp = "from . import sub\n"
-deeprelimp = "from .... import sub\n"
-futimp = "from __future__ import absolute_import\n"
-
-reload_src = test_src+"""\
-reloaded = True
-"""
-
-test_co = compile(test_src, "<???>", "exec")
-reload_co = compile(reload_src, "<???>", "exec")
-
-test2_oldabs_co = compile(absimp + test_src, "<???>", "exec")
-test2_newabs_co = compile(futimp + absimp + test_src, "<???>", "exec")
-test2_newrel_co = compile(relimp + test_src, "<???>", "exec")
-test2_deeprel_co = compile(deeprelimp + test_src, "<???>", "exec")
-test2_futrel_co = compile(futimp + relimp + test_src, "<???>", "exec")
-
-test_path = "!!!_test_!!!"
-
-
-class TestImporter:
-
- modules = {
- "hooktestmodule": (False, test_co),
- "hooktestpackage": (True, test_co),
- "hooktestpackage.sub": (True, test_co),
- "hooktestpackage.sub.subber": (True, test_co),
- "hooktestpackage.oldabs": (False, test2_oldabs_co),
- "hooktestpackage.newabs": (False, test2_newabs_co),
- "hooktestpackage.newrel": (False, test2_newrel_co),
- "hooktestpackage.sub.subber.subest": (True, test2_deeprel_co),
- "hooktestpackage.futrel": (False, test2_futrel_co),
- "sub": (False, test_co),
- "reloadmodule": (False, test_co),
- }
-
- def __init__(self, path=test_path):
- if path != test_path:
- # if our class is on sys.path_hooks, we must raise
- # ImportError for any path item that we can't handle.
- raise ImportError
- self.path = path
-
- def _get__path__(self):
- raise NotImplementedError
-
- def find_module(self, fullname, path=None):
- if fullname in self.modules:
- return self
- else:
- return None
-
- def load_module(self, fullname):
- ispkg, code = self.modules[fullname]
- mod = sys.modules.setdefault(fullname,imp.new_module(fullname))
- mod.__file__ = "<%s>" % self.__class__.__name__
- mod.__loader__ = self
- if ispkg:
- mod.__path__ = self._get__path__()
- exec(code, mod.__dict__)
- return mod
-
-
-class MetaImporter(TestImporter):
- def _get__path__(self):
- return []
-
-class PathImporter(TestImporter):
- def _get__path__(self):
- return [self.path]
-
-
-class ImportBlocker:
- """Place an ImportBlocker instance on sys.meta_path and you
- can be sure the modules you specified can't be imported, even
- if it's a builtin."""
- def __init__(self, *namestoblock):
- self.namestoblock = dict.fromkeys(namestoblock)
- def find_module(self, fullname, path=None):
- if fullname in self.namestoblock:
- return self
- return None
- def load_module(self, fullname):
- raise ImportError("I dare you")
-
-
-class ImpWrapper:
-
- def __init__(self, path=None):
- if path is not None and not os.path.isdir(path):
- raise ImportError
- self.path = path
-
- def find_module(self, fullname, path=None):
- subname = fullname.split(".")[-1]
- if subname != fullname and self.path is None:
- return None
- if self.path is None:
- path = None
- else:
- path = [self.path]
- try:
- file, filename, stuff = imp.find_module(subname, path)
- except ImportError:
- return None
- return ImpLoader(file, filename, stuff)
-
-
-class ImpLoader:
-
- def __init__(self, file, filename, stuff):
- self.file = file
- self.filename = filename
- self.stuff = stuff
-
- def load_module(self, fullname):
- mod = imp.load_module(fullname, self.file, self.filename, self.stuff)
- if self.file:
- self.file.close()
- mod.__loader__ = self # for introspection
- return mod
-
-
-class ImportHooksBaseTestCase(unittest.TestCase):
-
- def setUp(self):
- self.path = sys.path[:]
- self.meta_path = sys.meta_path[:]
- self.path_hooks = sys.path_hooks[:]
- sys.path_importer_cache.clear()
- self.modules_before = support.modules_setup()
-
- def tearDown(self):
- sys.path[:] = self.path
- sys.meta_path[:] = self.meta_path
- sys.path_hooks[:] = self.path_hooks
- sys.path_importer_cache.clear()
- support.modules_cleanup(*self.modules_before)
-
-
-class ImportHooksTestCase(ImportHooksBaseTestCase):
-
- def doTestImports(self, importer=None):
- import hooktestmodule
- import hooktestpackage
- import hooktestpackage.sub
- import hooktestpackage.sub.subber
- self.assertEqual(hooktestmodule.get_name(),
- "hooktestmodule")
- self.assertEqual(hooktestpackage.get_name(),
- "hooktestpackage")
- self.assertEqual(hooktestpackage.sub.get_name(),
- "hooktestpackage.sub")
- self.assertEqual(hooktestpackage.sub.subber.get_name(),
- "hooktestpackage.sub.subber")
- if importer:
- self.assertEqual(hooktestmodule.__loader__, importer)
- self.assertEqual(hooktestpackage.__loader__, importer)
- self.assertEqual(hooktestpackage.sub.__loader__, importer)
- self.assertEqual(hooktestpackage.sub.subber.__loader__, importer)
-
- TestImporter.modules['reloadmodule'] = (False, test_co)
- import reloadmodule
- self.assertFalse(hasattr(reloadmodule,'reloaded'))
-
- import hooktestpackage.newrel
- self.assertEqual(hooktestpackage.newrel.get_name(),
- "hooktestpackage.newrel")
- self.assertEqual(hooktestpackage.newrel.sub,
- hooktestpackage.sub)
-
- import hooktestpackage.sub.subber.subest as subest
- self.assertEqual(subest.get_name(),
- "hooktestpackage.sub.subber.subest")
- self.assertEqual(subest.sub,
- hooktestpackage.sub)
-
- import hooktestpackage.futrel
- self.assertEqual(hooktestpackage.futrel.get_name(),
- "hooktestpackage.futrel")
- self.assertEqual(hooktestpackage.futrel.sub,
- hooktestpackage.sub)
-
- import sub
- self.assertEqual(sub.get_name(), "sub")
-
- import hooktestpackage.oldabs
- self.assertEqual(hooktestpackage.oldabs.get_name(),
- "hooktestpackage.oldabs")
- self.assertEqual(hooktestpackage.oldabs.sub, sub)
-
- import hooktestpackage.newabs
- self.assertEqual(hooktestpackage.newabs.get_name(),
- "hooktestpackage.newabs")
- self.assertEqual(hooktestpackage.newabs.sub, sub)
-
- def testMetaPath(self):
- i = MetaImporter()
- sys.meta_path.append(i)
- self.doTestImports(i)
-
- def testPathHook(self):
- sys.path_hooks.insert(0, PathImporter)
- sys.path.append(test_path)
- self.doTestImports()
-
- def testBlocker(self):
- mname = "exceptions" # an arbitrary harmless builtin module
- support.unload(mname)
- sys.meta_path.append(ImportBlocker(mname))
- self.assertRaises(ImportError, __import__, mname)
-
- def testImpWrapper(self):
- i = ImpWrapper()
- sys.meta_path.append(i)
- sys.path_hooks.insert(0, ImpWrapper)
- mnames = (
- "colorsys", "urllib.parse", "distutils.core", "sys",
- )
- for mname in mnames:
- parent = mname.split(".")[0]
- for n in list(sys.modules):
- if n.startswith(parent):
- del sys.modules[n]
- for mname in mnames:
- m = __import__(mname, globals(), locals(), ["__dummy__"])
- # to make sure we actually handled the import
- self.assertTrue(hasattr(m, "__loader__"))
-
-
-def test_main():
- support.run_unittest(ImportHooksTestCase)
-
-if __name__ == "__main__":
- test_main()
diff --git a/Lib/test/test_importlib/__main__.py b/Lib/test/test_importlib/__main__.py
index c397128..14bd5bc 100644
--- a/Lib/test/test_importlib/__main__.py
+++ b/Lib/test/test_importlib/__main__.py
@@ -4,17 +4,6 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with
builtins.__import__ instead of importlib.__import__.
"""
-from . import test_main
-
-
if __name__ == '__main__':
- import argparse
-
- parser = argparse.ArgumentParser(description='Execute the importlib test '
- 'suite')
- parser.add_argument('-b', '--builtin', action='store_true', default=False,
- help='use builtins.__import__() instead of importlib')
- args = parser.parse_args()
- if args.builtin:
- util.using___import__ = True
+ from . import test_main
test_main()
diff --git a/Lib/test/test_importlib/abc.py b/Lib/test/test_importlib/abc.py
index 2c17ac3..2070dad 100644
--- a/Lib/test/test_importlib/abc.py
+++ b/Lib/test/test_importlib/abc.py
@@ -2,7 +2,7 @@ import abc
import unittest
-class FinderTests(unittest.TestCase, metaclass=abc.ABCMeta):
+class FinderTests(metaclass=abc.ABCMeta):
"""Basic tests for a finder to pass."""
@@ -39,7 +39,7 @@ class FinderTests(unittest.TestCase, metaclass=abc.ABCMeta):
pass
-class LoaderTests(unittest.TestCase, metaclass=abc.ABCMeta):
+class LoaderTests(metaclass=abc.ABCMeta):
@abc.abstractmethod
def test_module(self):
@@ -81,11 +81,6 @@ class LoaderTests(unittest.TestCase, metaclass=abc.ABCMeta):
pass
@abc.abstractmethod
- def test_module_reuse(self):
- """If a module is already in sys.modules, it should be reused."""
- pass
-
- @abc.abstractmethod
def test_state_after_failure(self):
"""If a module is already in sys.modules and a reload fails
(e.g. a SyntaxError), the module should be in the state it was before
diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py
index da0ef28..934562f 100644
--- a/Lib/test/test_importlib/builtin/test_finder.py
+++ b/Lib/test/test_importlib/builtin/test_finder.py
@@ -1,11 +1,53 @@
-from importlib import machinery
from .. import abc
from .. import util
from . import util as builtin_util
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+
import sys
import unittest
+
+class FindSpecTests(abc.FinderTests):
+
+ """Test find_spec() for built-in modules."""
+
+ def test_module(self):
+ # Common case.
+ with util.uncache(builtin_util.NAME):
+ found = self.machinery.BuiltinImporter.find_spec(builtin_util.NAME)
+ self.assertTrue(found)
+ self.assertEqual(found.origin, 'built-in')
+
+ # Built-in modules cannot be a package.
+ test_package = None
+
+ # Built-in modules cannobt be in a package.
+ test_module_in_package = None
+
+ # Built-in modules cannot be a package.
+ test_package_in_package = None
+
+ # Built-in modules cannot be a package.
+ test_package_over_module = None
+
+ def test_failure(self):
+ name = 'importlib'
+ assert name not in sys.builtin_module_names
+ spec = self.machinery.BuiltinImporter.find_spec(name)
+ self.assertIsNone(spec)
+
+ def test_ignore_path(self):
+ # The value for 'path' should always trigger a failed import.
+ with util.uncache(builtin_util.NAME):
+ spec = self.machinery.BuiltinImporter.find_spec(builtin_util.NAME,
+ ['pkg'])
+ self.assertIsNone(spec)
+
+Frozen_FindSpecTests, Source_FindSpecTests = util.test_both(FindSpecTests,
+ machinery=[frozen_machinery, source_machinery])
+
+
class FinderTests(abc.FinderTests):
"""Test find_module() for built-in modules."""
@@ -13,8 +55,9 @@ class FinderTests(abc.FinderTests):
def test_module(self):
# Common case.
with util.uncache(builtin_util.NAME):
- found = machinery.BuiltinImporter.find_module(builtin_util.NAME)
+ found = self.machinery.BuiltinImporter.find_module(builtin_util.NAME)
self.assertTrue(found)
+ self.assertTrue(hasattr(found, 'load_module'))
# Built-in modules cannot be a package.
test_package = test_package_in_package = test_package_over_module = None
@@ -24,22 +67,19 @@ class FinderTests(abc.FinderTests):
def test_failure(self):
assert 'importlib' not in sys.builtin_module_names
- loader = machinery.BuiltinImporter.find_module('importlib')
+ loader = self.machinery.BuiltinImporter.find_module('importlib')
self.assertIsNone(loader)
def test_ignore_path(self):
# The value for 'path' should always trigger a failed import.
with util.uncache(builtin_util.NAME):
- loader = machinery.BuiltinImporter.find_module(builtin_util.NAME,
+ loader = self.machinery.BuiltinImporter.find_module(builtin_util.NAME,
['pkg'])
self.assertIsNone(loader)
-
-
-def test_main():
- from test.support import run_unittest
- run_unittest(FinderTests)
+Frozen_FinderTests, Source_FinderTests = util.test_both(FinderTests,
+ machinery=[frozen_machinery, source_machinery])
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/builtin/test_loader.py b/Lib/test/test_importlib/builtin/test_loader.py
index 12a79c6..a636f77 100644
--- a/Lib/test/test_importlib/builtin/test_loader.py
+++ b/Lib/test/test_importlib/builtin/test_loader.py
@@ -1,9 +1,9 @@
-import importlib
-from importlib import machinery
from .. import abc
from .. import util
from . import util as builtin_util
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+
import sys
import types
import unittest
@@ -13,8 +13,9 @@ class LoaderTests(abc.LoaderTests):
"""Test load_module() for built-in modules."""
- verification = {'__name__': 'errno', '__package__': '',
- '__loader__': machinery.BuiltinImporter}
+ def setUp(self):
+ self.verification = {'__name__': 'errno', '__package__': '',
+ '__loader__': self.machinery.BuiltinImporter}
def verify(self, module):
"""Verify that the module matches against what it should have."""
@@ -23,8 +24,8 @@ class LoaderTests(abc.LoaderTests):
self.assertEqual(getattr(module, attr), value)
self.assertIn(module.__name__, sys.modules)
- load_module = staticmethod(lambda name:
- machinery.BuiltinImporter.load_module(name))
+ def load_module(self, name):
+ return self.machinery.BuiltinImporter.load_module(name)
def test_module(self):
# Common case.
@@ -55,45 +56,51 @@ class LoaderTests(abc.LoaderTests):
def test_already_imported(self):
# Using the name of a module already imported but not a built-in should
# still fail.
- assert hasattr(importlib, '__file__') # Not a built-in.
+ module_name = 'builtin_reload_test'
+ assert module_name not in sys.builtin_module_names
+ with util.uncache(module_name):
+ module = types.ModuleType(module_name)
+ sys.modules[module_name] = module
with self.assertRaises(ImportError) as cm:
- self.load_module('importlib')
- self.assertEqual(cm.exception.name, 'importlib')
+ self.load_module(module_name)
+ self.assertEqual(cm.exception.name, module_name)
+
+
+Frozen_LoaderTests, Source_LoaderTests = util.test_both(LoaderTests,
+ machinery=[frozen_machinery, source_machinery])
-class InspectLoaderTests(unittest.TestCase):
+class InspectLoaderTests:
"""Tests for InspectLoader methods for BuiltinImporter."""
def test_get_code(self):
# There is no code object.
- result = machinery.BuiltinImporter.get_code(builtin_util.NAME)
+ result = self.machinery.BuiltinImporter.get_code(builtin_util.NAME)
self.assertIsNone(result)
def test_get_source(self):
# There is no source.
- result = machinery.BuiltinImporter.get_source(builtin_util.NAME)
+ result = self.machinery.BuiltinImporter.get_source(builtin_util.NAME)
self.assertIsNone(result)
def test_is_package(self):
# Cannot be a package.
- result = machinery.BuiltinImporter.is_package(builtin_util.NAME)
+ result = self.machinery.BuiltinImporter.is_package(builtin_util.NAME)
self.assertTrue(not result)
def test_not_builtin(self):
# Modules not built-in should raise ImportError.
for meth_name in ('get_code', 'get_source', 'is_package'):
- method = getattr(machinery.BuiltinImporter, meth_name)
+ method = getattr(self.machinery.BuiltinImporter, meth_name)
with self.assertRaises(ImportError) as cm:
method(builtin_util.BAD_NAME)
self.assertRaises(builtin_util.BAD_NAME)
-
-
-def test_main():
- from test.support import run_unittest
- run_unittest(LoaderTests, InspectLoaderTests)
+Frozen_InspectLoaderTests, Source_InspectLoaderTests = util.test_both(
+ InspectLoaderTests,
+ machinery=[frozen_machinery, source_machinery])
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py
index 76c53e4..bb2528e 100644
--- a/Lib/test/test_importlib/extension/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py
@@ -1,22 +1,27 @@
-import imp
+from importlib import _bootstrap
import sys
from test import support
import unittest
-from importlib import _bootstrap
+
from .. import util
from . import util as ext_util
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+
+# XXX find_spec tests
+
+@unittest.skipIf(ext_util.FILENAME is None, '_testcapi not available')
@util.case_insensitive_tests
-class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
+class ExtensionModuleCaseSensitivityTest:
def find_module(self):
good_name = ext_util.NAME
bad_name = good_name.upper()
assert good_name != bad_name
- finder = _bootstrap.FileFinder(ext_util.PATH,
- (_bootstrap.ExtensionFileLoader,
- _bootstrap.EXTENSION_SUFFIXES))
+ finder = self.machinery.FileFinder(ext_util.PATH,
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))
return finder.find_module(bad_name)
def test_case_sensitive(self):
@@ -37,14 +42,10 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
loader = self.find_module()
self.assertTrue(hasattr(loader, 'load_module'))
-
-
-
-def test_main():
- if ext_util.FILENAME is None:
- return
- support.run_unittest(ExtensionModuleCaseSensitivityTest)
+Frozen_ExtensionCaseSensitivity, Source_ExtensionCaseSensitivity = util.test_both(
+ ExtensionModuleCaseSensitivityTest,
+ machinery=[frozen_machinery, source_machinery])
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py
index 37f6772..990f29c 100644
--- a/Lib/test/test_importlib/extension/test_finder.py
+++ b/Lib/test/test_importlib/extension/test_finder.py
@@ -1,18 +1,25 @@
-from importlib import machinery
from .. import abc
+from .. import util as test_util
from . import util
+machinery = test_util.import_importlib('importlib.machinery')
+
import unittest
+import warnings
+
+# XXX find_spec tests
class FinderTests(abc.FinderTests):
"""Test the finder for extension modules."""
def find_module(self, fullname):
- importer = machinery.FileFinder(util.PATH,
- (machinery.ExtensionFileLoader,
- machinery.EXTENSION_SUFFIXES))
- return importer.find_module(fullname)
+ importer = self.machinery.FileFinder(util.PATH,
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return importer.find_module(fullname)
def test_module(self):
self.assertTrue(self.find_module(util.NAME))
@@ -29,11 +36,9 @@ class FinderTests(abc.FinderTests):
def test_failure(self):
self.assertIsNone(self.find_module('asdfjkl;'))
-
-def test_main():
- from test.support import run_unittest
- run_unittest(FinderTests)
+Frozen_FinderTests, Source_FinderTests = test_util.test_both(
+ FinderTests, machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py
index e790675..fd9abf2 100644
--- a/Lib/test/test_importlib/extension/test_loader.py
+++ b/Lib/test/test_importlib/extension/test_loader.py
@@ -1,10 +1,12 @@
-from importlib import machinery
from . import util as ext_util
from .. import abc
from .. import util
+machinery = util.import_importlib('importlib.machinery')
+
import os.path
import sys
+import types
import unittest
@@ -13,8 +15,8 @@ class LoaderTests(abc.LoaderTests):
"""Test load_module() for extension modules."""
def setUp(self):
- self.loader = machinery.ExtensionFileLoader(ext_util.NAME,
- ext_util.FILEPATH)
+ self.loader = self.machinery.ExtensionFileLoader(ext_util.NAME,
+ ext_util.FILEPATH)
def load_module(self, fullname):
return self.loader.load_module(fullname)
@@ -26,6 +28,15 @@ class LoaderTests(abc.LoaderTests):
with self.assertRaises(ImportError):
self.load_module('XXX')
+ def test_equality(self):
+ other = self.machinery.ExtensionFileLoader(ext_util.NAME,
+ ext_util.FILEPATH)
+ self.assertEqual(self.loader, other)
+
+ def test_inequality(self):
+ other = self.machinery.ExtensionFileLoader('_' + ext_util.NAME,
+ ext_util.FILEPATH)
+ self.assertNotEqual(self.loader, other)
def test_module(self):
with util.uncache(ext_util.NAME):
@@ -36,7 +47,7 @@ class LoaderTests(abc.LoaderTests):
self.assertEqual(getattr(module, attr), value)
self.assertIn(ext_util.NAME, sys.modules)
self.assertIsInstance(module.__loader__,
- machinery.ExtensionFileLoader)
+ self.machinery.ExtensionFileLoader)
# No extension module as __init__ available for testing.
test_package = None
@@ -61,16 +72,15 @@ class LoaderTests(abc.LoaderTests):
def test_is_package(self):
self.assertFalse(self.loader.is_package(ext_util.NAME))
- for suffix in machinery.EXTENSION_SUFFIXES:
+ for suffix in self.machinery.EXTENSION_SUFFIXES:
path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
- loader = machinery.ExtensionFileLoader('pkg', path)
+ loader = self.machinery.ExtensionFileLoader('pkg', path)
self.assertTrue(loader.is_package('pkg'))
+Frozen_LoaderTests, Source_LoaderTests = util.test_both(
+ LoaderTests, machinery=machinery)
-def test_main():
- from test.support import run_unittest
- run_unittest(LoaderTests)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py
index 1d969a1..49d6734 100644
--- a/Lib/test/test_importlib/extension/test_path_hook.py
+++ b/Lib/test/test_importlib/extension/test_path_hook.py
@@ -1,32 +1,32 @@
-from importlib import machinery
+from .. import util as test_util
from . import util
+machinery = test_util.import_importlib('importlib.machinery')
+
import collections
-import imp
import sys
import unittest
-class PathHookTests(unittest.TestCase):
+class PathHookTests:
"""Test the path hook for extension modules."""
# XXX Should it only succeed for pre-existing directories?
# XXX Should it only work for directories containing an extension module?
def hook(self, entry):
- return machinery.FileFinder.path_hook((machinery.ExtensionFileLoader,
- machinery.EXTENSION_SUFFIXES))(entry)
+ return self.machinery.FileFinder.path_hook(
+ (self.machinery.ExtensionFileLoader,
+ self.machinery.EXTENSION_SUFFIXES))(entry)
def test_success(self):
# Path hook should handle a directory where a known extension module
# exists.
self.assertTrue(hasattr(self.hook(util.PATH), 'find_module'))
-
-def test_main():
- from test.support import run_unittest
- run_unittest(PathHookTests)
+Frozen_PathHooksTests, Source_PathHooksTests = test_util.test_both(
+ PathHookTests, machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/extension/util.py b/Lib/test/test_importlib/extension/util.py
index a266dd9..8d089f0 100644
--- a/Lib/test/test_importlib/extension/util.py
+++ b/Lib/test/test_importlib/extension/util.py
@@ -1,4 +1,3 @@
-import imp
from importlib import machinery
import os
import sys
diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
index f0abe0e..f9f97f3 100644
--- a/Lib/test/test_importlib/frozen/test_finder.py
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -1,15 +1,52 @@
-from importlib import machinery
from .. import abc
+from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
import unittest
+class FindSpecTests(abc.FinderTests):
+
+ """Test finding frozen modules."""
+
+ def find(self, name, path=None):
+ finder = self.machinery.FrozenImporter
+ return finder.find_spec(name, path)
+
+ def test_module(self):
+ name = '__hello__'
+ spec = self.find(name)
+ self.assertEqual(spec.origin, 'frozen')
+
+ def test_package(self):
+ spec = self.find('__phello__')
+ self.assertIsNotNone(spec)
+
+ def test_module_in_package(self):
+ spec = self.find('__phello__.spam', ['__phello__'])
+ self.assertIsNotNone(spec)
+
+ # No frozen package within another package to test with.
+ test_package_in_package = None
+
+ # No easy way to test.
+ test_package_over_module = None
+
+ def test_failure(self):
+ spec = self.find('<not real>')
+ self.assertIsNone(spec)
+
+Frozen_FindSpecTests, Source_FindSpecTests = util.test_both(FindSpecTests,
+ machinery=machinery)
+
+
class FinderTests(abc.FinderTests):
"""Test finding frozen modules."""
def find(self, name, path=None):
- finder = machinery.FrozenImporter
+ finder = self.machinery.FrozenImporter
return finder.find_module(name, path)
def test_module(self):
@@ -35,11 +72,9 @@ class FinderTests(abc.FinderTests):
loader = self.find('<not real>')
self.assertIsNone(loader)
-
-def test_main():
- from test.support import run_unittest
- run_unittest(FinderTests)
+Frozen_FinderTests, Source_FinderTests = util.test_both(FinderTests,
+ machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
index f3a8bf6..7c01464 100644
--- a/Lib/test/test_importlib/frozen/test_loader.py
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -1,18 +1,104 @@
-from importlib import machinery
-import imp
-import unittest
from .. import abc
from .. import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+
+import sys
from test.support import captured_stdout
+import types
+import unittest
+import warnings
+
+
+class ExecModuleTests(abc.LoaderTests):
+
+ def exec_module(self, name):
+ with util.uncache(name), captured_stdout() as stdout:
+ spec = self.machinery.ModuleSpec(
+ name, self.machinery.FrozenImporter, origin='frozen',
+ is_package=self.machinery.FrozenImporter.is_package(name))
+ module = types.ModuleType(name)
+ module.__spec__ = spec
+ assert not hasattr(module, 'initialized')
+ self.machinery.FrozenImporter.exec_module(module)
+ self.assertTrue(module.initialized)
+ self.assertTrue(hasattr(module, '__spec__'))
+ self.assertEqual(module.__spec__.origin, 'frozen')
+ return module, stdout.getvalue()
+
+ def test_module(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ self.assertEqual(getattr(module, attr), value)
+ self.assertEqual(output, 'Hello world!\n')
+ self.assertTrue(hasattr(module, '__spec__'))
+
+ def test_package(self):
+ name = '__phello__'
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ 'for {name}.{attr}, {given!r} != {expected!r}'.format(
+ name=name, attr=attr, given=attr_value,
+ expected=value))
+ self.assertEqual(output, 'Hello world!\n')
+
+ def test_lacking_parent(self):
+ name = '__phello__.spam'
+ with util.uncache('__phello__'):
+ module, output = self.exec_module(name)
+ check = {'__name__': name}
+ for attr, value in check.items():
+ attr_value = getattr(module, attr)
+ self.assertEqual(attr_value, value,
+ 'for {name}.{attr}, {given} != {expected!r}'.format(
+ name=name, attr=attr, given=attr_value,
+ expected=value))
+ self.assertEqual(output, 'Hello world!\n')
+
+ def test_module_repr(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ repr_str = self.machinery.FrozenImporter.module_repr(module)
+ self.assertEqual(repr_str,
+ "<module '__hello__' (frozen)>")
+
+ def test_module_repr_indirect(self):
+ name = '__hello__'
+ module, output = self.exec_module(name)
+ self.assertEqual(repr(module),
+ "<module '__hello__' (frozen)>")
+
+ # No way to trigger an error in a frozen module.
+ test_state_after_failure = None
+
+ def test_unloadable(self):
+ assert self.machinery.FrozenImporter.find_module('_not_real') is None
+ with self.assertRaises(ImportError) as cm:
+ self.exec_module('_not_real')
+ self.assertEqual(cm.exception.name, '_not_real')
+
+Frozen_ExecModuleTests, Source_ExecModuleTests = util.test_both(ExecModuleTests,
+ machinery=machinery)
+
class LoaderTests(abc.LoaderTests):
def test_module(self):
with util.uncache('__hello__'), captured_stdout() as stdout:
- module = machinery.FrozenImporter.load_module('__hello__')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__hello__')
check = {'__name__': '__hello__',
'__package__': '',
- '__loader__': machinery.FrozenImporter,
+ '__loader__': self.machinery.FrozenImporter,
}
for attr, value in check.items():
self.assertEqual(getattr(module, attr), value)
@@ -21,11 +107,13 @@ class LoaderTests(abc.LoaderTests):
def test_package(self):
with util.uncache('__phello__'), captured_stdout() as stdout:
- module = machinery.FrozenImporter.load_module('__phello__')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__phello__')
check = {'__name__': '__phello__',
'__package__': '__phello__',
- '__path__': ['__phello__'],
- '__loader__': machinery.FrozenImporter,
+ '__path__': [],
+ '__loader__': self.machinery.FrozenImporter,
}
for attr, value in check.items():
attr_value = getattr(module, attr)
@@ -38,10 +126,12 @@ class LoaderTests(abc.LoaderTests):
def test_lacking_parent(self):
with util.uncache('__phello__', '__phello__.spam'), \
captured_stdout() as stdout:
- module = machinery.FrozenImporter.load_module('__phello__.spam')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__phello__.spam')
check = {'__name__': '__phello__.spam',
'__package__': '__phello__',
- '__loader__': machinery.FrozenImporter,
+ '__loader__': self.machinery.FrozenImporter,
}
for attr, value in check.items():
attr_value = getattr(module, attr)
@@ -53,29 +143,43 @@ class LoaderTests(abc.LoaderTests):
def test_module_reuse(self):
with util.uncache('__hello__'), captured_stdout() as stdout:
- module1 = machinery.FrozenImporter.load_module('__hello__')
- module2 = machinery.FrozenImporter.load_module('__hello__')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module1 = self.machinery.FrozenImporter.load_module('__hello__')
+ module2 = self.machinery.FrozenImporter.load_module('__hello__')
self.assertIs(module1, module2)
self.assertEqual(stdout.getvalue(),
'Hello world!\nHello world!\n')
def test_module_repr(self):
with util.uncache('__hello__'), captured_stdout():
- module = machinery.FrozenImporter.load_module('__hello__')
- self.assertEqual(repr(module),
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.machinery.FrozenImporter.load_module('__hello__')
+ repr_str = self.machinery.FrozenImporter.module_repr(module)
+ self.assertEqual(repr_str,
"<module '__hello__' (frozen)>")
+ def test_module_repr_indirect(self):
+ with util.uncache('__hello__'), captured_stdout():
+ module = self.machinery.FrozenImporter.load_module('__hello__')
+ self.assertEqual(repr(module),
+ "<module '__hello__' (frozen)>")
+
# No way to trigger an error in a frozen module.
test_state_after_failure = None
def test_unloadable(self):
- assert machinery.FrozenImporter.find_module('_not_real') is None
+ assert self.machinery.FrozenImporter.find_module('_not_real') is None
with self.assertRaises(ImportError) as cm:
- machinery.FrozenImporter.load_module('_not_real')
+ self.machinery.FrozenImporter.load_module('_not_real')
self.assertEqual(cm.exception.name, '_not_real')
+Frozen_LoaderTests, Source_LoaderTests = util.test_both(LoaderTests,
+ machinery=machinery)
-class InspectLoaderTests(unittest.TestCase):
+
+class InspectLoaderTests:
"""Tests for the InspectLoader methods for FrozenImporter."""
@@ -83,15 +187,15 @@ class InspectLoaderTests(unittest.TestCase):
# Make sure that the code object is good.
name = '__hello__'
with captured_stdout() as stdout:
- code = machinery.FrozenImporter.get_code(name)
- mod = imp.new_module(name)
+ code = self.machinery.FrozenImporter.get_code(name)
+ mod = types.ModuleType(name)
exec(code, mod.__dict__)
self.assertTrue(hasattr(mod, 'initialized'))
self.assertEqual(stdout.getvalue(), 'Hello world!\n')
def test_get_source(self):
# Should always return None.
- result = machinery.FrozenImporter.get_source('__hello__')
+ result = self.machinery.FrozenImporter.get_source('__hello__')
self.assertIsNone(result)
def test_is_package(self):
@@ -99,22 +203,20 @@ class InspectLoaderTests(unittest.TestCase):
test_for = (('__hello__', False), ('__phello__', True),
('__phello__.spam', False))
for name, is_package in test_for:
- result = machinery.FrozenImporter.is_package(name)
+ result = self.machinery.FrozenImporter.is_package(name)
self.assertEqual(bool(result), is_package)
def test_failure(self):
# Raise ImportError for modules that are not frozen.
for meth_name in ('get_code', 'get_source', 'is_package'):
- method = getattr(machinery.FrozenImporter, meth_name)
+ method = getattr(self.machinery.FrozenImporter, meth_name)
with self.assertRaises(ImportError) as cm:
method('importlib')
self.assertEqual(cm.exception.name, 'importlib')
-
-def test_main():
- from test.support import run_unittest
- run_unittest(LoaderTests, InspectLoaderTests)
+Frozen_ILTests, Source_ILTests = util.test_both(InspectLoaderTests,
+ machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py
new file mode 100644
index 0000000..6df8010
--- /dev/null
+++ b/Lib/test/test_importlib/import_/test___loader__.py
@@ -0,0 +1,70 @@
+from importlib import machinery
+import sys
+import types
+import unittest
+
+from .. import util
+from . import util as import_util
+
+
+class SpecLoaderMock:
+
+ def find_spec(self, fullname, path=None, target=None):
+ return machinery.ModuleSpec(fullname, self)
+
+ def exec_module(self, module):
+ pass
+
+
+class SpecLoaderAttributeTests:
+
+ def test___loader__(self):
+ loader = SpecLoaderMock()
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+Frozen_SpecTests, Source_SpecTests = util.test_both(
+ SpecLoaderAttributeTests, __import__=import_util.__import__)
+
+
+class LoaderMock:
+
+ def find_module(self, fullname, path=None):
+ return self
+
+ def load_module(self, fullname):
+ sys.modules[fullname] = self.module
+ return self.module
+
+
+class LoaderAttributeTests:
+
+ def test___loader___missing(self):
+ module = types.ModuleType('blah')
+ try:
+ del module.__loader__
+ except AttributeError:
+ pass
+ loader = LoaderMock()
+ loader.module = module
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+ def test___loader___is_None(self):
+ module = types.ModuleType('blah')
+ module.__loader__ = None
+ loader = LoaderMock()
+ loader.module = module
+ with util.uncache('blah'), util.import_state(meta_path=[loader]):
+ returned_module = self.__import__('blah')
+ self.assertEqual(loader, module.__loader__)
+
+
+Frozen_Tests, Source_Tests = util.test_both(LoaderAttributeTests,
+ __import__=import_util.__import__)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py
index 783cde1..2e19725 100644
--- a/Lib/test/test_importlib/import_/test___package__.py
+++ b/Lib/test/test_importlib/import_/test___package__.py
@@ -9,7 +9,7 @@ from .. import util
from . import util as import_util
-class Using__package__(unittest.TestCase):
+class Using__package__:
"""Use of __package__ supercedes the use of __name__/__path__ to calculate
what package a module belongs to. The basic algorithm is [__package__]::
@@ -36,10 +36,10 @@ class Using__package__(unittest.TestCase):
def test_using___package__(self):
# [__package__]
- with util.mock_modules('pkg.__init__', 'pkg.fake') as importer:
+ with self.mock_modules('pkg.__init__', 'pkg.fake') as importer:
with util.import_state(meta_path=[importer]):
- import_util.import_('pkg.fake')
- module = import_util.import_('',
+ self.__import__('pkg.fake')
+ module = self.__import__('',
globals={'__package__': 'pkg.fake'},
fromlist=['attr'], level=2)
self.assertEqual(module.__name__, 'pkg')
@@ -49,10 +49,10 @@ class Using__package__(unittest.TestCase):
globals_ = {'__name__': 'pkg.fake', '__path__': []}
if package_as_None:
globals_['__package__'] = None
- with util.mock_modules('pkg.__init__', 'pkg.fake') as importer:
+ with self.mock_modules('pkg.__init__', 'pkg.fake') as importer:
with util.import_state(meta_path=[importer]):
- import_util.import_('pkg.fake')
- module = import_util.import_('', globals= globals_,
+ self.__import__('pkg.fake')
+ module = self.__import__('', globals= globals_,
fromlist=['attr'], level=2)
self.assertEqual(module.__name__, 'pkg')
@@ -63,16 +63,27 @@ class Using__package__(unittest.TestCase):
def test_bad__package__(self):
globals = {'__package__': '<not real>'}
with self.assertRaises(SystemError):
- import_util.import_('', globals, {}, ['relimport'], 1)
+ self.__import__('', globals, {}, ['relimport'], 1)
def test_bunk__package__(self):
globals = {'__package__': 42}
with self.assertRaises(TypeError):
- import_util.import_('', globals, {}, ['relimport'], 1)
+ self.__import__('', globals, {}, ['relimport'], 1)
+class Using__package__PEP302(Using__package__):
+ mock_modules = util.mock_modules
-@import_util.importlib_only
-class Setting__package__(unittest.TestCase):
+Frozen_UsingPackagePEP302, Source_UsingPackagePEP302 = util.test_both(
+ Using__package__PEP302, __import__=import_util.__import__)
+
+class Using__package__PEP302(Using__package__):
+ mock_modules = util.mock_spec
+
+Frozen_UsingPackagePEP451, Source_UsingPackagePEP451 = util.test_both(
+ Using__package__PEP302, __import__=import_util.__import__)
+
+
+class Setting__package__:
"""Because __package__ is a new feature, it is not always set by a loader.
Import will set it as needed to help with the transition to relying on
@@ -84,36 +95,39 @@ class Setting__package__(unittest.TestCase):
"""
+ __import__ = import_util.__import__[1]
+
# [top-level]
def test_top_level(self):
- with util.mock_modules('top_level') as mock:
+ with self.mock_modules('top_level') as mock:
with util.import_state(meta_path=[mock]):
del mock['top_level'].__package__
- module = import_util.import_('top_level')
+ module = self.__import__('top_level')
self.assertEqual(module.__package__, '')
# [package]
def test_package(self):
- with util.mock_modules('pkg.__init__') as mock:
+ with self.mock_modules('pkg.__init__') as mock:
with util.import_state(meta_path=[mock]):
del mock['pkg'].__package__
- module = import_util.import_('pkg')
+ module = self.__import__('pkg')
self.assertEqual(module.__package__, 'pkg')
# [submodule]
def test_submodule(self):
- with util.mock_modules('pkg.__init__', 'pkg.mod') as mock:
+ with self.mock_modules('pkg.__init__', 'pkg.mod') as mock:
with util.import_state(meta_path=[mock]):
del mock['pkg.mod'].__package__
- pkg = import_util.import_('pkg.mod')
+ pkg = self.__import__('pkg.mod')
module = getattr(pkg, 'mod')
self.assertEqual(module.__package__, 'pkg')
+class Setting__package__PEP302(Setting__package__, unittest.TestCase):
+ mock_modules = util.mock_modules
-def test_main():
- from test.support import run_unittest
- run_unittest(Using__package__, Setting__package__)
+class Setting__package__PEP451(Setting__package__, unittest.TestCase):
+ mock_modules = util.mock_spec
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
index 3d4cd94..439c105 100644
--- a/Lib/test/test_importlib/import_/test_api.py
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -1,23 +1,41 @@
-from .. import util as importlib_test_util
-from . import util
-import imp
+from .. import util
+from . import util as import_util
+
+from importlib import machinery
import sys
+import types
import unittest
+PKG_NAME = 'fine'
+SUBMOD_NAME = 'fine.bogus'
+
+
+class BadSpecFinderLoader:
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ if fullname == SUBMOD_NAME:
+ spec = machinery.ModuleSpec(fullname, cls)
+ return spec
+
+ @staticmethod
+ def exec_module(module):
+ if module.__name__ == SUBMOD_NAME:
+ raise ImportError('I cannot be loaded!')
+
class BadLoaderFinder:
- bad = 'fine.bogus'
@classmethod
def find_module(cls, fullname, path):
- if fullname == cls.bad:
+ if fullname == SUBMOD_NAME:
return cls
+
@classmethod
def load_module(cls, fullname):
- if fullname == cls.bad:
+ if fullname == SUBMOD_NAME:
raise ImportError('I cannot be loaded!')
-class APITest(unittest.TestCase):
+class APITest:
"""Test API-specific details for __import__ (e.g. raising the right
exception when passing in an int for the module name)."""
@@ -25,43 +43,52 @@ class APITest(unittest.TestCase):
def test_name_requires_rparition(self):
# Raise TypeError if a non-string is passed in for the module name.
with self.assertRaises(TypeError):
- util.import_(42)
+ self.__import__(42)
def test_negative_level(self):
# Raise ValueError when a negative level is specified.
# PEP 328 did away with sys.module None entries and the ambiguity of
# absolute/relative imports.
with self.assertRaises(ValueError):
- util.import_('os', globals(), level=-1)
+ self.__import__('os', globals(), level=-1)
def test_nonexistent_fromlist_entry(self):
# If something in fromlist doesn't exist, that's okay.
# issue15715
- mod = imp.new_module('fine')
+ mod = types.ModuleType(PKG_NAME)
mod.__path__ = ['XXX']
- with importlib_test_util.import_state(meta_path=[BadLoaderFinder]):
- with importlib_test_util.uncache('fine'):
- sys.modules['fine'] = mod
- util.import_('fine', fromlist=['not here'])
+ with util.import_state(meta_path=[self.bad_finder_loader]):
+ with util.uncache(PKG_NAME):
+ sys.modules[PKG_NAME] = mod
+ self.__import__(PKG_NAME, fromlist=['not here'])
def test_fromlist_load_error_propagates(self):
# If something in fromlist triggers an exception not related to not
# existing, let that exception propagate.
# issue15316
- mod = imp.new_module('fine')
+ mod = types.ModuleType(PKG_NAME)
mod.__path__ = ['XXX']
- with importlib_test_util.import_state(meta_path=[BadLoaderFinder]):
- with importlib_test_util.uncache('fine'):
- sys.modules['fine'] = mod
+ with util.import_state(meta_path=[self.bad_finder_loader]):
+ with util.uncache(PKG_NAME):
+ sys.modules[PKG_NAME] = mod
with self.assertRaises(ImportError):
- util.import_('fine', fromlist=['bogus'])
+ self.__import__(PKG_NAME,
+ fromlist=[SUBMOD_NAME.rpartition('.')[-1]])
+
+
+class OldAPITests(APITest):
+ bad_finder_loader = BadLoaderFinder
+
+Frozen_OldAPITests, Source_OldAPITests = util.test_both(
+ OldAPITests, __import__=import_util.__import__)
+class SpecAPITests(APITest):
+ bad_finder_loader = BadSpecFinderLoader
-def test_main():
- from test.support import run_unittest
- run_unittest(APITest)
+Frozen_SpecAPITests, Source_SpecAPITests = util.test_both(
+ SpecAPITests, __import__=import_util.__import__)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py
index 207378a..c292ee4 100644
--- a/Lib/test/test_importlib/import_/test_caching.py
+++ b/Lib/test/test_importlib/import_/test_caching.py
@@ -6,7 +6,7 @@ from types import MethodType
import unittest
-class UseCache(unittest.TestCase):
+class UseCache:
"""When it comes to sys.modules, import prefers it over anything else.
@@ -21,12 +21,13 @@ class UseCache(unittest.TestCase):
ImportError is raised [None in cache].
"""
+
def test_using_cache(self):
# [use cache]
module_to_use = "some module found!"
with util.uncache('some_module'):
sys.modules['some_module'] = module_to_use
- module = import_util.import_('some_module')
+ module = self.__import__('some_module')
self.assertEqual(id(module_to_use), id(module))
def test_None_in_cache(self):
@@ -35,9 +36,19 @@ class UseCache(unittest.TestCase):
with util.uncache(name):
sys.modules[name] = None
with self.assertRaises(ImportError) as cm:
- import_util.import_(name)
+ self.__import__(name)
self.assertEqual(cm.exception.name, name)
+Frozen_UseCache, Source_UseCache = util.test_both(
+ UseCache, __import__=import_util.__import__)
+
+
+class ImportlibUseCache(UseCache, unittest.TestCase):
+
+ # Pertinent only to PEP 302; exec_module() doesn't return a module.
+
+ __import__ = import_util.__import__[1]
+
def create_mock(self, *names, return_=None):
mock = util.mock_modules(*names)
original_load = mock.load_module
@@ -49,40 +60,33 @@ class UseCache(unittest.TestCase):
# __import__ inconsistent between loaders and built-in import when it comes
# to when to use the module in sys.modules and when not to.
- @import_util.importlib_only
def test_using_cache_after_loader(self):
# [from cache on return]
with self.create_mock('module') as mock:
with util.import_state(meta_path=[mock]):
- module = import_util.import_('module')
+ module = self.__import__('module')
self.assertEqual(id(module), id(sys.modules['module']))
# See test_using_cache_after_loader() for reasoning.
- @import_util.importlib_only
def test_using_cache_for_assigning_to_attribute(self):
# [from cache to attribute]
with self.create_mock('pkg.__init__', 'pkg.module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg.module')
+ module = self.__import__('pkg.module')
self.assertTrue(hasattr(module, 'module'))
self.assertEqual(id(module.module),
id(sys.modules['pkg.module']))
# See test_using_cache_after_loader() for reasoning.
- @import_util.importlib_only
def test_using_cache_for_fromlist(self):
# [from cache for fromlist]
with self.create_mock('pkg.__init__', 'pkg.module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg', fromlist=['module'])
+ module = self.__import__('pkg', fromlist=['module'])
self.assertTrue(hasattr(module, 'module'))
self.assertEqual(id(module.module),
id(sys.modules['pkg.module']))
-def test_main():
- from test.support import run_unittest
- run_unittest(UseCache)
-
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_fromlist.py b/Lib/test/test_importlib/import_/test_fromlist.py
index c16c337..58f244b 100644
--- a/Lib/test/test_importlib/import_/test_fromlist.py
+++ b/Lib/test/test_importlib/import_/test_fromlist.py
@@ -1,10 +1,10 @@
"""Test that the semantics relating to the 'fromlist' argument are correct."""
from .. import util
from . import util as import_util
-import imp
import unittest
-class ReturnValue(unittest.TestCase):
+
+class ReturnValue:
"""The use of fromlist influences what import returns.
@@ -17,20 +17,23 @@ class ReturnValue(unittest.TestCase):
def test_return_from_import(self):
# [import return]
- with util.mock_modules('pkg.__init__', 'pkg.module') as importer:
+ with util.mock_spec('pkg.__init__', 'pkg.module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg.module')
+ module = self.__import__('pkg.module')
self.assertEqual(module.__name__, 'pkg')
def test_return_from_from_import(self):
# [from return]
with util.mock_modules('pkg.__init__', 'pkg.module')as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg.module', fromlist=['attr'])
+ module = self.__import__('pkg.module', fromlist=['attr'])
self.assertEqual(module.__name__, 'pkg.module')
+Frozen_ReturnValue, Source_ReturnValue = util.test_both(
+ ReturnValue, __import__=import_util.__import__)
+
-class HandlingFromlist(unittest.TestCase):
+class HandlingFromlist:
"""Using fromlist triggers different actions based on what is being asked
of it.
@@ -49,14 +52,14 @@ class HandlingFromlist(unittest.TestCase):
# [object case]
with util.mock_modules('module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('module', fromlist=['attr'])
+ module = self.__import__('module', fromlist=['attr'])
self.assertEqual(module.__name__, 'module')
def test_nonexistent_object(self):
# [bad object]
with util.mock_modules('module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('module', fromlist=['non_existent'])
+ module = self.__import__('module', fromlist=['non_existent'])
self.assertEqual(module.__name__, 'module')
self.assertTrue(not hasattr(module, 'non_existent'))
@@ -64,7 +67,7 @@ class HandlingFromlist(unittest.TestCase):
# [module]
with util.mock_modules('pkg.__init__', 'pkg.module') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg', fromlist=['module'])
+ module = self.__import__('pkg', fromlist=['module'])
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'module'))
self.assertEqual(module.module.__name__, 'pkg.module')
@@ -79,13 +82,13 @@ class HandlingFromlist(unittest.TestCase):
module_code={'pkg.mod': module_code}) as importer:
with util.import_state(meta_path=[importer]):
with self.assertRaises(ImportError) as exc:
- import_util.import_('pkg', fromlist=['mod'])
+ self.__import__('pkg', fromlist=['mod'])
self.assertEqual('i_do_not_exist', exc.exception.name)
def test_empty_string(self):
with util.mock_modules('pkg.__init__', 'pkg.mod') as importer:
with util.import_state(meta_path=[importer]):
- module = import_util.import_('pkg.mod', fromlist=[''])
+ module = self.__import__('pkg.mod', fromlist=[''])
self.assertEqual(module.__name__, 'pkg.mod')
def basic_star_test(self, fromlist=['*']):
@@ -93,7 +96,7 @@ class HandlingFromlist(unittest.TestCase):
with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
with util.import_state(meta_path=[mock]):
mock['pkg'].__all__ = ['module']
- module = import_util.import_('pkg', fromlist=fromlist)
+ module = self.__import__('pkg', fromlist=fromlist)
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'module'))
self.assertEqual(module.module.__name__, 'pkg.module')
@@ -111,17 +114,16 @@ class HandlingFromlist(unittest.TestCase):
with context as mock:
with util.import_state(meta_path=[mock]):
mock['pkg'].__all__ = ['module1']
- module = import_util.import_('pkg', fromlist=['module2', '*'])
+ module = self.__import__('pkg', fromlist=['module2', '*'])
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'module1'))
self.assertTrue(hasattr(module, 'module2'))
self.assertEqual(module.module1.__name__, 'pkg.module1')
self.assertEqual(module.module2.__name__, 'pkg.module2')
+Frozen_FromList, Source_FromList = util.test_both(
+ HandlingFromlist, __import__=import_util.__import__)
-def test_main():
- from test.support import run_unittest
- run_unittest(ReturnValue, HandlingFromlist)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py
index 4d85f80..dc45420 100644
--- a/Lib/test/test_importlib/import_/test_meta_path.py
+++ b/Lib/test/test_importlib/import_/test_meta_path.py
@@ -7,7 +7,7 @@ import unittest
import warnings
-class CallingOrder(unittest.TestCase):
+class CallingOrder:
"""Calls to the importers on sys.meta_path happen in order that they are
specified in the sequence, starting with the first importer
@@ -18,23 +18,18 @@ class CallingOrder(unittest.TestCase):
def test_first_called(self):
# [first called]
mod = 'top_level'
- first = util.mock_modules(mod)
- second = util.mock_modules(mod)
- with util.mock_modules(mod) as first, util.mock_modules(mod) as second:
- first.modules[mod] = 42
- second.modules[mod] = -13
+ with util.mock_spec(mod) as first, util.mock_spec(mod) as second:
with util.import_state(meta_path=[first, second]):
- self.assertEqual(import_util.import_(mod), 42)
+ self.assertIs(self.__import__(mod), first.modules[mod])
def test_continuing(self):
# [continuing]
mod_name = 'for_real'
- with util.mock_modules('nonexistent') as first, \
- util.mock_modules(mod_name) as second:
- first.find_module = lambda self, fullname, path=None: None
- second.modules[mod_name] = 42
+ with util.mock_spec('nonexistent') as first, \
+ util.mock_spec(mod_name) as second:
+ first.find_spec = lambda self, fullname, path=None, parent=None: None
with util.import_state(meta_path=[first, second]):
- self.assertEqual(import_util.import_(mod_name), 42)
+ self.assertIs(self.__import__(mod_name), second.modules[mod_name])
def test_empty(self):
# Raise an ImportWarning if sys.meta_path is empty.
@@ -46,41 +41,42 @@ class CallingOrder(unittest.TestCase):
with util.import_state(meta_path=[]):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
- self.assertIsNone(importlib._bootstrap._find_module('nothing',
- None))
+ self.assertIsNone(importlib._bootstrap._find_spec('nothing',
+ None))
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[-1].category, ImportWarning))
+Frozen_CallingOrder, Source_CallingOrder = util.test_both(
+ CallingOrder, __import__=import_util.__import__)
-class CallSignature(unittest.TestCase):
+
+class CallSignature:
"""If there is no __path__ entry on the parent module, then 'path' is None
[no path]. Otherwise, the value for __path__ is passed in for the 'path'
argument [path set]."""
- def log(self, fxn):
+ def log_finder(self, importer):
+ fxn = getattr(importer, self.finder_name)
log = []
def wrapper(self, *args, **kwargs):
log.append([args, kwargs])
return fxn(*args, **kwargs)
return log, wrapper
-
def test_no_path(self):
# [no path]
mod_name = 'top_level'
assert '.' not in mod_name
- with util.mock_modules(mod_name) as importer:
- log, wrapped_call = self.log(importer.find_module)
- importer.find_module = MethodType(wrapped_call, importer)
+ with self.mock_modules(mod_name) as importer:
+ log, wrapped_call = self.log_finder(importer)
+ setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
with util.import_state(meta_path=[importer]):
- import_util.import_(mod_name)
+ self.__import__(mod_name)
assert len(log) == 1
args = log[0][0]
kwargs = log[0][1]
# Assuming all arguments are positional.
- self.assertEqual(len(args), 2)
- self.assertEqual(len(kwargs), 0)
self.assertEqual(args[0], mod_name)
self.assertIsNone(args[1])
@@ -90,12 +86,12 @@ class CallSignature(unittest.TestCase):
mod_name = pkg_name + '.module'
path = [42]
assert '.' in mod_name
- with util.mock_modules(pkg_name+'.__init__', mod_name) as importer:
+ with self.mock_modules(pkg_name+'.__init__', mod_name) as importer:
importer.modules[pkg_name].__path__ = path
- log, wrapped_call = self.log(importer.find_module)
- importer.find_module = MethodType(wrapped_call, importer)
+ log, wrapped_call = self.log_finder(importer)
+ setattr(importer, self.finder_name, MethodType(wrapped_call, importer))
with util.import_state(meta_path=[importer]):
- import_util.import_(mod_name)
+ self.__import__(mod_name)
assert len(log) == 2
args = log[1][0]
kwargs = log[1][1]
@@ -104,12 +100,20 @@ class CallSignature(unittest.TestCase):
self.assertEqual(args[0], mod_name)
self.assertIs(args[1], path)
+class CallSignaturePEP302(CallSignature):
+ mock_modules = util.mock_modules
+ finder_name = 'find_module'
+
+Frozen_CallSignaturePEP302, Source_CallSignaturePEP302 = util.test_both(
+ CallSignaturePEP302, __import__=import_util.__import__)
+class CallSignaturePEP451(CallSignature):
+ mock_modules = util.mock_spec
+ finder_name = 'find_spec'
-def test_main():
- from test.support import run_unittest
- run_unittest(CallingOrder, CallSignature)
+Frozen_CallSignaturePEP451, Source_CallSignaturePEP451 = util.test_both(
+ CallSignaturePEP451, __import__=import_util.__import__)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_packages.py b/Lib/test/test_importlib/import_/test_packages.py
index bfa18dc..55a5d14 100644
--- a/Lib/test/test_importlib/import_/test_packages.py
+++ b/Lib/test/test_importlib/import_/test_packages.py
@@ -6,37 +6,37 @@ import importlib
from test import support
-class ParentModuleTests(unittest.TestCase):
+class ParentModuleTests:
"""Importing a submodule should import the parent modules."""
def test_import_parent(self):
- with util.mock_modules('pkg.__init__', 'pkg.module') as mock:
+ with util.mock_spec('pkg.__init__', 'pkg.module') as mock:
with util.import_state(meta_path=[mock]):
- module = import_util.import_('pkg.module')
+ module = self.__import__('pkg.module')
self.assertIn('pkg', sys.modules)
def test_bad_parent(self):
- with util.mock_modules('pkg.module') as mock:
+ with util.mock_spec('pkg.module') as mock:
with util.import_state(meta_path=[mock]):
with self.assertRaises(ImportError) as cm:
- import_util.import_('pkg.module')
+ self.__import__('pkg.module')
self.assertEqual(cm.exception.name, 'pkg')
def test_raising_parent_after_importing_child(self):
def __init__():
import pkg.module
1/0
- mock = util.mock_modules('pkg.__init__', 'pkg.module',
+ mock = util.mock_spec('pkg.__init__', 'pkg.module',
module_code={'pkg': __init__})
with mock:
with util.import_state(meta_path=[mock]):
with self.assertRaises(ZeroDivisionError):
- import_util.import_('pkg')
+ self.__import__('pkg')
self.assertNotIn('pkg', sys.modules)
self.assertIn('pkg.module', sys.modules)
with self.assertRaises(ZeroDivisionError):
- import_util.import_('pkg.module')
+ self.__import__('pkg.module')
self.assertNotIn('pkg', sys.modules)
self.assertIn('pkg.module', sys.modules)
@@ -44,17 +44,17 @@ class ParentModuleTests(unittest.TestCase):
def __init__():
from . import module
1/0
- mock = util.mock_modules('pkg.__init__', 'pkg.module',
+ mock = util.mock_spec('pkg.__init__', 'pkg.module',
module_code={'pkg': __init__})
with mock:
with util.import_state(meta_path=[mock]):
with self.assertRaises((ZeroDivisionError, ImportError)):
# This raises ImportError on the "from . import module"
# line, not sure why.
- import_util.import_('pkg')
+ self.__import__('pkg')
self.assertNotIn('pkg', sys.modules)
with self.assertRaises((ZeroDivisionError, ImportError)):
- import_util.import_('pkg.module')
+ self.__import__('pkg.module')
self.assertNotIn('pkg', sys.modules)
# XXX False
#self.assertIn('pkg.module', sys.modules)
@@ -63,7 +63,7 @@ class ParentModuleTests(unittest.TestCase):
def __init__():
from ..subpkg import module
1/0
- mock = util.mock_modules('pkg.__init__', 'pkg.subpkg.__init__',
+ mock = util.mock_spec('pkg.__init__', 'pkg.subpkg.__init__',
'pkg.subpkg.module',
module_code={'pkg.subpkg': __init__})
with mock:
@@ -71,10 +71,10 @@ class ParentModuleTests(unittest.TestCase):
with self.assertRaises((ZeroDivisionError, ImportError)):
# This raises ImportError on the "from ..subpkg import module"
# line, not sure why.
- import_util.import_('pkg.subpkg')
+ self.__import__('pkg.subpkg')
self.assertNotIn('pkg.subpkg', sys.modules)
with self.assertRaises((ZeroDivisionError, ImportError)):
- import_util.import_('pkg.subpkg.module')
+ self.__import__('pkg.subpkg.module')
self.assertNotIn('pkg.subpkg', sys.modules)
# XXX False
#self.assertIn('pkg.subpkg.module', sys.modules)
@@ -83,7 +83,7 @@ class ParentModuleTests(unittest.TestCase):
# Try to import a submodule from a non-package should raise ImportError.
assert not hasattr(sys, '__path__')
with self.assertRaises(ImportError) as cm:
- import_util.import_('sys.no_submodules_here')
+ self.__import__('sys.no_submodules_here')
self.assertEqual(cm.exception.name, 'sys.no_submodules_here')
def test_module_not_package_but_side_effects(self):
@@ -93,20 +93,18 @@ class ParentModuleTests(unittest.TestCase):
subname = name + '.b'
def module_injection():
sys.modules[subname] = 'total bunk'
- mock_modules = util.mock_modules('mod',
+ mock_spec = util.mock_spec('mod',
module_code={'mod': module_injection})
- with mock_modules as mock:
+ with mock_spec as mock:
with util.import_state(meta_path=[mock]):
try:
- submodule = import_util.import_(subname)
+ submodule = self.__import__(subname)
finally:
support.unload(subname)
-
-def test_main():
- from test.support import run_unittest
- run_unittest(ParentModuleTests)
+Frozen_ParentTests, Source_ParentTests = util.test_both(
+ ParentModuleTests, __import__=import_util.__import__)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py
index d82b7f6..1274f8c 100644
--- a/Lib/test/test_importlib/import_/test_path.py
+++ b/Lib/test/test_importlib/import_/test_path.py
@@ -1,8 +1,9 @@
-from importlib import _bootstrap
-from importlib import machinery
-from importlib import import_module
from .. import util
from . import util as import_util
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
import os
import sys
from types import ModuleType
@@ -11,25 +12,25 @@ import warnings
import zipimport
-class FinderTests(unittest.TestCase):
+class FinderTests:
"""Tests for PathFinder."""
def test_failure(self):
- # Test None returned upon not finding a suitable finder.
+ # Test None returned upon not finding a suitable loader.
module = '<test module>'
with util.import_state():
- self.assertIsNone(machinery.PathFinder.find_module(module))
+ self.assertIsNone(self.machinery.PathFinder.find_module(module))
def test_sys_path(self):
# Test that sys.path is used when 'path' is None.
# Implicitly tests that sys.path_importer_cache is used.
module = '<test module>'
path = '<test path>'
- importer = util.mock_modules(module)
+ importer = util.mock_spec(module)
with util.import_state(path_importer_cache={path: importer},
path=[path]):
- loader = machinery.PathFinder.find_module(module)
+ loader = self.machinery.PathFinder.find_module(module)
self.assertIs(loader, importer)
def test_path(self):
@@ -37,29 +38,29 @@ class FinderTests(unittest.TestCase):
# Implicitly tests that sys.path_importer_cache is used.
module = '<test module>'
path = '<test path>'
- importer = util.mock_modules(module)
+ importer = util.mock_spec(module)
with util.import_state(path_importer_cache={path: importer}):
- loader = machinery.PathFinder.find_module(module, [path])
+ loader = self.machinery.PathFinder.find_module(module, [path])
self.assertIs(loader, importer)
def test_empty_list(self):
# An empty list should not count as asking for sys.path.
module = 'module'
path = '<test path>'
- importer = util.mock_modules(module)
+ importer = util.mock_spec(module)
with util.import_state(path_importer_cache={path: importer},
path=[path]):
- self.assertIsNone(machinery.PathFinder.find_module('module', []))
+ self.assertIsNone(self.machinery.PathFinder.find_module('module', []))
def test_path_hooks(self):
# Test that sys.path_hooks is used.
# Test that sys.path_importer_cache is set.
module = '<test module>'
path = '<test path>'
- importer = util.mock_modules(module)
+ importer = util.mock_spec(module)
hook = import_util.mock_path_hook(path, importer=importer)
with util.import_state(path_hooks=[hook]):
- loader = machinery.PathFinder.find_module(module, [path])
+ loader = self.machinery.PathFinder.find_module(module, [path])
self.assertIs(loader, importer)
self.assertIn(path, sys.path_importer_cache)
self.assertIs(sys.path_importer_cache[path], importer)
@@ -72,7 +73,7 @@ class FinderTests(unittest.TestCase):
path=[path_entry]):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
- self.assertIsNone(machinery.PathFinder.find_module('os'))
+ self.assertIsNone(self.machinery.PathFinder.find_module('os'))
self.assertIsNone(sys.path_importer_cache[path_entry])
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[-1].category, ImportWarning))
@@ -81,12 +82,12 @@ class FinderTests(unittest.TestCase):
# The empty string should create a finder using the cwd.
path = ''
module = '<test module>'
- importer = util.mock_modules(module)
- hook = import_util.mock_path_hook(os.curdir, importer=importer)
+ importer = util.mock_spec(module)
+ hook = import_util.mock_path_hook(os.getcwd(), importer=importer)
with util.import_state(path=[path], path_hooks=[hook]):
- loader = machinery.PathFinder.find_module(module)
+ loader = self.machinery.PathFinder.find_module(module)
self.assertIs(loader, importer)
- self.assertIn(os.curdir, sys.path_importer_cache)
+ self.assertIn(os.getcwd(), sys.path_importer_cache)
def test_None_on_sys_path(self):
# Putting None in sys.path[0] caused an import regression from Python
@@ -96,8 +97,8 @@ class FinderTests(unittest.TestCase):
new_path_importer_cache = sys.path_importer_cache.copy()
new_path_importer_cache.pop(None, None)
new_path_hooks = [zipimport.zipimporter,
- _bootstrap.FileFinder.path_hook(
- *_bootstrap._get_supported_file_loaders())]
+ self.machinery.FileFinder.path_hook(
+ *self.importlib._bootstrap._get_supported_file_loaders())]
missing = object()
email = sys.modules.pop('email', missing)
try:
@@ -105,16 +106,39 @@ class FinderTests(unittest.TestCase):
path=new_path,
path_importer_cache=new_path_importer_cache,
path_hooks=new_path_hooks):
- module = import_module('email')
+ module = self.importlib.import_module('email')
self.assertIsInstance(module, ModuleType)
finally:
if email is not missing:
sys.modules['email'] = email
+Frozen_FinderTests, Source_FinderTests = util.test_both(
+ FinderTests, importlib=importlib, machinery=machinery)
+
+
+class PathEntryFinderTests:
+
+ def test_finder_with_failing_find_module(self):
+ # PathEntryFinder with find_module() defined should work.
+ # Issue #20763.
+ class Finder:
+ path_location = 'test_finder_with_find_module'
+ def __init__(self, path):
+ if path != self.path_location:
+ raise ImportError
+
+ @staticmethod
+ def find_module(fullname):
+ return None
+
+
+ with util.import_state(path=[Finder.path_location]+sys.path[:],
+ path_hooks=[Finder]):
+ self.machinery.PathFinder.find_spec('importlib')
+
+Frozen_PEFTests, Source_PEFTests = util.test_both(
+ PathEntryFinderTests, machinery=machinery)
-def test_main():
- from test.support import run_unittest
- run_unittest(FinderTests)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py
index 4569c26..b216e9c 100644
--- a/Lib/test/test_importlib/import_/test_relative_imports.py
+++ b/Lib/test/test_importlib/import_/test_relative_imports.py
@@ -4,7 +4,7 @@ from . import util as import_util
import sys
import unittest
-class RelativeImports(unittest.TestCase):
+class RelativeImports:
"""PEP 328 introduced relative imports. This allows for imports to occur
from within a package without having to specify the actual package name.
@@ -64,7 +64,7 @@ class RelativeImports(unittest.TestCase):
uncache_names.append(name)
else:
uncache_names.append(name[:-len('.__init__')])
- with util.mock_modules(*create) as importer:
+ with util.mock_spec(*create) as importer:
with util.import_state(meta_path=[importer]):
for global_ in globals_:
with util.uncache(*uncache_names):
@@ -76,8 +76,8 @@ class RelativeImports(unittest.TestCase):
create = 'pkg.__init__', 'pkg.mod2'
globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
def callback(global_):
- import_util.import_('pkg') # For __import__().
- module = import_util.import_('', global_, fromlist=['mod2'], level=1)
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['mod2'], level=1)
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'mod2'))
self.assertEqual(module.mod2.attr, 'pkg.mod2')
@@ -88,8 +88,8 @@ class RelativeImports(unittest.TestCase):
create = 'pkg.__init__', 'pkg.mod2'
globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.mod1'}
def callback(global_):
- import_util.import_('pkg') # For __import__().
- module = import_util.import_('mod2', global_, fromlist=['attr'],
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('mod2', global_, fromlist=['attr'],
level=1)
self.assertEqual(module.__name__, 'pkg.mod2')
self.assertEqual(module.attr, 'pkg.mod2')
@@ -101,8 +101,8 @@ class RelativeImports(unittest.TestCase):
globals_ = ({'__package__': 'pkg'},
{'__name__': 'pkg', '__path__': ['blah']})
def callback(global_):
- import_util.import_('pkg') # For __import__().
- module = import_util.import_('', global_, fromlist=['module'],
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['module'],
level=1)
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'module'))
@@ -114,8 +114,8 @@ class RelativeImports(unittest.TestCase):
create = 'pkg.__init__', 'pkg.module'
globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
def callback(global_):
- import_util.import_('pkg') # For __import__().
- module = import_util.import_('', global_, fromlist=['attr'], level=1)
+ self.__import__('pkg') # For __import__().
+ module = self.__import__('', global_, fromlist=['attr'], level=1)
self.assertEqual(module.__name__, 'pkg')
self.relative_import_test(create, globals_, callback)
@@ -126,7 +126,7 @@ class RelativeImports(unittest.TestCase):
globals_ = ({'__package__': 'pkg.subpkg1'},
{'__name__': 'pkg.subpkg1', '__path__': ['blah']})
def callback(global_):
- module = import_util.import_('', global_, fromlist=['subpkg2'],
+ module = self.__import__('', global_, fromlist=['subpkg2'],
level=2)
self.assertEqual(module.__name__, 'pkg')
self.assertTrue(hasattr(module, 'subpkg2'))
@@ -142,8 +142,8 @@ class RelativeImports(unittest.TestCase):
{'__name__': 'pkg.pkg1.pkg2.pkg3.pkg4.pkg5',
'__path__': ['blah']})
def callback(global_):
- import_util.import_(globals_[0]['__package__'])
- module = import_util.import_('', global_, fromlist=['attr'], level=6)
+ self.__import__(globals_[0]['__package__'])
+ module = self.__import__('', global_, fromlist=['attr'], level=6)
self.assertEqual(module.__name__, 'pkg')
self.relative_import_test(create, globals_, callback)
@@ -153,9 +153,9 @@ class RelativeImports(unittest.TestCase):
globals_ = ({'__package__': 'pkg'},
{'__name__': 'pkg', '__path__': ['blah']})
def callback(global_):
- import_util.import_('pkg')
+ self.__import__('pkg')
with self.assertRaises(ValueError):
- import_util.import_('', global_, fromlist=['top_level'],
+ self.__import__('', global_, fromlist=['top_level'],
level=2)
self.relative_import_test(create, globals_, callback)
@@ -164,16 +164,16 @@ class RelativeImports(unittest.TestCase):
create = ['top_level', 'pkg.__init__', 'pkg.module']
globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'}
def callback(global_):
- import_util.import_('pkg')
+ self.__import__('pkg')
with self.assertRaises(ValueError):
- import_util.import_('', global_, fromlist=['top_level'],
+ self.__import__('', global_, fromlist=['top_level'],
level=2)
self.relative_import_test(create, globals_, callback)
def test_empty_name_w_level_0(self):
# [empty name]
with self.assertRaises(ValueError):
- import_util.import_('')
+ self.__import__('')
def test_import_from_different_package(self):
# Test importing from a different package than the caller.
@@ -186,8 +186,8 @@ class RelativeImports(unittest.TestCase):
'__runpy_pkg__.uncle.cousin.nephew']
globals_ = {'__package__': '__runpy_pkg__.__runpy_pkg__'}
def callback(global_):
- import_util.import_('__runpy_pkg__.__runpy_pkg__')
- module = import_util.import_('uncle.cousin', globals_, {},
+ self.__import__('__runpy_pkg__.__runpy_pkg__')
+ module = self.__import__('uncle.cousin', globals_, {},
fromlist=['nephew'],
level=2)
self.assertEqual(module.__name__, '__runpy_pkg__.uncle.cousin')
@@ -198,20 +198,19 @@ class RelativeImports(unittest.TestCase):
create = ['crash.__init__', 'crash.mod']
globals_ = [{'__package__': 'crash', '__name__': 'crash'}]
def callback(global_):
- import_util.import_('crash')
- mod = import_util.import_('mod', global_, {}, [], 1)
+ self.__import__('crash')
+ mod = self.__import__('mod', global_, {}, [], 1)
self.assertEqual(mod.__name__, 'crash.mod')
self.relative_import_test(create, globals_, callback)
def test_relative_import_no_globals(self):
# No globals for a relative import is an error.
with self.assertRaises(KeyError):
- import_util.import_('sys', level=1)
+ self.__import__('sys', level=1)
+Frozen_RelativeImports, Source_RelativeImports = util.test_both(
+ RelativeImports, __import__=import_util.__import__)
-def test_main():
- from test.support import run_unittest
- run_unittest(RelativeImports)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/import_/util.py b/Lib/test/test_importlib/import_/util.py
index 86ac065..dcb490f 100644
--- a/Lib/test/test_importlib/import_/util.py
+++ b/Lib/test/test_importlib/import_/util.py
@@ -1,22 +1,14 @@
+from .. import util
+
+frozen_importlib, source_importlib = util.import_importlib('importlib')
+
+import builtins
import functools
import importlib
import unittest
-using___import__ = False
-
-
-def import_(*args, **kwargs):
- """Delegate to allow for injecting different implementations of import."""
- if using___import__:
- return __import__(*args, **kwargs)
- else:
- return importlib.__import__(*args, **kwargs)
-
-
-def importlib_only(fxn):
- """Decorator to skip a test if using __builtins__.__import__."""
- return unittest.skipIf(using___import__, "importlib-specific test")(fxn)
+__import__ = staticmethod(builtins.__import__), staticmethod(source_importlib.__import__)
def mock_path_hook(*entries, importer):
diff --git a/Lib/test/namespace_pkgs/both_portions/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py
index 3080f6f..3080f6f 100644
--- a/Lib/test/namespace_pkgs/both_portions/foo/one.py
+++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/one.py
diff --git a/Lib/test/namespace_pkgs/both_portions/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py
index 4131d3d..4131d3d 100644
--- a/Lib/test/namespace_pkgs/both_portions/foo/two.py
+++ b/Lib/test/test_importlib/namespace_pkgs/both_portions/foo/two.py
diff --git a/Lib/test/namespace_pkgs/missing_directory.zip b/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip
index 836a910..836a910 100644
--- a/Lib/test/namespace_pkgs/missing_directory.zip
+++ b/Lib/test/test_importlib/namespace_pkgs/missing_directory.zip
Binary files differ
diff --git a/Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py
index 43cbedb..43cbedb 100644
--- a/Lib/test/namespace_pkgs/module_and_namespace_package/a_test.py
+++ b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test.py
diff --git a/Lib/test/namespace_pkgs/module_and_namespace_package/a_test/empty b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty
index e69de29..e69de29 100644
--- a/Lib/test/namespace_pkgs/module_and_namespace_package/a_test/empty
+++ b/Lib/test/test_importlib/namespace_pkgs/module_and_namespace_package/a_test/empty
diff --git a/Lib/test/namespace_pkgs/nested_portion1.zip b/Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip
index 8d22406..8d22406 100644
--- a/Lib/test/namespace_pkgs/nested_portion1.zip
+++ b/Lib/test/test_importlib/namespace_pkgs/nested_portion1.zip
Binary files differ
diff --git a/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
index e69de29..e69de29 100644
--- a/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
+++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py
diff --git a/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py
index d8f5c83..d8f5c83 100644
--- a/Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py
+++ b/Lib/test/test_importlib/namespace_pkgs/not_a_namespace_pkg/foo/one.py
diff --git a/Lib/test/namespace_pkgs/portion1/foo/one.py b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py
index d8f5c83..d8f5c83 100644
--- a/Lib/test/namespace_pkgs/portion1/foo/one.py
+++ b/Lib/test/test_importlib/namespace_pkgs/portion1/foo/one.py
diff --git a/Lib/test/namespace_pkgs/portion2/foo/two.py b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py
index d092e1e..d092e1e 100644
--- a/Lib/test/namespace_pkgs/portion2/foo/two.py
+++ b/Lib/test/test_importlib/namespace_pkgs/portion2/foo/two.py
diff --git a/Lib/test/namespace_pkgs/project1/parent/child/one.py b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py
index 2776fcd..2776fcd 100644
--- a/Lib/test/namespace_pkgs/project1/parent/child/one.py
+++ b/Lib/test/test_importlib/namespace_pkgs/project1/parent/child/one.py
diff --git a/Lib/test/namespace_pkgs/project2/parent/child/two.py b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py
index 8b037bc..8b037bc 100644
--- a/Lib/test/namespace_pkgs/project2/parent/child/two.py
+++ b/Lib/test/test_importlib/namespace_pkgs/project2/parent/child/two.py
diff --git a/Lib/test/namespace_pkgs/project3/parent/child/three.py b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py
index f8abfe1..f8abfe1 100644
--- a/Lib/test/namespace_pkgs/project3/parent/child/three.py
+++ b/Lib/test/test_importlib/namespace_pkgs/project3/parent/child/three.py
diff --git a/Lib/test/namespace_pkgs/top_level_portion1.zip b/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip
index 3b866c9..3b866c9 100644
--- a/Lib/test/namespace_pkgs/top_level_portion1.zip
+++ b/Lib/test/test_importlib/namespace_pkgs/top_level_portion1.zip
Binary files differ
diff --git a/Lib/test/test_importlib/source/test_abc_loader.py b/Lib/test/test_importlib/source/test_abc_loader.py
deleted file mode 100644
index 0d912b6..0000000
--- a/Lib/test/test_importlib/source/test_abc_loader.py
+++ /dev/null
@@ -1,906 +0,0 @@
-import importlib
-from importlib import abc
-
-from .. import abc as testing_abc
-from .. import util
-from . import util as source_util
-
-import imp
-import inspect
-import io
-import marshal
-import os
-import sys
-import types
-import unittest
-import warnings
-
-
-class SourceOnlyLoaderMock(abc.SourceLoader):
-
- # Globals that should be defined for all modules.
- source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, "
- b"repr(__loader__)])")
-
- def __init__(self, path):
- self.path = path
-
- def get_data(self, path):
- assert self.path == path
- return self.source
-
- def get_filename(self, fullname):
- return self.path
-
- def module_repr(self, module):
- return '<module>'
-
-
-class SourceLoaderMock(SourceOnlyLoaderMock):
-
- source_mtime = 1
-
- def __init__(self, path, magic=imp.get_magic()):
- super().__init__(path)
- self.bytecode_path = imp.cache_from_source(self.path)
- self.source_size = len(self.source)
- data = bytearray(magic)
- data.extend(importlib._w_long(self.source_mtime))
- data.extend(importlib._w_long(self.source_size))
- code_object = compile(self.source, self.path, 'exec',
- dont_inherit=True)
- data.extend(marshal.dumps(code_object))
- self.bytecode = bytes(data)
- self.written = {}
-
- def get_data(self, path):
- if path == self.path:
- return super().get_data(path)
- elif path == self.bytecode_path:
- return self.bytecode
- else:
- raise IOError
-
- def path_stats(self, path):
- assert path == self.path
- return {'mtime': self.source_mtime, 'size': self.source_size}
-
- def set_data(self, path, data):
- self.written[path] = bytes(data)
- return path == self.bytecode_path
-
-
-class PyLoaderMock(abc.PyLoader):
-
- # Globals that should be defined for all modules.
- source = (b"_ = '::'.join([__name__, __file__, __package__, "
- b"repr(__loader__)])")
-
- def __init__(self, data):
- """Take a dict of 'module_name: path' pairings.
-
- Paths should have no file extension, allowing packages to be denoted by
- ending in '__init__'.
-
- """
- self.module_paths = data
- self.path_to_module = {val:key for key,val in data.items()}
-
- def get_data(self, path):
- if path not in self.path_to_module:
- raise IOError
- return self.source
-
- def is_package(self, name):
- filename = os.path.basename(self.get_filename(name))
- return os.path.splitext(filename)[0] == '__init__'
-
- def source_path(self, name):
- try:
- return self.module_paths[name]
- except KeyError:
- raise ImportError
-
- def get_filename(self, name):
- """Silence deprecation warning."""
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- path = super().get_filename(name)
- assert len(w) == 1
- assert issubclass(w[0].category, DeprecationWarning)
- return path
-
- def module_repr(self):
- return '<module>'
-
-
-class PyLoaderCompatMock(PyLoaderMock):
-
- """Mock that matches what is suggested to have a loader that is compatible
- from Python 3.1 onwards."""
-
- def get_filename(self, fullname):
- try:
- return self.module_paths[fullname]
- except KeyError:
- raise ImportError
-
- def source_path(self, fullname):
- try:
- return self.get_filename(fullname)
- except ImportError:
- return None
-
-
-class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock):
-
- default_mtime = 1
-
- def __init__(self, source, bc={}):
- """Initialize mock.
-
- 'bc' is a dict keyed on a module's name. The value is dict with
- possible keys of 'path', 'mtime', 'magic', and 'bc'. Except for 'path',
- each of those keys control if any part of created bytecode is to
- deviate from default values.
-
- """
- super().__init__(source)
- self.module_bytecode = {}
- self.path_to_bytecode = {}
- self.bytecode_to_path = {}
- for name, data in bc.items():
- self.path_to_bytecode[data['path']] = name
- self.bytecode_to_path[name] = data['path']
- magic = data.get('magic', imp.get_magic())
- mtime = importlib._w_long(data.get('mtime', self.default_mtime))
- source_size = importlib._w_long(len(self.source) & 0xFFFFFFFF)
- if 'bc' in data:
- bc = data['bc']
- else:
- bc = self.compile_bc(name)
- self.module_bytecode[name] = magic + mtime + source_size + bc
-
- def compile_bc(self, name):
- source_path = self.module_paths.get(name, '<test>') or '<test>'
- code = compile(self.source, source_path, 'exec')
- return marshal.dumps(code)
-
- def source_mtime(self, name):
- if name in self.module_paths:
- return self.default_mtime
- elif name in self.module_bytecode:
- return None
- else:
- raise ImportError
-
- def bytecode_path(self, name):
- try:
- return self.bytecode_to_path[name]
- except KeyError:
- if name in self.module_paths:
- return None
- else:
- raise ImportError
-
- def write_bytecode(self, name, bytecode):
- self.module_bytecode[name] = bytecode
- return True
-
- def get_data(self, path):
- if path in self.path_to_module:
- return super().get_data(path)
- elif path in self.path_to_bytecode:
- name = self.path_to_bytecode[path]
- return self.module_bytecode[name]
- else:
- raise IOError
-
- def is_package(self, name):
- try:
- return super().is_package(name)
- except TypeError:
- return '__init__' in self.bytecode_to_path[name]
-
- def get_code(self, name):
- with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter("always")
- code_object = super().get_code(name)
- assert len(w) == 1
- assert issubclass(w[0].category, DeprecationWarning)
- return code_object
-
-class PyLoaderTests(testing_abc.LoaderTests):
-
- """Tests for importlib.abc.PyLoader."""
-
- mocker = PyLoaderMock
-
- def eq_attrs(self, ob, **kwargs):
- for attr, val in kwargs.items():
- found = getattr(ob, attr)
- self.assertEqual(found, val,
- "{} attribute: {} != {}".format(attr, found, val))
-
- def test_module(self):
- name = '<module>'
- path = os.path.join('', 'path', 'to', 'module')
- mock = self.mocker({name: path})
- with util.uncache(name):
- module = mock.load_module(name)
- self.assertIn(name, sys.modules)
- self.eq_attrs(module, __name__=name, __file__=path, __package__='',
- __loader__=mock)
- self.assertTrue(not hasattr(module, '__path__'))
- return mock, name
-
- def test_package(self):
- name = '<pkg>'
- path = os.path.join('path', 'to', name, '__init__')
- mock = self.mocker({name: path})
- with util.uncache(name):
- module = mock.load_module(name)
- self.assertIn(name, sys.modules)
- self.eq_attrs(module, __name__=name, __file__=path,
- __path__=[os.path.dirname(path)], __package__=name,
- __loader__=mock)
- return mock, name
-
- def test_lacking_parent(self):
- name = 'pkg.mod'
- path = os.path.join('path', 'to', 'pkg', 'mod')
- mock = self.mocker({name: path})
- with util.uncache(name):
- module = mock.load_module(name)
- self.assertIn(name, sys.modules)
- self.eq_attrs(module, __name__=name, __file__=path, __package__='pkg',
- __loader__=mock)
- self.assertFalse(hasattr(module, '__path__'))
- return mock, name
-
- def test_module_reuse(self):
- name = 'mod'
- path = os.path.join('path', 'to', 'mod')
- module = imp.new_module(name)
- mock = self.mocker({name: path})
- with util.uncache(name):
- sys.modules[name] = module
- loaded_module = mock.load_module(name)
- self.assertIs(loaded_module, module)
- self.assertIs(sys.modules[name], module)
- return mock, name
-
- def test_state_after_failure(self):
- name = "mod"
- module = imp.new_module(name)
- module.blah = None
- mock = self.mocker({name: os.path.join('path', 'to', 'mod')})
- mock.source = b"1/0"
- with util.uncache(name):
- sys.modules[name] = module
- with self.assertRaises(ZeroDivisionError):
- mock.load_module(name)
- self.assertIs(sys.modules[name], module)
- self.assertTrue(hasattr(module, 'blah'))
- return mock
-
- def test_unloadable(self):
- name = "mod"
- mock = self.mocker({name: os.path.join('path', 'to', 'mod')})
- mock.source = b"1/0"
- with util.uncache(name):
- with self.assertRaises(ZeroDivisionError):
- mock.load_module(name)
- self.assertNotIn(name, sys.modules)
- return mock
-
-
-class PyLoaderCompatTests(PyLoaderTests):
-
- """Test that the suggested code to make a loader that is compatible from
- Python 3.1 forward works."""
-
- mocker = PyLoaderCompatMock
-
-
-class PyLoaderInterfaceTests(unittest.TestCase):
-
- """Tests for importlib.abc.PyLoader to make sure that when source_path()
- doesn't return a path everything works as expected."""
-
- def test_no_source_path(self):
- # No source path should lead to ImportError.
- name = 'mod'
- mock = PyLoaderMock({})
- with util.uncache(name), self.assertRaises(ImportError):
- mock.load_module(name)
-
- def test_source_path_is_None(self):
- name = 'mod'
- mock = PyLoaderMock({name: None})
- with util.uncache(name), self.assertRaises(ImportError):
- mock.load_module(name)
-
- def test_get_filename_with_source_path(self):
- # get_filename() should return what source_path() returns.
- name = 'mod'
- path = os.path.join('path', 'to', 'source')
- mock = PyLoaderMock({name: path})
- with util.uncache(name):
- self.assertEqual(mock.get_filename(name), path)
-
- def test_get_filename_no_source_path(self):
- # get_filename() should raise ImportError if source_path returns None.
- name = 'mod'
- mock = PyLoaderMock({name: None})
- with util.uncache(name), self.assertRaises(ImportError):
- mock.get_filename(name)
-
-
-class PyPycLoaderTests(PyLoaderTests):
-
- """Tests for importlib.abc.PyPycLoader."""
-
- mocker = PyPycLoaderMock
-
- @source_util.writes_bytecode_files
- def verify_bytecode(self, mock, name):
- assert name in mock.module_paths
- self.assertIn(name, mock.module_bytecode)
- magic = mock.module_bytecode[name][:4]
- self.assertEqual(magic, imp.get_magic())
- mtime = importlib._r_long(mock.module_bytecode[name][4:8])
- self.assertEqual(mtime, 1)
- source_size = mock.module_bytecode[name][8:12]
- self.assertEqual(len(mock.source) & 0xFFFFFFFF,
- importlib._r_long(source_size))
- bc = mock.module_bytecode[name][12:]
- self.assertEqual(bc, mock.compile_bc(name))
-
- def test_module(self):
- mock, name = super().test_module()
- self.verify_bytecode(mock, name)
-
- def test_package(self):
- mock, name = super().test_package()
- self.verify_bytecode(mock, name)
-
- def test_lacking_parent(self):
- mock, name = super().test_lacking_parent()
- self.verify_bytecode(mock, name)
-
- def test_module_reuse(self):
- mock, name = super().test_module_reuse()
- self.verify_bytecode(mock, name)
-
- def test_state_after_failure(self):
- super().test_state_after_failure()
-
- def test_unloadable(self):
- super().test_unloadable()
-
-
-class PyPycLoaderInterfaceTests(unittest.TestCase):
-
- """Test for the interface of importlib.abc.PyPycLoader."""
-
- def get_filename_check(self, src_path, bc_path, expect):
- name = 'mod'
- mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}})
- with util.uncache(name):
- assert mock.source_path(name) == src_path
- assert mock.bytecode_path(name) == bc_path
- self.assertEqual(mock.get_filename(name), expect)
-
- def test_filename_with_source_bc(self):
- # When source and bytecode paths present, return the source path.
- self.get_filename_check('source_path', 'bc_path', 'source_path')
-
- def test_filename_with_source_no_bc(self):
- # With source but no bc, return source path.
- self.get_filename_check('source_path', None, 'source_path')
-
- def test_filename_with_no_source_bc(self):
- # With not source but bc, return the bc path.
- self.get_filename_check(None, 'bc_path', 'bc_path')
-
- def test_filename_with_no_source_or_bc(self):
- # With no source or bc, raise ImportError.
- name = 'mod'
- mock = PyPycLoaderMock({name: None}, {name: {'path': None}})
- with util.uncache(name), self.assertRaises(ImportError):
- mock.get_filename(name)
-
-
-class SkipWritingBytecodeTests(unittest.TestCase):
-
- """Test that bytecode is properly handled based on
- sys.dont_write_bytecode."""
-
- @source_util.writes_bytecode_files
- def run_test(self, dont_write_bytecode):
- name = 'mod'
- mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')})
- sys.dont_write_bytecode = dont_write_bytecode
- with util.uncache(name):
- mock.load_module(name)
- self.assertIsNot(name in mock.module_bytecode, dont_write_bytecode)
-
- def test_no_bytecode_written(self):
- self.run_test(True)
-
- def test_bytecode_written(self):
- self.run_test(False)
-
-
-class RegeneratedBytecodeTests(unittest.TestCase):
-
- """Test that bytecode is regenerated as expected."""
-
- @source_util.writes_bytecode_files
- def test_different_magic(self):
- # A different magic number should lead to new bytecode.
- name = 'mod'
- bad_magic = b'\x00\x00\x00\x00'
- assert bad_magic != imp.get_magic()
- mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')},
- {name: {'path': os.path.join('path', 'to',
- 'mod.bytecode'),
- 'magic': bad_magic}})
- with util.uncache(name):
- mock.load_module(name)
- self.assertIn(name, mock.module_bytecode)
- magic = mock.module_bytecode[name][:4]
- self.assertEqual(magic, imp.get_magic())
-
- @source_util.writes_bytecode_files
- def test_old_mtime(self):
- # Bytecode with an older mtime should be regenerated.
- name = 'mod'
- old_mtime = PyPycLoaderMock.default_mtime - 1
- mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')},
- {name: {'path': 'path/to/mod.bytecode', 'mtime': old_mtime}})
- with util.uncache(name):
- mock.load_module(name)
- self.assertIn(name, mock.module_bytecode)
- mtime = importlib._r_long(mock.module_bytecode[name][4:8])
- self.assertEqual(mtime, PyPycLoaderMock.default_mtime)
-
-
-class BadBytecodeFailureTests(unittest.TestCase):
-
- """Test import failures when there is no source and parts of the bytecode
- is bad."""
-
- def test_bad_magic(self):
- # A bad magic number should lead to an ImportError.
- name = 'mod'
- bad_magic = b'\x00\x00\x00\x00'
- bc = {name:
- {'path': os.path.join('path', 'to', 'mod'),
- 'magic': bad_magic}}
- mock = PyPycLoaderMock({name: None}, bc)
- with util.uncache(name), self.assertRaises(ImportError) as cm:
- mock.load_module(name)
- self.assertEqual(cm.exception.name, name)
-
- def test_no_bytecode(self):
- # Missing code object bytecode should lead to an EOFError.
- name = 'mod'
- bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}}
- mock = PyPycLoaderMock({name: None}, bc)
- with util.uncache(name), self.assertRaises(EOFError):
- mock.load_module(name)
-
- def test_bad_bytecode(self):
- # Malformed code object bytecode should lead to a ValueError.
- name = 'mod'
- bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}}
- mock = PyPycLoaderMock({name: None}, bc)
- with util.uncache(name), self.assertRaises(ValueError):
- mock.load_module(name)
-
-
-def raise_ImportError(*args, **kwargs):
- raise ImportError
-
-class MissingPathsTests(unittest.TestCase):
-
- """Test what happens when a source or bytecode path does not exist (either
- from *_path returning None or raising ImportError)."""
-
- def test_source_path_None(self):
- # Bytecode should be used when source_path returns None, along with
- # __file__ being set to the bytecode path.
- name = 'mod'
- bytecode_path = 'path/to/mod'
- mock = PyPycLoaderMock({name: None}, {name: {'path': bytecode_path}})
- with util.uncache(name):
- module = mock.load_module(name)
- self.assertEqual(module.__file__, bytecode_path)
-
- # Testing for bytecode_path returning None handled by all tests where no
- # bytecode initially exists.
-
- def test_all_paths_None(self):
- # If all *_path methods return None, raise ImportError.
- name = 'mod'
- mock = PyPycLoaderMock({name: None})
- with util.uncache(name), self.assertRaises(ImportError) as cm:
- mock.load_module(name)
- self.assertEqual(cm.exception.name, name)
-
- def test_source_path_ImportError(self):
- # An ImportError from source_path should trigger an ImportError.
- name = 'mod'
- mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to',
- 'mod')}})
- with util.uncache(name), self.assertRaises(ImportError):
- mock.load_module(name)
-
- def test_bytecode_path_ImportError(self):
- # An ImportError from bytecode_path should trigger an ImportError.
- name = 'mod'
- mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')})
- bad_meth = types.MethodType(raise_ImportError, mock)
- mock.bytecode_path = bad_meth
- with util.uncache(name), self.assertRaises(ImportError) as cm:
- mock.load_module(name)
-
-
-class SourceLoaderTestHarness(unittest.TestCase):
-
- def setUp(self, *, is_package=True, **kwargs):
- self.package = 'pkg'
- if is_package:
- self.path = os.path.join(self.package, '__init__.py')
- self.name = self.package
- else:
- module_name = 'mod'
- self.path = os.path.join(self.package, '.'.join(['mod', 'py']))
- self.name = '.'.join([self.package, module_name])
- self.cached = imp.cache_from_source(self.path)
- self.loader = self.loader_mock(self.path, **kwargs)
-
- def verify_module(self, module):
- self.assertEqual(module.__name__, self.name)
- self.assertEqual(module.__file__, self.path)
- self.assertEqual(module.__cached__, self.cached)
- self.assertEqual(module.__package__, self.package)
- self.assertEqual(module.__loader__, self.loader)
- values = module._.split('::')
- self.assertEqual(values[0], self.name)
- self.assertEqual(values[1], self.path)
- self.assertEqual(values[2], self.cached)
- self.assertEqual(values[3], self.package)
- self.assertEqual(values[4], repr(self.loader))
-
- def verify_code(self, code_object):
- module = imp.new_module(self.name)
- module.__file__ = self.path
- module.__cached__ = self.cached
- module.__package__ = self.package
- module.__loader__ = self.loader
- module.__path__ = []
- exec(code_object, module.__dict__)
- self.verify_module(module)
-
-
-class SourceOnlyLoaderTests(SourceLoaderTestHarness):
-
- """Test importlib.abc.SourceLoader for source-only loading.
-
- Reload testing is subsumed by the tests for
- importlib.util.module_for_loader.
-
- """
-
- loader_mock = SourceOnlyLoaderMock
-
- def test_get_source(self):
- # Verify the source code is returned as a string.
- # If an IOError is raised by get_data then raise ImportError.
- expected_source = self.loader.source.decode('utf-8')
- self.assertEqual(self.loader.get_source(self.name), expected_source)
- def raise_IOError(path):
- raise IOError
- self.loader.get_data = raise_IOError
- with self.assertRaises(ImportError) as cm:
- self.loader.get_source(self.name)
- self.assertEqual(cm.exception.name, self.name)
-
- def test_is_package(self):
- # Properly detect when loading a package.
- self.setUp(is_package=False)
- self.assertFalse(self.loader.is_package(self.name))
- self.setUp(is_package=True)
- self.assertTrue(self.loader.is_package(self.name))
- self.assertFalse(self.loader.is_package(self.name + '.__init__'))
-
- def test_get_code(self):
- # Verify the code object is created.
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object)
-
- def test_load_module(self):
- # Loading a module should set __name__, __loader__, __package__,
- # __path__ (for packages), __file__, and __cached__.
- # The module should also be put into sys.modules.
- with util.uncache(self.name):
- module = self.loader.load_module(self.name)
- self.verify_module(module)
- self.assertEqual(module.__path__, [os.path.dirname(self.path)])
- self.assertIn(self.name, sys.modules)
-
- def test_package_settings(self):
- # __package__ needs to be set, while __path__ is set on if the module
- # is a package.
- # Testing the values for a package are covered by test_load_module.
- self.setUp(is_package=False)
- with util.uncache(self.name):
- module = self.loader.load_module(self.name)
- self.verify_module(module)
- self.assertTrue(not hasattr(module, '__path__'))
-
- def test_get_source_encoding(self):
- # Source is considered encoded in UTF-8 by default unless otherwise
- # specified by an encoding line.
- source = "_ = 'ü'"
- self.loader.source = source.encode('utf-8')
- returned_source = self.loader.get_source(self.name)
- self.assertEqual(returned_source, source)
- source = "# coding: latin-1\n_ = ü"
- self.loader.source = source.encode('latin-1')
- returned_source = self.loader.get_source(self.name)
- self.assertEqual(returned_source, source)
-
-
-@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true")
-class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
-
- """Test importlib.abc.SourceLoader's use of bytecode.
-
- Source-only testing handled by SourceOnlyLoaderTests.
-
- """
-
- loader_mock = SourceLoaderMock
-
- def verify_code(self, code_object, *, bytecode_written=False):
- super().verify_code(code_object)
- if bytecode_written:
- self.assertIn(self.cached, self.loader.written)
- data = bytearray(imp.get_magic())
- data.extend(importlib._w_long(self.loader.source_mtime))
- data.extend(importlib._w_long(self.loader.source_size))
- data.extend(marshal.dumps(code_object))
- self.assertEqual(self.loader.written[self.cached], bytes(data))
-
- def test_code_with_everything(self):
- # When everything should work.
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object)
-
- def test_no_bytecode(self):
- # If no bytecode exists then move on to the source.
- self.loader.bytecode_path = "<does not exist>"
- # Sanity check
- with self.assertRaises(IOError):
- bytecode_path = imp.cache_from_source(self.path)
- self.loader.get_data(bytecode_path)
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object, bytecode_written=True)
-
- def test_code_bad_timestamp(self):
- # Bytecode is only used when the timestamp matches the source EXACTLY.
- for source_mtime in (0, 2):
- assert source_mtime != self.loader.source_mtime
- original = self.loader.source_mtime
- self.loader.source_mtime = source_mtime
- # If bytecode is used then EOFError would be raised by marshal.
- self.loader.bytecode = self.loader.bytecode[8:]
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object, bytecode_written=True)
- self.loader.source_mtime = original
-
- def test_code_bad_magic(self):
- # Skip over bytecode with a bad magic number.
- self.setUp(magic=b'0000')
- # If bytecode is used then EOFError would be raised by marshal.
- self.loader.bytecode = self.loader.bytecode[8:]
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object, bytecode_written=True)
-
- def test_dont_write_bytecode(self):
- # Bytecode is not written if sys.dont_write_bytecode is true.
- # Can assume it is false already thanks to the skipIf class decorator.
- try:
- sys.dont_write_bytecode = True
- self.loader.bytecode_path = "<does not exist>"
- code_object = self.loader.get_code(self.name)
- self.assertNotIn(self.cached, self.loader.written)
- finally:
- sys.dont_write_bytecode = False
-
- def test_no_set_data(self):
- # If set_data is not defined, one can still read bytecode.
- self.setUp(magic=b'0000')
- original_set_data = self.loader.__class__.set_data
- try:
- del self.loader.__class__.set_data
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object)
- finally:
- self.loader.__class__.set_data = original_set_data
-
- def test_set_data_raises_exceptions(self):
- # Raising NotImplementedError or IOError is okay for set_data.
- def raise_exception(exc):
- def closure(*args, **kwargs):
- raise exc
- return closure
-
- self.setUp(magic=b'0000')
- self.loader.set_data = raise_exception(NotImplementedError)
- code_object = self.loader.get_code(self.name)
- self.verify_code(code_object)
-
-
-class SourceLoaderGetSourceTests(unittest.TestCase):
-
- """Tests for importlib.abc.SourceLoader.get_source()."""
-
- def test_default_encoding(self):
- # Should have no problems with UTF-8 text.
- name = 'mod'
- mock = SourceOnlyLoaderMock('mod.file')
- source = 'x = "ü"'
- mock.source = source.encode('utf-8')
- returned_source = mock.get_source(name)
- self.assertEqual(returned_source, source)
-
- def test_decoded_source(self):
- # Decoding should work.
- name = 'mod'
- mock = SourceOnlyLoaderMock("mod.file")
- source = "# coding: Latin-1\nx='ü'"
- assert source.encode('latin-1') != source.encode('utf-8')
- mock.source = source.encode('latin-1')
- returned_source = mock.get_source(name)
- self.assertEqual(returned_source, source)
-
- def test_universal_newlines(self):
- # PEP 302 says universal newlines should be used.
- name = 'mod'
- mock = SourceOnlyLoaderMock('mod.file')
- source = "x = 42\r\ny = -13\r\n"
- mock.source = source.encode('utf-8')
- expect = io.IncrementalNewlineDecoder(None, True).decode(source)
- self.assertEqual(mock.get_source(name), expect)
-
-
-class AbstractMethodImplTests(unittest.TestCase):
-
- """Test the concrete abstractmethod implementations."""
-
- class MetaPathFinder(abc.MetaPathFinder):
- def find_module(self, fullname, path):
- super().find_module(fullname, path)
-
- class PathEntryFinder(abc.PathEntryFinder):
- def find_module(self, _):
- super().find_module(_)
-
- def find_loader(self, _):
- super().find_loader(_)
-
- class Finder(abc.Finder):
- def find_module(self, fullname, path):
- super().find_module(fullname, path)
-
- class Loader(abc.Loader):
- def load_module(self, fullname):
- super().load_module(fullname)
- def module_repr(self, module):
- super().module_repr(module)
-
- class ResourceLoader(Loader, abc.ResourceLoader):
- def get_data(self, _):
- super().get_data(_)
-
- class InspectLoader(Loader, abc.InspectLoader):
- def is_package(self, _):
- super().is_package(_)
-
- def get_code(self, _):
- super().get_code(_)
-
- def get_source(self, _):
- super().get_source(_)
-
- class ExecutionLoader(InspectLoader, abc.ExecutionLoader):
- def get_filename(self, _):
- super().get_filename(_)
-
- class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader):
- pass
-
- class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader):
- def source_path(self, _):
- super().source_path(_)
-
- class PyPycLoader(PyLoader, abc.PyPycLoader):
- def bytecode_path(self, _):
- super().bytecode_path(_)
-
- def source_mtime(self, _):
- super().source_mtime(_)
-
- def write_bytecode(self, _, _2):
- super().write_bytecode(_, _2)
-
- def raises_NotImplementedError(self, ins, *args):
- for method_name in args:
- method = getattr(ins, method_name)
- arg_count = len(inspect.getfullargspec(method)[0]) - 1
- args = [''] * arg_count
- try:
- method(*args)
- except NotImplementedError:
- pass
- else:
- msg = "{}.{} did not raise NotImplementedError"
- self.fail(msg.format(ins.__class__.__name__, method_name))
-
- def test_Loader(self):
- self.raises_NotImplementedError(self.Loader(), 'load_module')
-
- # XXX misplaced; should be somewhere else
- def test_Finder(self):
- self.raises_NotImplementedError(self.Finder(), 'find_module')
-
- def test_ResourceLoader(self):
- self.raises_NotImplementedError(self.ResourceLoader(), 'load_module',
- 'get_data')
-
- def test_InspectLoader(self):
- self.raises_NotImplementedError(self.InspectLoader(), 'load_module',
- 'is_package', 'get_code', 'get_source')
-
- def test_ExecutionLoader(self):
- self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module',
- 'is_package', 'get_code', 'get_source',
- 'get_filename')
-
- def test_SourceLoader(self):
- ins = self.SourceLoader()
- # Required abstractmethods.
- self.raises_NotImplementedError(ins, 'get_filename', 'get_data')
- # Optional abstractmethods.
- self.raises_NotImplementedError(ins,'path_stats', 'set_data')
-
- def test_PyLoader(self):
- self.raises_NotImplementedError(self.PyLoader(), 'source_path',
- 'get_data', 'is_package')
-
- def test_PyPycLoader(self):
- self.raises_NotImplementedError(self.PyPycLoader(), 'source_path',
- 'source_mtime', 'bytecode_path',
- 'write_bytecode')
-
-
-def test_main():
- from test.support import run_unittest
- run_unittest(PyLoaderTests, PyLoaderCompatTests,
- PyLoaderInterfaceTests,
- PyPycLoaderTests, PyPycLoaderInterfaceTests,
- SkipWritingBytecodeTests, RegeneratedBytecodeTests,
- BadBytecodeFailureTests, MissingPathsTests,
- SourceOnlyLoaderTests,
- SourceLoaderBytecodeTests,
- SourceLoaderGetSourceTests,
- AbstractMethodImplTests)
-
-
-if __name__ == '__main__':
- test_main()
diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
index 241173f..efd3146 100644
--- a/Lib/test/test_importlib/source/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -1,9 +1,10 @@
"""Test case-sensitivity (PEP 235)."""
-from importlib import _bootstrap
-from importlib import machinery
from .. import util
from . import util as source_util
-import imp
+
+importlib = util.import_importlib('importlib')
+machinery = util.import_importlib('importlib.machinery')
+
import os
import sys
from test import support as test_support
@@ -11,7 +12,7 @@ import unittest
@util.case_insensitive_tests
-class CaseSensitivityTest(unittest.TestCase):
+class CaseSensitivityTest:
"""PEP 235 dictates that on case-preserving, case-insensitive file systems
that imports are case-sensitive unless the PYTHONCASEOK environment
@@ -20,13 +21,12 @@ class CaseSensitivityTest(unittest.TestCase):
name = 'MoDuLe'
assert name != name.lower()
- def find(self, path):
- finder = machinery.FileFinder(path,
- (machinery.SourceFileLoader,
- machinery.SOURCE_SUFFIXES),
- (machinery.SourcelessFileLoader,
- machinery.BYTECODE_SUFFIXES))
- return finder.find_module(self.name)
+ def finder(self, path):
+ return self.machinery.FileFinder(path,
+ (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES),
+ (self.machinery.SourcelessFileLoader,
+ self.machinery.BYTECODE_SUFFIXES))
def sensitivity_test(self):
"""Look for a module with matching and non-matching sensitivity."""
@@ -36,35 +36,48 @@ class CaseSensitivityTest(unittest.TestCase):
with context as mapping:
sensitive_path = os.path.join(mapping['.root'], 'sensitive')
insensitive_path = os.path.join(mapping['.root'], 'insensitive')
- return self.find(sensitive_path), self.find(insensitive_path)
+ sensitive_finder = self.finder(sensitive_path)
+ insensitive_finder = self.finder(insensitive_path)
+ return self.find(sensitive_finder), self.find(insensitive_finder)
def test_sensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.unset('PYTHONCASEOK')
- if b'PYTHONCASEOK' in _bootstrap._os.environ:
+ if b'PYTHONCASEOK' in self.importlib._bootstrap._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
sensitive, insensitive = self.sensitivity_test()
- self.assertTrue(hasattr(sensitive, 'load_module'))
+ self.assertIsNotNone(sensitive)
self.assertIn(self.name, sensitive.get_filename(self.name))
self.assertIsNone(insensitive)
def test_insensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.set('PYTHONCASEOK', '1')
- if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+ if b'PYTHONCASEOK' not in self.importlib._bootstrap._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
sensitive, insensitive = self.sensitivity_test()
- self.assertTrue(hasattr(sensitive, 'load_module'))
+ self.assertIsNotNone(sensitive)
self.assertIn(self.name, sensitive.get_filename(self.name))
- self.assertTrue(hasattr(insensitive, 'load_module'))
+ self.assertIsNotNone(insensitive)
self.assertIn(self.name, insensitive.get_filename(self.name))
+class CaseSensitivityTestPEP302(CaseSensitivityTest):
+ def find(self, finder):
+ return finder.find_module(self.name)
+
+Frozen_CaseSensitivityTestPEP302, Source_CaseSensitivityTestPEP302 = util.test_both(
+ CaseSensitivityTestPEP302, importlib=importlib, machinery=machinery)
+
+class CaseSensitivityTestPEP451(CaseSensitivityTest):
+ def find(self, finder):
+ found = finder.find_spec(self.name)
+ return found.loader if found is not None else found
-def test_main():
- test_support.run_unittest(CaseSensitivityTest)
+Frozen_CaseSensitivityTestPEP451, Source_CaseSensitivityTestPEP451 = util.test_both(
+ CaseSensitivityTestPEP451, importlib=importlib, machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index d711cb3..2d415f9 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -1,52 +1,52 @@
-from importlib import machinery
-import importlib
-import importlib.abc
from .. import abc
from .. import util
from . import util as source_util
+importlib = util.import_importlib('importlib')
+importlib_abc = util.import_importlib('importlib.abc')
+machinery = util.import_importlib('importlib.machinery')
+importlib_util = util.import_importlib('importlib.util')
+
import errno
-import imp
import marshal
import os
import py_compile
import shutil
import stat
import sys
+import types
import unittest
+import warnings
-from test.support import make_legacy_pyc
+from test.support import make_legacy_pyc, unload
-class SimpleTest(unittest.TestCase):
+class SimpleTest(abc.LoaderTests):
"""Should have no issue importing a source module [basic]. And if there is
a syntax error, it should raise a SyntaxError [syntax error].
"""
- def test_load_module_API(self):
- # If fullname is not specified that assume self.name is desired.
- class TesterMixin(importlib.abc.Loader):
- def load_module(self, fullname): return fullname
- def module_repr(self, module): return '<module>'
+ def setUp(self):
+ self.name = 'spam'
+ self.filepath = os.path.join('ham', self.name + '.py')
+ self.loader = self.machinery.SourceFileLoader(self.name, self.filepath)
- class Tester(importlib.abc.FileLoader, TesterMixin):
- def get_code(self, _): pass
- def get_source(self, _): pass
- def is_package(self, _): pass
+ def test_load_module_API(self):
+ class Tester(self.abc.FileLoader):
+ def get_source(self, _): return 'attr = 42'
+ def is_package(self, _): return False
- name = 'mod_name'
- loader = Tester(name, 'some_path')
- self.assertEqual(name, loader.load_module())
- self.assertEqual(name, loader.load_module(None))
- self.assertEqual(name, loader.load_module(name))
- with self.assertRaises(ImportError):
- loader.load_module(loader.name + 'XXX')
+ loader = Tester('blah', 'blah.py')
+ self.addCleanup(unload, 'blah')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module() # Should not raise an exception.
def test_get_filename_API(self):
# If fullname is not set then assume self.path is desired.
- class Tester(importlib.abc.FileLoader):
+ class Tester(self.abc.FileLoader):
def get_code(self, _): pass
def get_source(self, _): pass
def is_package(self, _): pass
@@ -61,11 +61,21 @@ class SimpleTest(unittest.TestCase):
with self.assertRaises(ImportError):
loader.get_filename(name + 'XXX')
+ def test_equality(self):
+ other = self.machinery.SourceFileLoader(self.name, self.filepath)
+ self.assertEqual(self.loader, other)
+
+ def test_inequality(self):
+ other = self.machinery.SourceFileLoader('_' + self.name, self.filepath)
+ self.assertNotEqual(self.loader, other)
+
# [basic]
def test_module(self):
with source_util.create_modules('_temp') as mapping:
- loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
- module = loader.load_module('_temp')
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
self.assertIn('_temp', sys.modules)
check = {'__name__': '_temp', '__file__': mapping['_temp'],
'__package__': ''}
@@ -74,9 +84,11 @@ class SimpleTest(unittest.TestCase):
def test_package(self):
with source_util.create_modules('_pkg.__init__') as mapping:
- loader = machinery.SourceFileLoader('_pkg',
+ loader = self.machinery.SourceFileLoader('_pkg',
mapping['_pkg.__init__'])
- module = loader.load_module('_pkg')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_pkg')
self.assertIn('_pkg', sys.modules)
check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'],
'__path__': [os.path.dirname(mapping['_pkg.__init__'])],
@@ -87,9 +99,11 @@ class SimpleTest(unittest.TestCase):
def test_lacking_parent(self):
with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping:
- loader = machinery.SourceFileLoader('_pkg.mod',
+ loader = self.machinery.SourceFileLoader('_pkg.mod',
mapping['_pkg.mod'])
- module = loader.load_module('_pkg.mod')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_pkg.mod')
self.assertIn('_pkg.mod', sys.modules)
check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'],
'__package__': '_pkg'}
@@ -102,13 +116,17 @@ class SimpleTest(unittest.TestCase):
def test_module_reuse(self):
with source_util.create_modules('_temp') as mapping:
- loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
- module = loader.load_module('_temp')
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
module_id = id(module)
module_dict_id = id(module.__dict__)
with open(mapping['_temp'], 'w') as file:
file.write("testing_var = 42\n")
- module = loader.load_module('_temp')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module('_temp')
self.assertIn('testing_var', module.__dict__,
"'testing_var' not in "
"{0}".format(list(module.__dict__.keys())))
@@ -122,14 +140,20 @@ class SimpleTest(unittest.TestCase):
value = '<test>'
name = '_temp'
with source_util.create_modules(name) as mapping:
- orig_module = imp.new_module(name)
+ orig_module = types.ModuleType(name)
for attr in attributes:
setattr(orig_module, attr, value)
with open(mapping[name], 'w') as file:
file.write('+++ bad syntax +++')
- loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ with self.assertRaises(SyntaxError):
+ loader.exec_module(orig_module)
+ for attr in attributes:
+ self.assertEqual(getattr(orig_module, attr), value)
with self.assertRaises(SyntaxError):
- loader.load_module(name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module(name)
for attr in attributes:
self.assertEqual(getattr(orig_module, attr), value)
@@ -138,9 +162,11 @@ class SimpleTest(unittest.TestCase):
with source_util.create_modules('_temp') as mapping:
with open(mapping['_temp'], 'w') as file:
file.write('=')
- loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
with self.assertRaises(SyntaxError):
- loader.load_module('_temp')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module('_temp')
self.assertNotIn('_temp', sys.modules)
def test_file_from_empty_string_dir(self):
@@ -151,14 +177,16 @@ class SimpleTest(unittest.TestCase):
file.write("# test file for importlib")
try:
with util.uncache('_temp'):
- loader = machinery.SourceFileLoader('_temp', file_path)
- mod = loader.load_module('_temp')
+ loader = self.machinery.SourceFileLoader('_temp', file_path)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ mod = loader.load_module('_temp')
self.assertEqual(file_path, mod.__file__)
- self.assertEqual(imp.cache_from_source(file_path),
+ self.assertEqual(self.util.cache_from_source(file_path),
mod.__cached__)
finally:
os.unlink(file_path)
- pycache = os.path.dirname(imp.cache_from_source(file_path))
+ pycache = os.path.dirname(self.util.cache_from_source(file_path))
if os.path.exists(pycache):
shutil.rmtree(pycache)
@@ -168,7 +196,7 @@ class SimpleTest(unittest.TestCase):
# truncated rather than raise an OverflowError.
with source_util.create_modules('_temp') as mapping:
source = mapping['_temp']
- compiled = imp.cache_from_source(source)
+ compiled = self.util.cache_from_source(source)
with open(source, 'w') as f:
f.write("x = 5")
try:
@@ -179,20 +207,48 @@ class SimpleTest(unittest.TestCase):
if e.errno != getattr(errno, 'EOVERFLOW', None):
raise
self.skipTest("cannot set modification time to large integer ({})".format(e))
- loader = machinery.SourceFileLoader('_temp', mapping['_temp'])
- mod = loader.load_module('_temp')
+ loader = self.machinery.SourceFileLoader('_temp', mapping['_temp'])
+ # PEP 451
+ module = types.ModuleType('_temp')
+ module.__spec__ = self.util.spec_from_loader('_temp', loader)
+ loader.exec_module(module)
+ self.assertEqual(module.x, 5)
+ self.assertTrue(os.path.exists(compiled))
+ os.unlink(compiled)
+ # PEP 302
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ mod = loader.load_module('_temp') # XXX
# Sanity checks.
self.assertEqual(mod.__cached__, compiled)
self.assertEqual(mod.x, 5)
# The pyc file was created.
- os.stat(compiled)
+ self.assertTrue(os.path.exists(compiled))
+
+ def test_unloadable(self):
+ loader = self.machinery.SourceFileLoader('good name', {})
+ module = types.ModuleType('bad name')
+ module.__spec__ = self.machinery.ModuleSpec('bad name', loader)
+ with self.assertRaises(ImportError):
+ loader.exec_module(module)
+ with self.assertRaises(ImportError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module('bad name')
+
+Frozen_SimpleTest, Source_SimpleTest = util.test_both(
+ SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc,
+ util=importlib_util)
-class BadBytecodeTest(unittest.TestCase):
+class BadBytecodeTest:
def import_(self, file, module_name):
loader = self.loader(module_name, file)
- module = loader.load_module(module_name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ # XXX Change to use exec_module().
+ module = loader.load_module(module_name)
self.assertIn(module_name, sys.modules)
def manipulate_bytecode(self, name, mapping, manipulator, *,
@@ -205,7 +261,7 @@ class BadBytecodeTest(unittest.TestCase):
pass
py_compile.compile(mapping[name])
if not del_source:
- bytecode_path = imp.cache_from_source(mapping[name])
+ bytecode_path = self.util.cache_from_source(mapping[name])
else:
os.unlink(mapping[name])
bytecode_path = make_legacy_pyc(mapping[name])
@@ -291,10 +347,29 @@ class BadBytecodeTest(unittest.TestCase):
lambda bc: b'\x00\x00\x00\x00' + bc[4:])
test('_temp', mapping, bc_path)
+class BadBytecodeTestPEP451(BadBytecodeTest):
+
+ def import_(self, file, module_name):
+ loader = self.loader(module_name, file)
+ module = types.ModuleType(module_name)
+ module.__spec__ = self.util.spec_from_loader(module_name, loader)
+ loader.exec_module(module)
+
+class BadBytecodeTestPEP302(BadBytecodeTest):
+
+ def import_(self, file, module_name):
+ loader = self.loader(module_name, file)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = loader.load_module(module_name)
+ self.assertIn(module_name, sys.modules)
-class SourceLoaderBadBytecodeTest(BadBytecodeTest):
- loader = machinery.SourceFileLoader
+class SourceLoaderBadBytecodeTest:
+
+ @classmethod
+ def setUpClass(cls):
+ cls.loader = cls.machinery.SourceFileLoader
@source_util.writes_bytecode_files
def test_empty_file(self):
@@ -333,7 +408,8 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as bytecode_file:
- self.assertEqual(bytecode_file.read(4), imp.get_magic())
+ self.assertEqual(bytecode_file.read(4),
+ self.util.MAGIC_NUMBER)
self._test_bad_magic(test)
@@ -383,13 +459,13 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
zeros = b'\x00\x00\x00\x00'
with source_util.create_modules('_temp') as mapping:
py_compile.compile(mapping['_temp'])
- bytecode_path = imp.cache_from_source(mapping['_temp'])
+ bytecode_path = self.util.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(4)
bytecode_file.write(zeros)
self.import_(mapping['_temp'], '_temp')
source_mtime = os.path.getmtime(mapping['_temp'])
- source_timestamp = importlib._w_long(source_mtime)
+ source_timestamp = self.importlib._w_long(source_mtime)
with open(bytecode_path, 'rb') as bytecode_file:
bytecode_file.seek(4)
self.assertEqual(bytecode_file.read(4), source_timestamp)
@@ -401,7 +477,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
with source_util.create_modules('_temp') as mapping:
# Create bytecode that will need to be re-created.
py_compile.compile(mapping['_temp'])
- bytecode_path = imp.cache_from_source(mapping['_temp'])
+ bytecode_path = self.util.cache_from_source(mapping['_temp'])
with open(bytecode_path, 'r+b') as bytecode_file:
bytecode_file.seek(0)
bytecode_file.write(b'\x00\x00\x00\x00')
@@ -409,16 +485,34 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
os.chmod(bytecode_path,
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
try:
- # Should not raise IOError!
+ # Should not raise OSError!
self.import_(mapping['_temp'], '_temp')
finally:
# Make writable for eventual clean-up.
os.chmod(bytecode_path, stat.S_IWUSR)
+class SourceLoaderBadBytecodeTestPEP451(
+ SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451):
+ pass
+
+Frozen_SourceBadBytecodePEP451, Source_SourceBadBytecodePEP451 = util.test_both(
+ SourceLoaderBadBytecodeTestPEP451, importlib=importlib, machinery=machinery,
+ abc=importlib_abc, util=importlib_util)
+
+class SourceLoaderBadBytecodeTestPEP302(
+ SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302):
+ pass
+
+Frozen_SourceBadBytecodePEP302, Source_SourceBadBytecodePEP302 = util.test_both(
+ SourceLoaderBadBytecodeTestPEP302, importlib=importlib, machinery=machinery,
+ abc=importlib_abc, util=importlib_util)
-class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
- loader = machinery.SourcelessFileLoader
+class SourcelessLoaderBadBytecodeTest:
+
+ @classmethod
+ def setUpClass(cls):
+ cls.loader = cls.machinery.SourcelessFileLoader
def test_empty_file(self):
def test(name, mapping, bytecode_path):
@@ -473,14 +567,22 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
def test_non_code_marshal(self):
self._test_non_code_marshal(del_source=True)
+class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest,
+ BadBytecodeTestPEP451):
+ pass
+
+Frozen_SourcelessBadBytecodePEP451, Source_SourcelessBadBytecodePEP451 = util.test_both(
+ SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib,
+ machinery=machinery, abc=importlib_abc, util=importlib_util)
+
+class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest,
+ BadBytecodeTestPEP302):
+ pass
-def test_main():
- from test.support import run_unittest
- run_unittest(SimpleTest,
- SourceLoaderBadBytecodeTest,
- SourcelessLoaderBadBytecodeTest
- )
+Frozen_SourcelessBadBytecodePEP302, Source_SourcelessBadBytecodePEP302 = util.test_both(
+ SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib,
+ machinery=machinery, abc=importlib_abc, util=importlib_util)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py
index 8e49868..473297b 100644
--- a/Lib/test/test_importlib/source/test_finder.py
+++ b/Lib/test/test_importlib/source/test_finder.py
@@ -1,9 +1,10 @@
from .. import abc
+from .. import util
from . import util as source_util
-from importlib import machinery
+machinery = util.import_importlib('importlib.machinery')
+
import errno
-import imp
import os
import py_compile
import stat
@@ -39,14 +40,15 @@ class FinderTests(abc.FinderTests):
"""
def get_finder(self, root):
- loader_details = [(machinery.SourceFileLoader,
- machinery.SOURCE_SUFFIXES),
- (machinery.SourcelessFileLoader,
- machinery.BYTECODE_SUFFIXES)]
- return machinery.FileFinder(root, *loader_details)
+ loader_details = [(self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES),
+ (self.machinery.SourcelessFileLoader,
+ self.machinery.BYTECODE_SUFFIXES)]
+ return self.machinery.FileFinder(root, *loader_details)
def import_(self, root, module):
- return self.get_finder(root).find_module(module)
+ finder = self.get_finder(root)
+ return self._find(finder, module, loader_only=True)
def run_test(self, test, create=None, *, compile_=None, unlink=None):
"""Test the finding of 'test' with the creation of modules listed in
@@ -124,20 +126,20 @@ class FinderTests(abc.FinderTests):
def test_empty_string_for_dir(self):
# The empty string from sys.path means to search in the cwd.
- finder = machinery.FileFinder('', (machinery.SourceFileLoader,
- machinery.SOURCE_SUFFIXES))
+ finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
with open('mod.py', 'w') as file:
file.write("# test file for importlib")
try:
- loader = finder.find_module('mod')
+ loader = self._find(finder, 'mod', loader_only=True)
self.assertTrue(hasattr(loader, 'load_module'))
finally:
os.unlink('mod.py')
def test_invalidate_caches(self):
# invalidate_caches() should reset the mtime.
- finder = machinery.FileFinder('', (machinery.SourceFileLoader,
- machinery.SOURCE_SUFFIXES))
+ finder = self.machinery.FileFinder('', (self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
finder._path_mtime = 42
finder.invalidate_caches()
self.assertEqual(finder._path_mtime, -1)
@@ -147,8 +149,10 @@ class FinderTests(abc.FinderTests):
mod = 'mod'
with source_util.create_modules(mod) as mapping:
finder = self.get_finder(mapping['.root'])
- self.assertIsNotNone(finder.find_module(mod))
- self.assertIsNone(finder.find_module(mod))
+ found = self._find(finder, 'mod', loader_only=True)
+ self.assertIsNotNone(found)
+ found = self._find(finder, 'mod', loader_only=True)
+ self.assertIsNone(found)
@unittest.skipUnless(sys.platform != 'win32',
'os.chmod() does not support the needed arguments under Windows')
@@ -172,20 +176,57 @@ class FinderTests(abc.FinderTests):
self.addCleanup(cleanup, tempdir)
os.chmod(tempdir.name, stat.S_IWUSR | stat.S_IXUSR)
finder = self.get_finder(tempdir.name)
- self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+ found = self._find(finder, 'doesnotexist')
+ self.assertEqual(found, self.NOT_FOUND)
def test_ignore_file(self):
# If a directory got changed to a file from underneath us, then don't
# worry about looking for submodules.
with tempfile.NamedTemporaryFile() as file_obj:
finder = self.get_finder(file_obj.name)
- self.assertEqual((None, []), finder.find_loader('doesnotexist'))
+ found = self._find(finder, 'doesnotexist')
+ self.assertEqual(found, self.NOT_FOUND)
+
+
+class FinderTestsPEP451(FinderTests):
+
+ NOT_FOUND = None
+
+ def _find(self, finder, name, loader_only=False):
+ spec = finder.find_spec(name)
+ return spec.loader if spec is not None else spec
+
+Frozen_FinderTestsPEP451, Source_FinderTestsPEP451 = util.test_both(
+ FinderTestsPEP451, machinery=machinery)
+
+
+class FinderTestsPEP420(FinderTests):
+
+ NOT_FOUND = (None, [])
+
+ def _find(self, finder, name, loader_only=False):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ loader_portions = finder.find_loader(name)
+ return loader_portions[0] if loader_only else loader_portions
+
+Frozen_FinderTestsPEP420, Source_FinderTestsPEP420 = util.test_both(
+ FinderTestsPEP420, machinery=machinery)
+
+
+class FinderTestsPEP302(FinderTests):
+
+ NOT_FOUND = None
+
+ def _find(self, finder, name, loader_only=False):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+ return finder.find_module(name)
+Frozen_FinderTestsPEP302, Source_FinderTestsPEP302 = util.test_both(
+ FinderTestsPEP302, machinery=machinery)
-def test_main():
- from test.support import run_unittest
- run_unittest(FinderTests)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py
index 6a78792..92da772 100644
--- a/Lib/test/test_importlib/source/test_path_hook.py
+++ b/Lib/test/test_importlib/source/test_path_hook.py
@@ -1,17 +1,18 @@
+from .. import util
from . import util as source_util
-from importlib import machinery
-import imp
+machinery = util.import_importlib('importlib.machinery')
+
import unittest
-class PathHookTest(unittest.TestCase):
+class PathHookTest:
"""Test the path hook for source."""
def path_hook(self):
- return machinery.FileFinder.path_hook((machinery.SourceFileLoader,
- machinery.SOURCE_SUFFIXES))
+ return self.machinery.FileFinder.path_hook((self.machinery.SourceFileLoader,
+ self.machinery.SOURCE_SUFFIXES))
def test_success(self):
with source_util.create_modules('dummy') as mapping:
@@ -22,11 +23,8 @@ class PathHookTest(unittest.TestCase):
# The empty string represents the cwd.
self.assertTrue(hasattr(self.path_hook()(''), 'find_module'))
-
-def test_main():
- from test.support import run_unittest
- run_unittest(PathHookTest)
+Frozen_PathHookTest, Source_PathHooktest = util.test_both(PathHookTest, machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/test_source_encoding.py b/Lib/test/test_importlib/source/test_source_encoding.py
index ba02b44..c62dfa1 100644
--- a/Lib/test/test_importlib/source/test_source_encoding.py
+++ b/Lib/test/test_importlib/source/test_source_encoding.py
@@ -1,19 +1,24 @@
+from .. import util
from . import util as source_util
-from importlib import _bootstrap
+machinery = util.import_importlib('importlib.machinery')
+
import codecs
+import importlib.util
import re
import sys
+import types
# Because sys.path gets essentially blanked, need to have unicodedata already
# imported for the parser to use.
import unicodedata
import unittest
+import warnings
CODING_RE = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)', re.ASCII)
-class EncodingTest(unittest.TestCase):
+class EncodingTest:
"""PEP 3120 makes UTF-8 the default encoding for source code
[default encoding].
@@ -35,9 +40,9 @@ class EncodingTest(unittest.TestCase):
with source_util.create_modules(self.module_name) as mapping:
with open(mapping[self.module_name], 'wb') as file:
file.write(source)
- loader = _bootstrap.SourceFileLoader(self.module_name,
+ loader = self.machinery.SourceFileLoader(self.module_name,
mapping[self.module_name])
- return loader.load_module(self.module_name)
+ return self.load(loader)
def create_source(self, encoding):
encoding_line = "# coding={0}".format(encoding)
@@ -84,8 +89,29 @@ class EncodingTest(unittest.TestCase):
with self.assertRaises(SyntaxError):
self.run_test(source)
+class EncodingTestPEP451(EncodingTest):
+
+ def load(self, loader):
+ module = types.ModuleType(self.module_name)
+ module.__spec__ = importlib.util.spec_from_loader(self.module_name, loader)
+ loader.exec_module(module)
+ return module
+
+Frozen_EncodingTestPEP451, Source_EncodingTestPEP451 = util.test_both(
+ EncodingTestPEP451, machinery=machinery)
+
+class EncodingTestPEP302(EncodingTest):
+
+ def load(self, loader):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return loader.load_module(self.module_name)
+
+Frozen_EncodingTestPEP302, Source_EncodingTestPEP302 = util.test_both(
+ EncodingTestPEP302, machinery=machinery)
-class LineEndingTest(unittest.TestCase):
+
+class LineEndingTest:
r"""Source written with the three types of line endings (\n, \r\n, \r)
need to be readable [cr][crlf][lf]."""
@@ -97,9 +123,9 @@ class LineEndingTest(unittest.TestCase):
with source_util.create_modules(module_name) as mapping:
with open(mapping[module_name], 'wb') as file:
file.write(source)
- loader = _bootstrap.SourceFileLoader(module_name,
- mapping[module_name])
- return loader.load_module(module_name)
+ loader = self.machinery.SourceFileLoader(module_name,
+ mapping[module_name])
+ return self.load(loader, module_name)
# [cr]
def test_cr(self):
@@ -113,11 +139,27 @@ class LineEndingTest(unittest.TestCase):
def test_lf(self):
self.run_test(b'\n')
+class LineEndingTestPEP451(LineEndingTest):
+
+ def load(self, loader, module_name):
+ module = types.ModuleType(module_name)
+ module.__spec__ = importlib.util.spec_from_loader(module_name, loader)
+ loader.exec_module(module)
+ return module
+
+Frozen_LineEndingTestPEP451, Source_LineEndingTestPEP451 = util.test_both(
+ LineEndingTestPEP451, machinery=machinery)
+
+class LineEndingTestPEP302(LineEndingTest):
+
+ def load(self, loader, module_name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return loader.load_module(module_name)
-def test_main():
- from test.support import run_unittest
- run_unittest(EncodingTest, LineEndingTest)
+Frozen_LineEndingTestPEP302, Source_LineEndingTestPEP302 = util.test_both(
+ LineEndingTestPEP302, machinery=machinery)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/source/util.py b/Lib/test/test_importlib/source/util.py
index ae65663..63cd25a 100644
--- a/Lib/test/test_importlib/source/util.py
+++ b/Lib/test/test_importlib/source/util.py
@@ -2,7 +2,6 @@ from .. import util
import contextlib
import errno
import functools
-import imp
import os
import os.path
import sys
diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py
index c620c37..09b7294 100644
--- a/Lib/test/test_importlib/test_abc.py
+++ b/Lib/test/test_importlib/test_abc.py
@@ -1,9 +1,23 @@
-from importlib import abc
-from importlib import machinery
+import contextlib
import inspect
+import io
+import marshal
+import os
+import sys
+from test import support
+import types
import unittest
+from unittest import mock
+import warnings
+from . import util
+frozen_init, source_init = util.import_importlib('importlib')
+frozen_abc, source_abc = util.import_importlib('importlib.abc')
+machinery = util.import_importlib('importlib.machinery')
+frozen_util, source_util = util.import_importlib('importlib.util')
+
+##### Inheritance ##############################################################
class InheritanceTests:
"""Test that the specified class is a subclass/superclass of the expected
@@ -14,8 +28,19 @@ class InheritanceTests:
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
+ self.superclasses = [getattr(self.abc, class_name)
+ for class_name in self.superclass_names]
+ if hasattr(self, 'subclass_names'):
+ # Because test.support.import_fresh_module() creates a new
+ # importlib._bootstrap per module, inheritance checks fail when
+ # checking across module boundaries (i.e. the _bootstrap in abc is
+ # not the same as the one in machinery). That means stealing one of
+ # the modules from the other to make sure the same instance is used.
+ self.subclasses = [getattr(self.abc.machinery, class_name)
+ for class_name in self.subclass_names]
assert self.subclasses or self.superclasses, self.__class__
- self.__test = getattr(abc, self.__class__.__name__)
+ testing = self.__class__.__name__.partition('_')[2]
+ self.__test = getattr(self.abc, testing)
def test_subclasses(self):
# Test that the expected subclasses inherit.
@@ -29,75 +54,898 @@ class InheritanceTests:
self.assertTrue(issubclass(self.__test, superclass),
"{0} is not a superclass of {1}".format(superclass, self.__test))
+def create_inheritance_tests(base_class):
+ def set_frozen(ns):
+ ns['abc'] = frozen_abc
+ def set_source(ns):
+ ns['abc'] = source_abc
-class MetaPathFinder(InheritanceTests, unittest.TestCase):
+ classes = []
+ for prefix, ns_set in [('Frozen', set_frozen), ('Source', set_source)]:
+ classes.append(types.new_class('_'.join([prefix, base_class.__name__]),
+ (base_class, unittest.TestCase),
+ exec_body=ns_set))
+ return classes
- superclasses = [abc.Finder]
- subclasses = [machinery.BuiltinImporter, machinery.FrozenImporter,
- machinery.PathFinder, machinery.WindowsRegistryFinder]
+class MetaPathFinder(InheritanceTests):
+ superclass_names = ['Finder']
+ subclass_names = ['BuiltinImporter', 'FrozenImporter', 'PathFinder',
+ 'WindowsRegistryFinder']
-class PathEntryFinder(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(MetaPathFinder)
+Frozen_MetaPathFinderInheritanceTests, Source_MetaPathFinderInheritanceTests = tests
- superclasses = [abc.Finder]
- subclasses = [machinery.FileFinder]
+class PathEntryFinder(InheritanceTests):
+ superclass_names = ['Finder']
+ subclass_names = ['FileFinder']
-class Loader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(PathEntryFinder)
+Frozen_PathEntryFinderInheritanceTests, Source_PathEntryFinderInheritanceTests = tests
- subclasses = [abc.PyLoader]
+class ResourceLoader(InheritanceTests):
+ superclass_names = ['Loader']
-class ResourceLoader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(ResourceLoader)
+Frozen_ResourceLoaderInheritanceTests, Source_ResourceLoaderInheritanceTests = tests
- superclasses = [abc.Loader]
+class InspectLoader(InheritanceTests):
+ superclass_names = ['Loader']
+ subclass_names = ['BuiltinImporter', 'FrozenImporter', 'ExtensionFileLoader']
-class InspectLoader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(InspectLoader)
+Frozen_InspectLoaderInheritanceTests, Source_InspectLoaderInheritanceTests = tests
- superclasses = [abc.Loader]
- subclasses = [abc.PyLoader, machinery.BuiltinImporter,
- machinery.FrozenImporter, machinery.ExtensionFileLoader]
+class ExecutionLoader(InheritanceTests):
+ superclass_names = ['InspectLoader']
+ subclass_names = ['ExtensionFileLoader']
-class ExecutionLoader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(ExecutionLoader)
+Frozen_ExecutionLoaderInheritanceTests, Source_ExecutionLoaderInheritanceTests = tests
- superclasses = [abc.InspectLoader]
- subclasses = [abc.PyLoader]
+class FileLoader(InheritanceTests):
+ superclass_names = ['ResourceLoader', 'ExecutionLoader']
+ subclass_names = ['SourceFileLoader', 'SourcelessFileLoader']
-class FileLoader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(FileLoader)
+Frozen_FileLoaderInheritanceTests, Source_FileLoaderInheritanceTests = tests
- superclasses = [abc.ResourceLoader, abc.ExecutionLoader]
- subclasses = [machinery.SourceFileLoader, machinery.SourcelessFileLoader]
+class SourceLoader(InheritanceTests):
+ superclass_names = ['ResourceLoader', 'ExecutionLoader']
+ subclass_names = ['SourceFileLoader']
-class SourceLoader(InheritanceTests, unittest.TestCase):
+tests = create_inheritance_tests(SourceLoader)
+Frozen_SourceLoaderInheritanceTests, Source_SourceLoaderInheritanceTests = tests
- superclasses = [abc.ResourceLoader, abc.ExecutionLoader]
- subclasses = [machinery.SourceFileLoader]
+##### Default return values ####################################################
+def make_abc_subclasses(base_class):
+ classes = []
+ for kind, abc in [('Frozen', frozen_abc), ('Source', source_abc)]:
+ name = '_'.join([kind, base_class.__name__])
+ base_classes = base_class, getattr(abc, base_class.__name__)
+ classes.append(types.new_class(name, base_classes))
+ return classes
+def make_return_value_tests(base_class, test_class):
+ frozen_class, source_class = make_abc_subclasses(base_class)
+ tests = []
+ for prefix, class_in_test in [('Frozen', frozen_class), ('Source', source_class)]:
+ def set_ns(ns):
+ ns['ins'] = class_in_test()
+ tests.append(types.new_class('_'.join([prefix, test_class.__name__]),
+ (test_class, unittest.TestCase),
+ exec_body=set_ns))
+ return tests
-class PyLoader(InheritanceTests, unittest.TestCase):
- superclasses = [abc.Loader, abc.ResourceLoader, abc.ExecutionLoader]
+class MetaPathFinder:
+ def find_module(self, fullname, path):
+ return super().find_module(fullname, path)
-class PyPycLoader(InheritanceTests, unittest.TestCase):
+Frozen_MPF, Source_MPF = make_abc_subclasses(MetaPathFinder)
- superclasses = [abc.PyLoader]
+class MetaPathFinderDefaultsTests:
-def test_main():
- from test.support import run_unittest
- classes = []
- for class_ in globals().values():
- if (inspect.isclass(class_) and
- issubclass(class_, unittest.TestCase) and
- issubclass(class_, InheritanceTests)):
- classes.append(class_)
- run_unittest(*classes)
+ def test_find_module(self):
+ # Default should return None.
+ self.assertIsNone(self.ins.find_module('something', None))
+
+ def test_invalidate_caches(self):
+ # Calling the method is a no-op.
+ self.ins.invalidate_caches()
+
+
+tests = make_return_value_tests(MetaPathFinder, MetaPathFinderDefaultsTests)
+Frozen_MPFDefaultTests, Source_MPFDefaultTests = tests
+
+
+class PathEntryFinder:
+
+ def find_loader(self, fullname):
+ return super().find_loader(fullname)
+
+Frozen_PEF, Source_PEF = make_abc_subclasses(PathEntryFinder)
+
+
+class PathEntryFinderDefaultsTests:
+
+ def test_find_loader(self):
+ self.assertEqual((None, []), self.ins.find_loader('something'))
+
+ def find_module(self):
+ self.assertEqual(None, self.ins.find_module('something'))
+
+ def test_invalidate_caches(self):
+ # Should be a no-op.
+ self.ins.invalidate_caches()
+
+
+tests = make_return_value_tests(PathEntryFinder, PathEntryFinderDefaultsTests)
+Frozen_PEFDefaultTests, Source_PEFDefaultTests = tests
+
+
+class Loader:
+
+ def load_module(self, fullname):
+ return super().load_module(fullname)
+
+
+Frozen_L, Source_L = make_abc_subclasses(Loader)
+
+
+class LoaderDefaultsTests:
+
+ def test_load_module(self):
+ with self.assertRaises(ImportError):
+ self.ins.load_module('something')
+
+ def test_module_repr(self):
+ mod = types.ModuleType('blah')
+ with self.assertRaises(NotImplementedError):
+ self.ins.module_repr(mod)
+ original_repr = repr(mod)
+ mod.__loader__ = self.ins
+ # Should still return a proper repr.
+ self.assertTrue(repr(mod))
+
+
+tests = make_return_value_tests(Loader, LoaderDefaultsTests)
+Frozen_LDefaultTests, SourceLDefaultTests = tests
+
+
+class ResourceLoader(Loader):
+
+ def get_data(self, path):
+ return super().get_data(path)
+
+
+Frozen_RL, Source_RL = make_abc_subclasses(ResourceLoader)
+
+
+class ResourceLoaderDefaultsTests:
+
+ def test_get_data(self):
+ with self.assertRaises(IOError):
+ self.ins.get_data('/some/path')
+
+
+tests = make_return_value_tests(ResourceLoader, ResourceLoaderDefaultsTests)
+Frozen_RLDefaultTests, Source_RLDefaultTests = tests
+
+
+class InspectLoader(Loader):
+
+ def is_package(self, fullname):
+ return super().is_package(fullname)
+
+ def get_source(self, fullname):
+ return super().get_source(fullname)
+
+
+Frozen_IL, Source_IL = make_abc_subclasses(InspectLoader)
+
+
+class InspectLoaderDefaultsTests:
+
+ def test_is_package(self):
+ with self.assertRaises(ImportError):
+ self.ins.is_package('blah')
+
+ def test_get_source(self):
+ with self.assertRaises(ImportError):
+ self.ins.get_source('blah')
+
+
+tests = make_return_value_tests(InspectLoader, InspectLoaderDefaultsTests)
+Frozen_ILDefaultTests, Source_ILDefaultTests = tests
+
+
+class ExecutionLoader(InspectLoader):
+
+ def get_filename(self, fullname):
+ return super().get_filename(fullname)
+
+Frozen_EL, Source_EL = make_abc_subclasses(ExecutionLoader)
+
+
+class ExecutionLoaderDefaultsTests:
+
+ def test_get_filename(self):
+ with self.assertRaises(ImportError):
+ self.ins.get_filename('blah')
+
+
+tests = make_return_value_tests(ExecutionLoader, InspectLoaderDefaultsTests)
+Frozen_ELDefaultTests, Source_ELDefaultsTests = tests
+
+##### MetaPathFinder concrete methods ##########################################
+
+class MetaPathFinderFindModuleTests:
+
+ @classmethod
+ def finder(cls, spec):
+ class MetaPathSpecFinder(cls.abc.MetaPathFinder):
+
+ def find_spec(self, fullname, path, target=None):
+ self.called_for = fullname, path
+ return spec
+
+ return MetaPathSpecFinder()
+
+ def test_no_spec(self):
+ finder = self.finder(None)
+ path = ['a', 'b', 'c']
+ name = 'blah'
+ found = finder.find_module(name, path)
+ self.assertIsNone(found)
+ self.assertEqual(name, finder.called_for[0])
+ self.assertEqual(path, finder.called_for[1])
+
+ def test_spec(self):
+ loader = object()
+ spec = self.util.spec_from_loader('blah', loader)
+ finder = self.finder(spec)
+ found = finder.find_module('blah', None)
+ self.assertIs(found, spec.loader)
+
+
+Frozen_MPFFindModuleTests, Source_MPFFindModuleTests = util.test_both(
+ MetaPathFinderFindModuleTests,
+ abc=(frozen_abc, source_abc),
+ util=(frozen_util, source_util))
+
+##### PathEntryFinder concrete methods #########################################
+
+class PathEntryFinderFindLoaderTests:
+
+ @classmethod
+ def finder(cls, spec):
+ class PathEntrySpecFinder(cls.abc.PathEntryFinder):
+
+ def find_spec(self, fullname, target=None):
+ self.called_for = fullname
+ return spec
+
+ return PathEntrySpecFinder()
+
+ def test_no_spec(self):
+ finder = self.finder(None)
+ name = 'blah'
+ found = finder.find_loader(name)
+ self.assertIsNone(found[0])
+ self.assertEqual([], found[1])
+ self.assertEqual(name, finder.called_for)
+
+ def test_spec_with_loader(self):
+ loader = object()
+ spec = self.util.spec_from_loader('blah', loader)
+ finder = self.finder(spec)
+ found = finder.find_loader('blah')
+ self.assertIs(found[0], spec.loader)
+
+ def test_spec_with_portions(self):
+ spec = self.machinery.ModuleSpec('blah', None)
+ paths = ['a', 'b', 'c']
+ spec.submodule_search_locations = paths
+ finder = self.finder(spec)
+ found = finder.find_loader('blah')
+ self.assertIsNone(found[0])
+ self.assertEqual(paths, found[1])
+
+
+Frozen_PEFFindLoaderTests, Source_PEFFindLoaderTests = util.test_both(
+ PathEntryFinderFindLoaderTests,
+ abc=(frozen_abc, source_abc),
+ machinery=machinery,
+ util=(frozen_util, source_util))
+
+
+##### Loader concrete methods ##################################################
+class LoaderLoadModuleTests:
+
+ def loader(self):
+ class SpecLoader(self.abc.Loader):
+ found = None
+ def exec_module(self, module):
+ self.found = module
+
+ def is_package(self, fullname):
+ """Force some non-default module state to be set."""
+ return True
+
+ return SpecLoader()
+
+ def test_fresh(self):
+ loader = self.loader()
+ name = 'blah'
+ with util.uncache(name):
+ loader.load_module(name)
+ module = loader.found
+ self.assertIs(sys.modules[name], module)
+ self.assertEqual(loader, module.__loader__)
+ self.assertEqual(loader, module.__spec__.loader)
+ self.assertEqual(name, module.__name__)
+ self.assertEqual(name, module.__spec__.name)
+ self.assertIsNotNone(module.__path__)
+ self.assertIsNotNone(module.__path__,
+ module.__spec__.submodule_search_locations)
+
+ def test_reload(self):
+ name = 'blah'
+ loader = self.loader()
+ module = types.ModuleType(name)
+ module.__spec__ = self.util.spec_from_loader(name, loader)
+ module.__loader__ = loader
+ with util.uncache(name):
+ sys.modules[name] = module
+ loader.load_module(name)
+ found = loader.found
+ self.assertIs(found, sys.modules[name])
+ self.assertIs(module, sys.modules[name])
+
+
+Frozen_LoaderLoadModuleTests, Source_LoaderLoadModuleTests = util.test_both(
+ LoaderLoadModuleTests,
+ abc=(frozen_abc, source_abc),
+ util=(frozen_util, source_util))
+
+
+##### InspectLoader concrete methods ###########################################
+class InspectLoaderSourceToCodeTests:
+
+ def source_to_module(self, data, path=None):
+ """Help with source_to_code() tests."""
+ module = types.ModuleType('blah')
+ loader = self.InspectLoaderSubclass()
+ if path is None:
+ code = loader.source_to_code(data)
+ else:
+ code = loader.source_to_code(data, path)
+ exec(code, module.__dict__)
+ return module
+
+ def test_source_to_code_source(self):
+ # Since compile() can handle strings, so should source_to_code().
+ source = 'attr = 42'
+ module = self.source_to_module(source)
+ self.assertTrue(hasattr(module, 'attr'))
+ self.assertEqual(module.attr, 42)
+
+ def test_source_to_code_bytes(self):
+ # Since compile() can handle bytes, so should source_to_code().
+ source = b'attr = 42'
+ module = self.source_to_module(source)
+ self.assertTrue(hasattr(module, 'attr'))
+ self.assertEqual(module.attr, 42)
+
+ def test_source_to_code_path(self):
+ # Specifying a path should set it for the code object.
+ path = 'path/to/somewhere'
+ loader = self.InspectLoaderSubclass()
+ code = loader.source_to_code('', path)
+ self.assertEqual(code.co_filename, path)
+
+ def test_source_to_code_no_path(self):
+ # Not setting a path should still work and be set to <string> since that
+ # is a pre-existing practice as a default to compile().
+ loader = self.InspectLoaderSubclass()
+ code = loader.source_to_code('')
+ self.assertEqual(code.co_filename, '<string>')
+
+
+class Frozen_ILSourceToCodeTests(InspectLoaderSourceToCodeTests, unittest.TestCase):
+ InspectLoaderSubclass = Frozen_IL
+
+class Source_ILSourceToCodeTests(InspectLoaderSourceToCodeTests, unittest.TestCase):
+ InspectLoaderSubclass = Source_IL
+
+
+class InspectLoaderGetCodeTests:
+
+ def test_get_code(self):
+ # Test success.
+ module = types.ModuleType('blah')
+ with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked:
+ mocked.return_value = 'attr = 42'
+ loader = self.InspectLoaderSubclass()
+ code = loader.get_code('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+ def test_get_code_source_is_None(self):
+ # If get_source() is None then this should be None.
+ with mock.patch.object(self.InspectLoaderSubclass, 'get_source') as mocked:
+ mocked.return_value = None
+ loader = self.InspectLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertIsNone(code)
+
+ def test_get_code_source_not_found(self):
+ # If there is no source then there is no code object.
+ loader = self.InspectLoaderSubclass()
+ with self.assertRaises(ImportError):
+ loader.get_code('blah')
+
+
+class Frozen_ILGetCodeTests(InspectLoaderGetCodeTests, unittest.TestCase):
+ InspectLoaderSubclass = Frozen_IL
+
+class Source_ILGetCodeTests(InspectLoaderGetCodeTests, unittest.TestCase):
+ InspectLoaderSubclass = Source_IL
+
+
+class InspectLoaderLoadModuleTests:
+
+ """Test InspectLoader.load_module()."""
+
+ module_name = 'blah'
+
+ def setUp(self):
+ support.unload(self.module_name)
+ self.addCleanup(support.unload, self.module_name)
+
+ def mock_get_code(self):
+ return mock.patch.object(self.InspectLoaderSubclass, 'get_code')
+
+ def test_get_code_ImportError(self):
+ # If get_code() raises ImportError, it should propagate.
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.side_effect = ImportError
+ with self.assertRaises(ImportError):
+ loader = self.InspectLoaderSubclass()
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ loader.load_module(self.module_name)
+
+ def test_get_code_None(self):
+ # If get_code() returns None, raise ImportError.
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.return_value = None
+ with self.assertRaises(ImportError):
+ loader = self.InspectLoaderSubclass()
+ loader.load_module(self.module_name)
+
+ def test_module_returned(self):
+ # The loaded module should be returned.
+ code = compile('attr = 42', '<string>', 'exec')
+ with self.mock_get_code() as mocked_get_code:
+ mocked_get_code.return_value = code
+ loader = self.InspectLoaderSubclass()
+ module = loader.load_module(self.module_name)
+ self.assertEqual(module, sys.modules[self.module_name])
+
+
+class Frozen_ILLoadModuleTests(InspectLoaderLoadModuleTests, unittest.TestCase):
+ InspectLoaderSubclass = Frozen_IL
+
+class Source_ILLoadModuleTests(InspectLoaderLoadModuleTests, unittest.TestCase):
+ InspectLoaderSubclass = Source_IL
+
+
+##### ExecutionLoader concrete methods #########################################
+class ExecutionLoaderGetCodeTests:
+
+ def mock_methods(self, *, get_source=False, get_filename=False):
+ source_mock_context, filename_mock_context = None, None
+ if get_source:
+ source_mock_context = mock.patch.object(self.ExecutionLoaderSubclass,
+ 'get_source')
+ if get_filename:
+ filename_mock_context = mock.patch.object(self.ExecutionLoaderSubclass,
+ 'get_filename')
+ return source_mock_context, filename_mock_context
+
+ def test_get_code(self):
+ path = 'blah.py'
+ source_mock_context, filename_mock_context = self.mock_methods(
+ get_source=True, get_filename=True)
+ with source_mock_context as source_mock, filename_mock_context as name_mock:
+ source_mock.return_value = 'attr = 42'
+ name_mock.return_value = path
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertEqual(code.co_filename, path)
+ module = types.ModuleType('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+ def test_get_code_source_is_None(self):
+ # If get_source() is None then this should be None.
+ source_mock_context, _ = self.mock_methods(get_source=True)
+ with source_mock_context as mocked:
+ mocked.return_value = None
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertIsNone(code)
+
+ def test_get_code_source_not_found(self):
+ # If there is no source then there is no code object.
+ loader = self.ExecutionLoaderSubclass()
+ with self.assertRaises(ImportError):
+ loader.get_code('blah')
+
+ def test_get_code_no_path(self):
+ # If get_filename() raises ImportError then simply skip setting the path
+ # on the code object.
+ source_mock_context, filename_mock_context = self.mock_methods(
+ get_source=True, get_filename=True)
+ with source_mock_context as source_mock, filename_mock_context as name_mock:
+ source_mock.return_value = 'attr = 42'
+ name_mock.side_effect = ImportError
+ loader = self.ExecutionLoaderSubclass()
+ code = loader.get_code('blah')
+ self.assertEqual(code.co_filename, '<string>')
+ module = types.ModuleType('blah')
+ exec(code, module.__dict__)
+ self.assertEqual(module.attr, 42)
+
+
+class Frozen_ELGetCodeTests(ExecutionLoaderGetCodeTests, unittest.TestCase):
+ ExecutionLoaderSubclass = Frozen_EL
+
+class Source_ELGetCodeTests(ExecutionLoaderGetCodeTests, unittest.TestCase):
+ ExecutionLoaderSubclass = Source_EL
+
+
+##### SourceLoader concrete methods ############################################
+class SourceLoader:
+
+ # Globals that should be defined for all modules.
+ source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, "
+ b"repr(__loader__)])")
+
+ def __init__(self, path):
+ self.path = path
+
+ def get_data(self, path):
+ if path != self.path:
+ raise IOError
+ return self.source
+
+ def get_filename(self, fullname):
+ return self.path
+
+ def module_repr(self, module):
+ return '<module>'
+
+
+Frozen_SourceOnlyL, Source_SourceOnlyL = make_abc_subclasses(SourceLoader)
+
+
+class SourceLoader(SourceLoader):
+
+ source_mtime = 1
+
+ def __init__(self, path, magic=None):
+ super().__init__(path)
+ self.bytecode_path = self.util.cache_from_source(self.path)
+ self.source_size = len(self.source)
+ if magic is None:
+ magic = self.util.MAGIC_NUMBER
+ data = bytearray(magic)
+ data.extend(self.init._w_long(self.source_mtime))
+ data.extend(self.init._w_long(self.source_size))
+ code_object = compile(self.source, self.path, 'exec',
+ dont_inherit=True)
+ data.extend(marshal.dumps(code_object))
+ self.bytecode = bytes(data)
+ self.written = {}
+
+ def get_data(self, path):
+ if path == self.path:
+ return super().get_data(path)
+ elif path == self.bytecode_path:
+ return self.bytecode
+ else:
+ raise OSError
+
+ def path_stats(self, path):
+ if path != self.path:
+ raise IOError
+ return {'mtime': self.source_mtime, 'size': self.source_size}
+
+ def set_data(self, path, data):
+ self.written[path] = bytes(data)
+ return path == self.bytecode_path
+
+
+Frozen_SL, Source_SL = make_abc_subclasses(SourceLoader)
+Frozen_SL.util = frozen_util
+Source_SL.util = source_util
+Frozen_SL.init = frozen_init
+Source_SL.init = source_init
+
+
+class SourceLoaderTestHarness:
+
+ def setUp(self, *, is_package=True, **kwargs):
+ self.package = 'pkg'
+ if is_package:
+ self.path = os.path.join(self.package, '__init__.py')
+ self.name = self.package
+ else:
+ module_name = 'mod'
+ self.path = os.path.join(self.package, '.'.join(['mod', 'py']))
+ self.name = '.'.join([self.package, module_name])
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = self.loader_mock(self.path, **kwargs)
+
+ def verify_module(self, module):
+ self.assertEqual(module.__name__, self.name)
+ self.assertEqual(module.__file__, self.path)
+ self.assertEqual(module.__cached__, self.cached)
+ self.assertEqual(module.__package__, self.package)
+ self.assertEqual(module.__loader__, self.loader)
+ values = module._.split('::')
+ self.assertEqual(values[0], self.name)
+ self.assertEqual(values[1], self.path)
+ self.assertEqual(values[2], self.cached)
+ self.assertEqual(values[3], self.package)
+ self.assertEqual(values[4], repr(self.loader))
+
+ def verify_code(self, code_object):
+ module = types.ModuleType(self.name)
+ module.__file__ = self.path
+ module.__cached__ = self.cached
+ module.__package__ = self.package
+ module.__loader__ = self.loader
+ module.__path__ = []
+ exec(code_object, module.__dict__)
+ self.verify_module(module)
+
+
+class SourceOnlyLoaderTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader for source-only loading.
+
+ Reload testing is subsumed by the tests for
+ importlib.util.module_for_loader.
+
+ """
+
+ def test_get_source(self):
+ # Verify the source code is returned as a string.
+ # If an OSError is raised by get_data then raise ImportError.
+ expected_source = self.loader.source.decode('utf-8')
+ self.assertEqual(self.loader.get_source(self.name), expected_source)
+ def raise_OSError(path):
+ raise OSError
+ self.loader.get_data = raise_OSError
+ with self.assertRaises(ImportError) as cm:
+ self.loader.get_source(self.name)
+ self.assertEqual(cm.exception.name, self.name)
+
+ def test_is_package(self):
+ # Properly detect when loading a package.
+ self.setUp(is_package=False)
+ self.assertFalse(self.loader.is_package(self.name))
+ self.setUp(is_package=True)
+ self.assertTrue(self.loader.is_package(self.name))
+ self.assertFalse(self.loader.is_package(self.name + '.__init__'))
+
+ def test_get_code(self):
+ # Verify the code object is created.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_source_to_code(self):
+ # Verify the compiled code object.
+ code = self.loader.source_to_code(self.loader.source, self.path)
+ self.verify_code(code)
+
+ def test_load_module(self):
+ # Loading a module should set __name__, __loader__, __package__,
+ # __path__ (for packages), __file__, and __cached__.
+ # The module should also be put into sys.modules.
+ with util.uncache(self.name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertEqual(module.__path__, [os.path.dirname(self.path)])
+ self.assertIn(self.name, sys.modules)
+
+ def test_package_settings(self):
+ # __package__ needs to be set, while __path__ is set on if the module
+ # is a package.
+ # Testing the values for a package are covered by test_load_module.
+ self.setUp(is_package=False)
+ with util.uncache(self.name):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ module = self.loader.load_module(self.name)
+ self.verify_module(module)
+ self.assertTrue(not hasattr(module, '__path__'))
+
+ def test_get_source_encoding(self):
+ # Source is considered encoded in UTF-8 by default unless otherwise
+ # specified by an encoding line.
+ source = "_ = 'ü'"
+ self.loader.source = source.encode('utf-8')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+ source = "# coding: latin-1\n_ = ü"
+ self.loader.source = source.encode('latin-1')
+ returned_source = self.loader.get_source(self.name)
+ self.assertEqual(returned_source, source)
+
+
+class Frozen_SourceOnlyLTests(SourceOnlyLoaderTests, unittest.TestCase):
+ loader_mock = Frozen_SourceOnlyL
+ util = frozen_util
+
+class Source_SourceOnlyLTests(SourceOnlyLoaderTests, unittest.TestCase):
+ loader_mock = Source_SourceOnlyL
+ util = source_util
+
+
+@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true")
+class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
+
+ """Test importlib.abc.SourceLoader's use of bytecode.
+
+ Source-only testing handled by SourceOnlyLoaderTests.
+
+ """
+
+ def verify_code(self, code_object, *, bytecode_written=False):
+ super().verify_code(code_object)
+ if bytecode_written:
+ self.assertIn(self.cached, self.loader.written)
+ data = bytearray(self.util.MAGIC_NUMBER)
+ data.extend(self.init._w_long(self.loader.source_mtime))
+ data.extend(self.init._w_long(self.loader.source_size))
+ data.extend(marshal.dumps(code_object))
+ self.assertEqual(self.loader.written[self.cached], bytes(data))
+
+ def test_code_with_everything(self):
+ # When everything should work.
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+ def test_no_bytecode(self):
+ # If no bytecode exists then move on to the source.
+ self.loader.bytecode_path = "<does not exist>"
+ # Sanity check
+ with self.assertRaises(OSError):
+ bytecode_path = self.util.cache_from_source(self.path)
+ self.loader.get_data(bytecode_path)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_code_bad_timestamp(self):
+ # Bytecode is only used when the timestamp matches the source EXACTLY.
+ for source_mtime in (0, 2):
+ assert source_mtime != self.loader.source_mtime
+ original = self.loader.source_mtime
+ self.loader.source_mtime = source_mtime
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+ self.loader.source_mtime = original
+
+ def test_code_bad_magic(self):
+ # Skip over bytecode with a bad magic number.
+ self.setUp(magic=b'0000')
+ # If bytecode is used then EOFError would be raised by marshal.
+ self.loader.bytecode = self.loader.bytecode[8:]
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object, bytecode_written=True)
+
+ def test_dont_write_bytecode(self):
+ # Bytecode is not written if sys.dont_write_bytecode is true.
+ # Can assume it is false already thanks to the skipIf class decorator.
+ try:
+ sys.dont_write_bytecode = True
+ self.loader.bytecode_path = "<does not exist>"
+ code_object = self.loader.get_code(self.name)
+ self.assertNotIn(self.cached, self.loader.written)
+ finally:
+ sys.dont_write_bytecode = False
+
+ def test_no_set_data(self):
+ # If set_data is not defined, one can still read bytecode.
+ self.setUp(magic=b'0000')
+ original_set_data = self.loader.__class__.mro()[1].set_data
+ try:
+ del self.loader.__class__.mro()[1].set_data
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+ finally:
+ self.loader.__class__.mro()[1].set_data = original_set_data
+
+ def test_set_data_raises_exceptions(self):
+ # Raising NotImplementedError or OSError is okay for set_data.
+ def raise_exception(exc):
+ def closure(*args, **kwargs):
+ raise exc
+ return closure
+
+ self.setUp(magic=b'0000')
+ self.loader.set_data = raise_exception(NotImplementedError)
+ code_object = self.loader.get_code(self.name)
+ self.verify_code(code_object)
+
+
+class Frozen_SLBytecodeTests(SourceLoaderBytecodeTests, unittest.TestCase):
+ loader_mock = Frozen_SL
+ init = frozen_init
+ util = frozen_util
+
+class SourceSLBytecodeTests(SourceLoaderBytecodeTests, unittest.TestCase):
+ loader_mock = Source_SL
+ init = source_init
+ util = source_util
+
+
+class SourceLoaderGetSourceTests:
+
+ """Tests for importlib.abc.SourceLoader.get_source()."""
+
+ def test_default_encoding(self):
+ # Should have no problems with UTF-8 text.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock('mod.file')
+ source = 'x = "ü"'
+ mock.source = source.encode('utf-8')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_decoded_source(self):
+ # Decoding should work.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock("mod.file")
+ source = "# coding: Latin-1\nx='ü'"
+ assert source.encode('latin-1') != source.encode('utf-8')
+ mock.source = source.encode('latin-1')
+ returned_source = mock.get_source(name)
+ self.assertEqual(returned_source, source)
+
+ def test_universal_newlines(self):
+ # PEP 302 says universal newlines should be used.
+ name = 'mod'
+ mock = self.SourceOnlyLoaderMock('mod.file')
+ source = "x = 42\r\ny = -13\r\n"
+ mock.source = source.encode('utf-8')
+ expect = io.IncrementalNewlineDecoder(None, True).decode(source)
+ self.assertEqual(mock.get_source(name), expect)
+
+
+class Frozen_SourceOnlyLGetSourceTests(SourceLoaderGetSourceTests, unittest.TestCase):
+ SourceOnlyLoaderMock = Frozen_SourceOnlyL
+
+class Source_SourceOnlyLGetSourceTests(SourceLoaderGetSourceTests, unittest.TestCase):
+ SourceOnlyLoaderMock = Source_SourceOnlyL
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index b1a5894..744001b 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -1,14 +1,18 @@
from . import util
-import imp
-import importlib
-from importlib import machinery
+
+frozen_init, source_init = util.import_importlib('importlib')
+frozen_util, source_util = util.import_importlib('importlib.util')
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+
+import os.path
import sys
from test import support
import types
import unittest
+import warnings
-class ImportModuleTests(unittest.TestCase):
+class ImportModuleTests:
"""Test importlib.import_module."""
@@ -16,7 +20,7 @@ class ImportModuleTests(unittest.TestCase):
# Test importing a top-level module.
with util.mock_modules('top_level') as mock:
with util.import_state(meta_path=[mock]):
- module = importlib.import_module('top_level')
+ module = self.init.import_module('top_level')
self.assertEqual(module.__name__, 'top_level')
def test_absolute_package_import(self):
@@ -26,7 +30,7 @@ class ImportModuleTests(unittest.TestCase):
name = '{0}.mod'.format(pkg_name)
with util.mock_modules(pkg_long_name, name) as mock:
with util.import_state(meta_path=[mock]):
- module = importlib.import_module(name)
+ module = self.init.import_module(name)
self.assertEqual(module.__name__, name)
def test_shallow_relative_package_import(self):
@@ -38,17 +42,17 @@ class ImportModuleTests(unittest.TestCase):
relative_name = '.{0}'.format(module_name)
with util.mock_modules(pkg_long_name, absolute_name) as mock:
with util.import_state(meta_path=[mock]):
- importlib.import_module(pkg_name)
- module = importlib.import_module(relative_name, pkg_name)
+ self.init.import_module(pkg_name)
+ module = self.init.import_module(relative_name, pkg_name)
self.assertEqual(module.__name__, absolute_name)
def test_deep_relative_package_import(self):
modules = ['a.__init__', 'a.b.__init__', 'a.c']
with util.mock_modules(*modules) as mock:
with util.import_state(meta_path=[mock]):
- importlib.import_module('a')
- importlib.import_module('a.b')
- module = importlib.import_module('..c', 'a.b')
+ self.init.import_module('a')
+ self.init.import_module('a.b')
+ module = self.init.import_module('..c', 'a.b')
self.assertEqual(module.__name__, 'a.c')
def test_absolute_import_with_package(self):
@@ -59,15 +63,15 @@ class ImportModuleTests(unittest.TestCase):
name = '{0}.mod'.format(pkg_name)
with util.mock_modules(pkg_long_name, name) as mock:
with util.import_state(meta_path=[mock]):
- importlib.import_module(pkg_name)
- module = importlib.import_module(name, pkg_name)
+ self.init.import_module(pkg_name)
+ module = self.init.import_module(name, pkg_name)
self.assertEqual(module.__name__, name)
def test_relative_import_wo_package(self):
# Relative imports cannot happen without the 'package' argument being
# set.
with self.assertRaises(TypeError):
- importlib.import_module('.support')
+ self.init.import_module('.support')
def test_loaded_once(self):
@@ -76,7 +80,7 @@ class ImportModuleTests(unittest.TestCase):
# module currently being imported.
b_load_count = 0
def load_a():
- importlib.import_module('a.b')
+ self.init.import_module('a.b')
def load_b():
nonlocal b_load_count
b_load_count += 1
@@ -84,11 +88,17 @@ class ImportModuleTests(unittest.TestCase):
modules = ['a.__init__', 'a.b']
with util.mock_modules(*modules, module_code=code) as mock:
with util.import_state(meta_path=[mock]):
- importlib.import_module('a.b')
+ self.init.import_module('a.b')
self.assertEqual(b_load_count, 1)
+class Frozen_ImportModuleTests(ImportModuleTests, unittest.TestCase):
+ init = frozen_init
+
+class Source_ImportModuleTests(ImportModuleTests, unittest.TestCase):
+ init = source_init
-class FindLoaderTests(unittest.TestCase):
+
+class FindLoaderTests:
class FakeMetaFinder:
@staticmethod
@@ -98,29 +108,51 @@ class FindLoaderTests(unittest.TestCase):
# If a module with __loader__ is in sys.modules, then return it.
name = 'some_mod'
with util.uncache(name):
- module = imp.new_module(name)
+ module = types.ModuleType(name)
loader = 'a loader!'
module.__loader__ = loader
sys.modules[name] = module
- found = importlib.find_loader(name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ found = self.init.find_loader(name)
self.assertEqual(loader, found)
def test_sys_modules_loader_is_None(self):
# If sys.modules[name].__loader__ is None, raise ValueError.
name = 'some_mod'
with util.uncache(name):
- module = imp.new_module(name)
+ module = types.ModuleType(name)
module.__loader__ = None
sys.modules[name] = module
with self.assertRaises(ValueError):
- importlib.find_loader(name)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.init.find_loader(name)
+
+ def test_sys_modules_loader_is_not_set(self):
+ # Should raise ValueError
+ # Issue #17099
+ name = 'some_mod'
+ with util.uncache(name):
+ module = types.ModuleType(name)
+ try:
+ del module.__loader__
+ except AttributeError:
+ pass
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.init.find_loader(name)
def test_success(self):
# Return the loader found on sys.meta_path.
name = 'some_mod'
with util.uncache(name):
with util.import_state(meta_path=[self.FakeMetaFinder]):
- self.assertEqual((name, None), importlib.find_loader(name))
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual((name, None), self.init.find_loader(name))
def test_success_path(self):
# Searching on a path should work.
@@ -128,15 +160,200 @@ class FindLoaderTests(unittest.TestCase):
path = 'path to some place'
with util.uncache(name):
with util.import_state(meta_path=[self.FakeMetaFinder]):
- self.assertEqual((name, path),
- importlib.find_loader(name, path))
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual((name, path),
+ self.init.find_loader(name, path))
def test_nothing(self):
# None is returned upon failure to find a loader.
- self.assertIsNone(importlib.find_loader('nevergoingtofindthismodule'))
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule'))
+
+class Frozen_FindLoaderTests(FindLoaderTests, unittest.TestCase):
+ init = frozen_init
+
+class Source_FindLoaderTests(FindLoaderTests, unittest.TestCase):
+ init = source_init
-class InvalidateCacheTests(unittest.TestCase):
+class ReloadTests:
+
+ """Test module reloading for builtin and extension modules."""
+
+ def test_reload_modules(self):
+ for mod in ('tokenize', 'time', 'marshal'):
+ with self.subTest(module=mod):
+ with support.CleanImport(mod):
+ module = self.init.import_module(mod)
+ self.init.reload(module)
+
+ def test_module_replaced(self):
+ def code():
+ import sys
+ module = type(sys)('top_level')
+ module.spam = 3
+ sys.modules['top_level'] = module
+ mock = util.mock_modules('top_level',
+ module_code={'top_level': code})
+ with mock:
+ with util.import_state(meta_path=[mock]):
+ module = self.init.import_module('top_level')
+ reloaded = self.init.reload(module)
+ actual = sys.modules['top_level']
+ self.assertEqual(actual.spam, 3)
+ self.assertEqual(reloaded.spam, 3)
+
+ def test_reload_missing_loader(self):
+ with support.CleanImport('types'):
+ import types
+ loader = types.__loader__
+ del types.__loader__
+ reloaded = self.init.reload(types)
+
+ self.assertIs(reloaded, types)
+ self.assertIs(sys.modules['types'], types)
+ self.assertEqual(reloaded.__loader__.path, loader.path)
+
+ def test_reload_loader_replaced(self):
+ with support.CleanImport('types'):
+ import types
+ types.__loader__ = None
+ self.init.invalidate_caches()
+ reloaded = self.init.reload(types)
+
+ self.assertIsNot(reloaded.__loader__, None)
+ self.assertIs(reloaded, types)
+ self.assertIs(sys.modules['types'], types)
+
+ def test_reload_location_changed(self):
+ name = 'spam'
+ with support.temp_cwd(None) as cwd:
+ with util.uncache('spam'):
+ with support.DirsOnSysPath(cwd):
+ # Start as a plain module.
+ self.init.invalidate_caches()
+ path = os.path.join(cwd, name + '.py')
+ cached = self.util.cache_from_source(path)
+ expected = {'__name__': name,
+ '__package__': '',
+ '__file__': path,
+ '__cached__': cached,
+ '__doc__': None,
+ '__builtins__': __builtins__,
+ }
+ support.create_empty_file(path)
+ module = self.init.import_module(name)
+ ns = vars(module)
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(loader.path, path)
+ self.assertEqual(ns, expected)
+
+ # Change to a package.
+ self.init.invalidate_caches()
+ init_path = os.path.join(cwd, name, '__init__.py')
+ cached = self.util.cache_from_source(init_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__file__': init_path,
+ '__cached__': cached,
+ '__path__': [os.path.dirname(init_path)],
+ '__doc__': None,
+ '__builtins__': __builtins__,
+ }
+ os.mkdir(name)
+ os.rename(path, init_path)
+ reloaded = self.init.reload(module)
+ ns = vars(reloaded)
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(reloaded, module)
+ self.assertEqual(loader.path, init_path)
+ self.maxDiff = None
+ self.assertEqual(ns, expected)
+
+ def test_reload_namespace_changed(self):
+ name = 'spam'
+ with support.temp_cwd(None) as cwd:
+ with util.uncache('spam'):
+ with support.DirsOnSysPath(cwd):
+ # Start as a namespace package.
+ self.init.invalidate_caches()
+ bad_path = os.path.join(cwd, name, '__init.py')
+ cached = self.util.cache_from_source(bad_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__doc__': None,
+ }
+ os.mkdir(name)
+ with open(bad_path, 'w') as init_file:
+ init_file.write('eggs = None')
+ module = self.init.import_module(name)
+ ns = vars(module)
+ loader = ns.pop('__loader__')
+ path = ns.pop('__path__')
+ spec = ns.pop('__spec__')
+ self.assertEqual(spec.name, name)
+ self.assertIs(spec.loader, None)
+ self.assertIsNot(loader, None)
+ self.assertEqual(set(path),
+ set([os.path.dirname(bad_path)]))
+ with self.assertRaises(AttributeError):
+ # a NamespaceLoader
+ loader.path
+ self.assertEqual(ns, expected)
+
+ # Change to a regular package.
+ self.init.invalidate_caches()
+ init_path = os.path.join(cwd, name, '__init__.py')
+ cached = self.util.cache_from_source(init_path)
+ expected = {'__name__': name,
+ '__package__': name,
+ '__file__': init_path,
+ '__cached__': cached,
+ '__path__': [os.path.dirname(init_path)],
+ '__doc__': None,
+ '__builtins__': __builtins__,
+ 'eggs': None,
+ }
+ os.rename(bad_path, init_path)
+ reloaded = self.init.reload(module)
+ ns = vars(reloaded)
+ loader = ns.pop('__loader__')
+ spec = ns.pop('__spec__')
+ self.assertEqual(spec.name, name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(reloaded, module)
+ self.assertEqual(loader.path, init_path)
+ self.assertEqual(ns, expected)
+
+ def test_reload_submodule(self):
+ # See #19851.
+ name = 'spam'
+ subname = 'ham'
+ with util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = util.submodule(name, subname, pkg_dir)
+ ham = self.init.import_module(fullname)
+ reloaded = self.init.reload(ham)
+ self.assertIs(reloaded, ham)
+
+
+class Frozen_ReloadTests(ReloadTests, unittest.TestCase):
+ init = frozen_init
+ util = frozen_util
+
+class Source_ReloadTests(ReloadTests, unittest.TestCase):
+ init = source_init
+ util = source_util
+
+
+class InvalidateCacheTests:
def test_method_called(self):
# If defined the method should be called.
@@ -155,48 +372,65 @@ class InvalidateCacheTests(unittest.TestCase):
self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
sys.path_importer_cache[key] = path_ins
self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
- importlib.invalidate_caches()
+ self.init.invalidate_caches()
self.assertTrue(meta_ins.called)
self.assertTrue(path_ins.called)
def test_method_lacking(self):
# There should be no issues if the method is not defined.
key = 'gobbledeegook'
- sys.path_importer_cache[key] = imp.NullImporter('abc')
+ sys.path_importer_cache[key] = None
self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
- importlib.invalidate_caches() # Shouldn't trigger an exception.
+ self.init.invalidate_caches() # Shouldn't trigger an exception.
+
+class Frozen_InvalidateCacheTests(InvalidateCacheTests, unittest.TestCase):
+ init = frozen_init
+
+class Source_InvalidateCacheTests(InvalidateCacheTests, unittest.TestCase):
+ init = source_init
class FrozenImportlibTests(unittest.TestCase):
def test_no_frozen_importlib(self):
# Should be able to import w/o _frozen_importlib being defined.
- module = support.import_fresh_module('importlib', blocked=['_frozen_importlib'])
- self.assertFalse(isinstance(module.__loader__,
- machinery.FrozenImporter))
+ # Can't do an isinstance() check since separate copies of importlib
+ # may have been used for import, so just check the name is not for the
+ # frozen loader.
+ self.assertNotEqual(source_init.__loader__.__class__.__name__,
+ 'FrozenImporter')
-class StartupTests(unittest.TestCase):
+class StartupTests:
def test_everyone_has___loader__(self):
# Issue #17098: all modules should have __loader__ defined.
for name, module in sys.modules.items():
if isinstance(module, types.ModuleType):
- if name in sys.builtin_module_names:
- self.assertEqual(importlib.machinery.BuiltinImporter,
- module.__loader__)
- elif imp.is_frozen(name):
- self.assertEqual(importlib.machinery.FrozenImporter,
- module.__loader__)
-
-def test_main():
- from test.support import run_unittest
- run_unittest(ImportModuleTests,
- FindLoaderTests,
- InvalidateCacheTests,
- FrozenImportlibTests,
- StartupTests)
+ with self.subTest(name=name):
+ self.assertTrue(hasattr(module, '__loader__'),
+ '{!r} lacks a __loader__ attribute'.format(name))
+ if self.machinery.BuiltinImporter.find_module(name):
+ self.assertIsNot(module.__loader__, None)
+ elif self.machinery.FrozenImporter.find_module(name):
+ self.assertIsNot(module.__loader__, None)
+
+ def test_everyone_has___spec__(self):
+ for name, module in sys.modules.items():
+ if isinstance(module, types.ModuleType):
+ with self.subTest(name=name):
+ self.assertTrue(hasattr(module, '__spec__'))
+ if self.machinery.BuiltinImporter.find_module(name):
+ self.assertIsNot(module.__spec__, None)
+ elif self.machinery.FrozenImporter.find_module(name):
+ self.assertIsNot(module.__spec__, None)
+
+class Frozen_StartupTests(StartupTests, unittest.TestCase):
+ machinery = frozen_machinery
+
+class Source_StartupTests(StartupTests, unittest.TestCase):
+ machinery = source_machinery
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py
index c373b11..dc97ba1 100644
--- a/Lib/test/test_importlib/test_locks.py
+++ b/Lib/test/test_importlib/test_locks.py
@@ -1,4 +1,8 @@
-from importlib import _bootstrap
+from . import util
+frozen_init, source_init = util.import_importlib('importlib')
+frozen_bootstrap = frozen_init._bootstrap
+source_bootstrap = source_init._bootstrap
+
import sys
import time
import unittest
@@ -13,14 +17,9 @@ except ImportError:
else:
from test import lock_tests
-
-LockType = _bootstrap._ModuleLock
-DeadlockError = _bootstrap._DeadlockError
-
-
if threading is not None:
- class ModuleLockAsRLockTests(lock_tests.RLockTests):
- locktype = staticmethod(lambda: LockType("some_lock"))
+ class ModuleLockAsRLockTests:
+ locktype = classmethod(lambda cls: cls.LockType("some_lock"))
# _is_owned() unsupported
test__is_owned = None
@@ -34,13 +33,21 @@ if threading is not None:
# _release_save() unsupported
test_release_save_unacquired = None
+ class Frozen_ModuleLockAsRLockTests(ModuleLockAsRLockTests, lock_tests.RLockTests):
+ LockType = frozen_bootstrap._ModuleLock
+
+ class Source_ModuleLockAsRLockTests(ModuleLockAsRLockTests, lock_tests.RLockTests):
+ LockType = source_bootstrap._ModuleLock
+
else:
- class ModuleLockAsRLockTests(unittest.TestCase):
+ class Frozen_ModuleLockAsRLockTests(unittest.TestCase):
pass
+ class Source_ModuleLockAsRLockTests(unittest.TestCase):
+ pass
-@unittest.skipUnless(threading, "threads needed for this test")
-class DeadlockAvoidanceTests(unittest.TestCase):
+
+class DeadlockAvoidanceTests:
def setUp(self):
try:
@@ -55,7 +62,7 @@ class DeadlockAvoidanceTests(unittest.TestCase):
def run_deadlock_avoidance_test(self, create_deadlock):
NLOCKS = 10
- locks = [LockType(str(i)) for i in range(NLOCKS)]
+ locks = [self.LockType(str(i)) for i in range(NLOCKS)]
pairs = [(locks[i], locks[(i+1)%NLOCKS]) for i in range(NLOCKS)]
if create_deadlock:
NTHREADS = NLOCKS
@@ -67,7 +74,7 @@ class DeadlockAvoidanceTests(unittest.TestCase):
"""Try to acquire the lock. Return True on success, False on deadlock."""
try:
lock.acquire()
- except DeadlockError:
+ except self.DeadlockError:
return False
else:
return True
@@ -99,30 +106,50 @@ class DeadlockAvoidanceTests(unittest.TestCase):
self.assertEqual(results.count((True, False)), 0)
self.assertEqual(results.count((True, True)), len(results))
+@unittest.skipUnless(threading, "threads needed for this test")
+class Frozen_DeadlockAvoidanceTests(DeadlockAvoidanceTests, unittest.TestCase):
+ LockType = frozen_bootstrap._ModuleLock
+ DeadlockError = frozen_bootstrap._DeadlockError
+
+@unittest.skipUnless(threading, "threads needed for this test")
+class Source_DeadlockAvoidanceTests(DeadlockAvoidanceTests, unittest.TestCase):
+ LockType = source_bootstrap._ModuleLock
+ DeadlockError = source_bootstrap._DeadlockError
-class LifetimeTests(unittest.TestCase):
+
+class LifetimeTests:
def test_lock_lifetime(self):
name = "xyzzy"
- self.assertNotIn(name, _bootstrap._module_locks)
- lock = _bootstrap._get_module_lock(name)
- self.assertIn(name, _bootstrap._module_locks)
+ self.assertNotIn(name, self.bootstrap._module_locks)
+ lock = self.bootstrap._get_module_lock(name)
+ self.assertIn(name, self.bootstrap._module_locks)
wr = weakref.ref(lock)
del lock
support.gc_collect()
- self.assertNotIn(name, _bootstrap._module_locks)
+ self.assertNotIn(name, self.bootstrap._module_locks)
self.assertIsNone(wr())
def test_all_locks(self):
support.gc_collect()
- self.assertEqual(0, len(_bootstrap._module_locks), _bootstrap._module_locks)
+ self.assertEqual(0, len(self.bootstrap._module_locks),
+ self.bootstrap._module_locks)
+
+class Frozen_LifetimeTests(LifetimeTests, unittest.TestCase):
+ bootstrap = frozen_bootstrap
+
+class Source_LifetimeTests(LifetimeTests, unittest.TestCase):
+ bootstrap = source_bootstrap
@support.reap_threads
def test_main():
- support.run_unittest(ModuleLockAsRLockTests,
- DeadlockAvoidanceTests,
- LifetimeTests)
+ support.run_unittest(Frozen_ModuleLockAsRLockTests,
+ Source_ModuleLockAsRLockTests,
+ Frozen_DeadlockAvoidanceTests,
+ Source_DeadlockAvoidanceTests,
+ Frozen_LifetimeTests,
+ Source_LifetimeTests)
if __name__ == '__main__':
diff --git a/Lib/test/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py
index 7067b12..6639612 100644
--- a/Lib/test/test_namespace_pkgs.py
+++ b/Lib/test/test_importlib/test_namespace_pkgs.py
@@ -1,7 +1,10 @@
-import sys
import contextlib
-import unittest
+import importlib.abc
+import importlib.machinery
import os
+import sys
+import types
+import unittest
from test.test_importlib import util
from test.support import run_unittest
@@ -286,9 +289,5 @@ class ModuleAndNamespacePackageInSameDir(NamespacePackageTest):
self.assertEqual(a_test.attr, 'in module')
-def test_main():
- run_unittest(*NamespacePackageTest.__subclasses__())
-
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_spec.py b/Lib/test/test_importlib/test_spec.py
new file mode 100644
index 0000000..71541f6
--- /dev/null
+++ b/Lib/test/test_importlib/test_spec.py
@@ -0,0 +1,957 @@
+from . import util
+
+frozen_init, source_init = util.import_importlib('importlib')
+frozen_bootstrap = frozen_init._bootstrap
+source_bootstrap = source_init._bootstrap
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+frozen_util, source_util = util.import_importlib('importlib.util')
+
+import os.path
+from test.support import CleanImport
+import unittest
+import sys
+import warnings
+
+
+
+class TestLoader:
+
+ def __init__(self, path=None, is_package=None):
+ self.path = path
+ self.package = is_package
+
+ def __repr__(self):
+ return '<TestLoader object>'
+
+ def __getattr__(self, name):
+ if name == 'get_filename' and self.path is not None:
+ return self._get_filename
+ if name == 'is_package':
+ return self._is_package
+ raise AttributeError(name)
+
+ def _get_filename(self, name):
+ return self.path
+
+ def _is_package(self, name):
+ return self.package
+
+
+class NewLoader(TestLoader):
+
+ EGGS = 1
+
+ def exec_module(self, module):
+ module.eggs = self.EGGS
+
+
+class LegacyLoader(TestLoader):
+
+ HAM = -1
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", DeprecationWarning)
+
+ @frozen_util.module_for_loader
+ def load_module(self, module):
+ module.ham = self.HAM
+ return module
+
+
+class ModuleSpecTests:
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+ self.loc_spec._set_fileattr = True
+
+ def test_default(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_no_loader(self):
+ spec = self.machinery.ModuleSpec(self.name, None)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertIs(spec.loader, None)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_is_package_false(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_default_is_package_true(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_has_location_setter(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin='somewhere')
+ self.assertFalse(spec.has_location)
+ spec.has_location = True
+ self.assertTrue(spec.has_location)
+
+ def test_equality(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ origin=None,
+ submodule_search_locations=None,
+ has_location=False,
+ cached=None,
+ )
+
+ self.assertTrue(self.spec == other)
+
+ def test_equality_location(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ origin=self.path,
+ submodule_search_locations=None,
+ has_location=True,
+ cached=self.cached,
+ )
+
+ self.assertEqual(self.loc_spec, other)
+
+ def test_inequality(self):
+ other = type(sys.implementation)(name='ham',
+ loader=self.loader,
+ origin=None,
+ submodule_search_locations=None,
+ has_location=False,
+ cached=None,
+ )
+
+ self.assertNotEqual(self.spec, other)
+
+ def test_inequality_incomplete(self):
+ other = type(sys.implementation)(name=self.name,
+ loader=self.loader,
+ )
+
+ self.assertNotEqual(self.spec, other)
+
+ def test_package(self):
+ spec = self.machinery.ModuleSpec('spam.eggs', self.loader)
+
+ self.assertEqual(spec.parent, 'spam')
+
+ def test_package_is_package(self):
+ spec = self.machinery.ModuleSpec('spam.eggs', self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.parent, 'spam.eggs')
+
+ # cached
+
+ def test_cached_set(self):
+ before = self.spec.cached
+ self.spec.cached = 'there'
+ after = self.spec.cached
+
+ self.assertIs(before, None)
+ self.assertEqual(after, 'there')
+
+ def test_cached_no_origin(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+
+ self.assertIs(spec.cached, None)
+
+ def test_cached_with_origin_not_location(self):
+ spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+
+ self.assertIs(spec.cached, None)
+
+ def test_cached_source(self):
+ expected = self.util.cache_from_source(self.path)
+
+ self.assertEqual(self.loc_spec.cached, expected)
+
+ def test_cached_source_unknown_suffix(self):
+ self.loc_spec.origin = 'spam.spamspamspam'
+
+ self.assertIs(self.loc_spec.cached, None)
+
+ def test_cached_source_missing_cache_tag(self):
+ original = sys.implementation.cache_tag
+ sys.implementation.cache_tag = None
+ try:
+ cached = self.loc_spec.cached
+ finally:
+ sys.implementation.cache_tag = original
+
+ self.assertIs(cached, None)
+
+ def test_cached_sourceless(self):
+ self.loc_spec.origin = 'spam.pyc'
+
+ self.assertEqual(self.loc_spec.cached, 'spam.pyc')
+
+
+class Frozen_ModuleSpecTests(ModuleSpecTests, unittest.TestCase):
+ util = frozen_util
+ machinery = frozen_machinery
+
+
+class Source_ModuleSpecTests(ModuleSpecTests, unittest.TestCase):
+ util = source_util
+ machinery = source_machinery
+
+
+class ModuleSpecMethodsTests:
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.loc_spec = self.machinery.ModuleSpec(self.name, self.loader,
+ origin=self.path)
+ self.loc_spec._set_fileattr = True
+
+ # init_module_attrs
+
+ def test_init_module_attrs(self):
+ module = type(sys)(self.name)
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertEqual(module.__name__, spec.name)
+ self.assertIs(module.__loader__, spec.loader)
+ self.assertEqual(module.__package__, spec.parent)
+ self.assertIs(module.__spec__, spec)
+ self.assertFalse(hasattr(module, '__path__'))
+ self.assertFalse(hasattr(module, '__file__'))
+ self.assertFalse(hasattr(module, '__cached__'))
+
+ def test_init_module_attrs_package(self):
+ module = type(sys)(self.name)
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+ spec.submodule_search_locations = ['spam', 'ham']
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertEqual(module.__name__, spec.name)
+ self.assertIs(module.__loader__, spec.loader)
+ self.assertEqual(module.__package__, spec.parent)
+ self.assertIs(module.__spec__, spec)
+ self.assertIs(module.__path__, spec.submodule_search_locations)
+ self.assertFalse(hasattr(module, '__file__'))
+ self.assertFalse(hasattr(module, '__cached__'))
+
+ def test_init_module_attrs_location(self):
+ module = type(sys)(self.name)
+ spec = self.loc_spec
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertEqual(module.__name__, spec.name)
+ self.assertIs(module.__loader__, spec.loader)
+ self.assertEqual(module.__package__, spec.parent)
+ self.assertIs(module.__spec__, spec)
+ self.assertFalse(hasattr(module, '__path__'))
+ self.assertEqual(module.__file__, spec.origin)
+ self.assertEqual(module.__cached__,
+ self.util.cache_from_source(spec.origin))
+
+ def test_init_module_attrs_different_name(self):
+ module = type(sys)('eggs')
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertEqual(module.__name__, spec.name)
+
+ def test_init_module_attrs_different_spec(self):
+ module = type(sys)(self.name)
+ module.__spec__ = self.machinery.ModuleSpec('eggs', object())
+ spec = self.machinery.ModuleSpec(self.name, self.loader)
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertEqual(module.__name__, spec.name)
+ self.assertIs(module.__loader__, spec.loader)
+ self.assertEqual(module.__package__, spec.parent)
+ self.assertIs(module.__spec__, spec)
+
+ def test_init_module_attrs_already_set(self):
+ module = type(sys)('ham.eggs')
+ module.__loader__ = object()
+ module.__package__ = 'ham'
+ module.__path__ = ['eggs']
+ module.__file__ = 'ham/eggs/__init__.py'
+ module.__cached__ = self.util.cache_from_source(module.__file__)
+ original = vars(module).copy()
+ spec = self.loc_spec
+ spec.submodule_search_locations = ['']
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertIs(module.__loader__, original['__loader__'])
+ self.assertEqual(module.__package__, original['__package__'])
+ self.assertIs(module.__path__, original['__path__'])
+ self.assertEqual(module.__file__, original['__file__'])
+ self.assertEqual(module.__cached__, original['__cached__'])
+
+ def test_init_module_attrs_immutable(self):
+ module = object()
+ spec = self.loc_spec
+ spec.submodule_search_locations = ['']
+ self.bootstrap._SpecMethods(spec).init_module_attrs(module)
+
+ self.assertFalse(hasattr(module, '__name__'))
+ self.assertFalse(hasattr(module, '__loader__'))
+ self.assertFalse(hasattr(module, '__package__'))
+ self.assertFalse(hasattr(module, '__spec__'))
+ self.assertFalse(hasattr(module, '__path__'))
+ self.assertFalse(hasattr(module, '__file__'))
+ self.assertFalse(hasattr(module, '__cached__'))
+
+ # create()
+
+ def test_create(self):
+ created = self.bootstrap._SpecMethods(self.spec).create()
+
+ self.assertEqual(created.__name__, self.spec.name)
+ self.assertIs(created.__loader__, self.spec.loader)
+ self.assertEqual(created.__package__, self.spec.parent)
+ self.assertIs(created.__spec__, self.spec)
+ self.assertFalse(hasattr(created, '__path__'))
+ self.assertFalse(hasattr(created, '__file__'))
+ self.assertFalse(hasattr(created, '__cached__'))
+
+ def test_create_from_loader(self):
+ module = type(sys.implementation)()
+ class CreatingLoader(TestLoader):
+ def create_module(self, spec):
+ return module
+ self.spec.loader = CreatingLoader()
+ created = self.bootstrap._SpecMethods(self.spec).create()
+
+ self.assertIs(created, module)
+ self.assertEqual(created.__name__, self.spec.name)
+ self.assertIs(created.__loader__, self.spec.loader)
+ self.assertEqual(created.__package__, self.spec.parent)
+ self.assertIs(created.__spec__, self.spec)
+ self.assertFalse(hasattr(created, '__path__'))
+ self.assertFalse(hasattr(created, '__file__'))
+ self.assertFalse(hasattr(created, '__cached__'))
+
+ def test_create_from_loader_not_handled(self):
+ class CreatingLoader(TestLoader):
+ def create_module(self, spec):
+ return None
+ self.spec.loader = CreatingLoader()
+ created = self.bootstrap._SpecMethods(self.spec).create()
+
+ self.assertEqual(created.__name__, self.spec.name)
+ self.assertIs(created.__loader__, self.spec.loader)
+ self.assertEqual(created.__package__, self.spec.parent)
+ self.assertIs(created.__spec__, self.spec)
+ self.assertFalse(hasattr(created, '__path__'))
+ self.assertFalse(hasattr(created, '__file__'))
+ self.assertFalse(hasattr(created, '__cached__'))
+
+ # exec()
+
+ def test_exec(self):
+ self.spec.loader = NewLoader()
+ module = self.bootstrap._SpecMethods(self.spec).create()
+ sys.modules[self.name] = module
+ self.assertFalse(hasattr(module, 'eggs'))
+ self.bootstrap._SpecMethods(self.spec).exec(module)
+
+ self.assertEqual(module.eggs, 1)
+
+ # load()
+
+ def test_load(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(loaded, installed)
+
+ def test_load_replaced(self):
+ replacement = object()
+ class ReplacingLoader(TestLoader):
+ def exec_module(self, module):
+ sys.modules[module.__name__] = replacement
+ self.spec.loader = ReplacingLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ installed = sys.modules[self.spec.name]
+
+ self.assertIs(loaded, replacement)
+ self.assertIs(installed, replacement)
+
+ def test_load_failed(self):
+ class FailedLoader(TestLoader):
+ def exec_module(self, module):
+ raise RuntimeError
+ self.spec.loader = FailedLoader()
+ with CleanImport(self.spec.name):
+ with self.assertRaises(RuntimeError):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ self.assertNotIn(self.spec.name, sys.modules)
+
+ def test_load_failed_removed(self):
+ class FailedLoader(TestLoader):
+ def exec_module(self, module):
+ del sys.modules[module.__name__]
+ raise RuntimeError
+ self.spec.loader = FailedLoader()
+ with CleanImport(self.spec.name):
+ with self.assertRaises(RuntimeError):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ self.assertNotIn(self.spec.name, sys.modules)
+
+ def test_load_legacy(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+
+ self.assertEqual(loaded.ham, -1)
+
+ def test_load_legacy_attributes(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+
+ self.assertIs(loaded.__loader__, self.spec.loader)
+ self.assertEqual(loaded.__package__, self.spec.parent)
+ self.assertIs(loaded.__spec__, self.spec)
+
+ def test_load_legacy_attributes_immutable(self):
+ module = object()
+ class ImmutableLoader(TestLoader):
+ def load_module(self, name):
+ sys.modules[name] = module
+ return module
+ self.spec.loader = ImmutableLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+
+ self.assertIs(sys.modules[self.spec.name], module)
+
+ # reload()
+
+ def test_reload(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(reloaded, loaded)
+ self.assertIs(installed, loaded)
+
+ def test_reload_modified(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ loaded.eggs = 2
+ reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
+
+ self.assertEqual(loaded.eggs, 1)
+ self.assertIs(reloaded, loaded)
+
+ def test_reload_extra_attributes(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ loaded.available = False
+ reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
+
+ self.assertFalse(loaded.available)
+ self.assertIs(reloaded, loaded)
+
+ def test_reload_init_module_attrs(self):
+ self.spec.loader = NewLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ loaded.__name__ = 'ham'
+ del loaded.__loader__
+ del loaded.__package__
+ del loaded.__spec__
+ self.bootstrap._SpecMethods(self.spec).exec(loaded)
+
+ self.assertEqual(loaded.__name__, self.spec.name)
+ self.assertIs(loaded.__loader__, self.spec.loader)
+ self.assertEqual(loaded.__package__, self.spec.parent)
+ self.assertIs(loaded.__spec__, self.spec)
+ self.assertFalse(hasattr(loaded, '__path__'))
+ self.assertFalse(hasattr(loaded, '__file__'))
+ self.assertFalse(hasattr(loaded, '__cached__'))
+
+ def test_reload_legacy(self):
+ self.spec.loader = LegacyLoader()
+ with CleanImport(self.spec.name):
+ loaded = self.bootstrap._SpecMethods(self.spec).load()
+ reloaded = self.bootstrap._SpecMethods(self.spec).exec(loaded)
+ installed = sys.modules[self.spec.name]
+
+ self.assertEqual(loaded.ham, -1)
+ self.assertIs(reloaded, loaded)
+ self.assertIs(installed, loaded)
+
+
+class Frozen_ModuleSpecMethodsTests(ModuleSpecMethodsTests, unittest.TestCase):
+ bootstrap = frozen_bootstrap
+ machinery = frozen_machinery
+ util = frozen_util
+
+
+class Source_ModuleSpecMethodsTests(ModuleSpecMethodsTests, unittest.TestCase):
+ bootstrap = source_bootstrap
+ machinery = source_machinery
+ util = source_util
+
+
+class ModuleReprTests:
+
+ def setUp(self):
+ self.module = type(os)('spam')
+ self.spec = self.machinery.ModuleSpec('spam', TestLoader())
+
+ def test_module___loader___module_repr(self):
+ class Loader:
+ def module_repr(self, module):
+ return '<delicious {}>'.format(module.__name__)
+ self.module.__loader__ = Loader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '<delicious spam>')
+
+ def test_module___loader___module_repr_bad(self):
+ class Loader(TestLoader):
+ def module_repr(self, module):
+ raise Exception
+ self.module.__loader__ = Loader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+ def test_module___spec__(self):
+ origin = 'in a hole, in the ground'
+ self.spec.origin = origin
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '<module {!r} ({})>'.format('spam', origin))
+
+ def test_module___spec___location(self):
+ location = 'in_a_galaxy_far_far_away.py'
+ self.spec.origin = location
+ self.spec._set_fileattr = True
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ '<module {!r} from {!r}>'.format('spam', location))
+
+ def test_module___spec___no_origin(self):
+ self.spec.loader = TestLoader()
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+ def test_module___spec___no_origin_no_loader(self):
+ self.spec.loader = None
+ self.module.__spec__ = self.spec
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '<module {!r}>'.format('spam'))
+
+ def test_module_no_name(self):
+ del self.module.__name__
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '<module {!r}>'.format('?'))
+
+ def test_module_with_file(self):
+ filename = 'e/i/e/i/o/spam.py'
+ self.module.__file__ = filename
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ '<module {!r} from {!r}>'.format('spam', filename))
+
+ def test_module_no_file(self):
+ self.module.__loader__ = TestLoader()
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr,
+ '<module {!r} (<TestLoader object>)>'.format('spam'))
+
+ def test_module_no_file_no_loader(self):
+ modrepr = self.bootstrap._module_repr(self.module)
+
+ self.assertEqual(modrepr, '<module {!r}>'.format('spam'))
+
+
+class Frozen_ModuleReprTests(ModuleReprTests, unittest.TestCase):
+ bootstrap = frozen_bootstrap
+ machinery = frozen_machinery
+ util = frozen_util
+
+
+class Source_ModuleReprTests(ModuleReprTests, unittest.TestCase):
+ bootstrap = source_bootstrap
+ machinery = source_machinery
+ util = source_util
+
+
+class FactoryTests:
+
+ def setUp(self):
+ self.name = 'spam'
+ self.path = 'spam.py'
+ self.cached = self.util.cache_from_source(self.path)
+ self.loader = TestLoader()
+ self.fileloader = TestLoader(self.path)
+ self.pkgloader = TestLoader(self.path, True)
+
+ # spec_from_loader()
+
+ def test_spec_from_loader_default(self):
+ spec = self.util.spec_from_loader(self.name, self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_default_with_bad_is_package(self):
+ class Loader:
+ def is_package(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_origin(self):
+ origin = 'somewhere over the rainbow'
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ origin=origin)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, origin)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_false(self):
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_true(self):
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_origin_and_is_package(self):
+ origin = 'where the streets have no name'
+ spec = self.util.spec_from_loader(self.name, self.loader,
+ origin=origin, is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertIs(spec.origin, origin)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_with_loader_false(self):
+ loader = TestLoader(is_package=False)
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_is_package_with_loader_true(self):
+ loader = TestLoader(is_package=True)
+ spec = self.util.spec_from_loader(self.name, loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertIs(spec.origin, None)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [])
+ self.assertIs(spec.cached, None)
+ self.assertFalse(spec.has_location)
+
+ def test_spec_from_loader_default_with_file_loader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_loader_is_package_false_with_fileloader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader,
+ is_package=False)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_loader_is_package_true_with_fileloader(self):
+ spec = self.util.spec_from_loader(self.name, self.fileloader,
+ is_package=True)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ # spec_from_file_location()
+
+ def test_spec_from_file_location_default(self):
+ if self.machinery is source_machinery:
+ raise unittest.SkipTest('not sure why this is breaking...')
+ spec = self.util.spec_from_file_location(self.name, self.path)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertIsInstance(spec.loader,
+ self.machinery.SourceFileLoader)
+ self.assertEqual(spec.loader.name, self.name)
+ self.assertEqual(spec.loader.path, self.path)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_default_without_location(self):
+ spec = self.util.spec_from_file_location(self.name)
+
+ self.assertIs(spec, None)
+
+ def test_spec_from_file_location_default_bad_suffix(self):
+ spec = self.util.spec_from_file_location(self.name, 'spam.eggs')
+
+ self.assertIs(spec, None)
+
+ def test_spec_from_file_location_loader_no_location(self):
+ spec = self.util.spec_from_file_location(self.name,
+ loader=self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_loader_no_location_no_get_filename(self):
+ spec = self.util.spec_from_file_location(self.name,
+ loader=self.loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.loader)
+ self.assertEqual(spec.origin, '<unknown>')
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_loader_no_location_bad_get_filename(self):
+ class Loader:
+ def get_filename(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, '<unknown>')
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertIs(spec.cached, None)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_none(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=None)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_empty(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=[])
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_not_empty(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader,
+ submodule_search_locations=['eggs'])
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, ['eggs'])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.pkgloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.pkgloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertEqual(spec.submodule_search_locations, [''])
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_not_package(self):
+ class Loader:
+ def is_package(self, name):
+ return False
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_no_is_package(self):
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=self.fileloader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, self.fileloader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+ def test_spec_from_file_location_smsl_default_bad_is_package(self):
+ class Loader:
+ def is_package(self, name):
+ raise ImportError
+ loader = Loader()
+ spec = self.util.spec_from_file_location(self.name, self.path,
+ loader=loader)
+
+ self.assertEqual(spec.name, self.name)
+ self.assertEqual(spec.loader, loader)
+ self.assertEqual(spec.origin, self.path)
+ self.assertIs(spec.loader_state, None)
+ self.assertIs(spec.submodule_search_locations, None)
+ self.assertEqual(spec.cached, self.cached)
+ self.assertTrue(spec.has_location)
+
+
+class Frozen_FactoryTests(FactoryTests, unittest.TestCase):
+ util = frozen_util
+ machinery = frozen_machinery
+
+
+class Source_FactoryTests(FactoryTests, unittest.TestCase):
+ util = source_util
+ machinery = source_machinery
diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py
index efc8977..b2823c6 100644
--- a/Lib/test/test_importlib/test_util.py
+++ b/Lib/test/test_importlib/test_util.py
@@ -1,23 +1,66 @@
from importlib import util
from . import util as test_util
-import imp
+frozen_init, source_init = test_util.import_importlib('importlib')
+frozen_machinery, source_machinery = test_util.import_importlib('importlib.machinery')
+frozen_util, source_util = test_util.import_importlib('importlib.util')
+
+import os
import sys
+from test import support
import types
import unittest
+import warnings
+
+
+class DecodeSourceBytesTests:
+
+ source = "string ='ü'"
+
+ def test_ut8_default(self):
+ source_bytes = self.source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes), self.source)
+
+ def test_specified_encoding(self):
+ source = '# coding=latin-1\n' + self.source
+ source_bytes = source.encode('latin-1')
+ assert source_bytes != source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes), source)
+ def test_universal_newlines(self):
+ source = '\r\n'.join([self.source, self.source])
+ source_bytes = source.encode('utf-8')
+ self.assertEqual(self.util.decode_source(source_bytes),
+ '\n'.join([self.source, self.source]))
-class ModuleForLoaderTests(unittest.TestCase):
+Frozen_DecodeSourceBytesTests, Source_DecodeSourceBytesTests = test_util.test_both(
+ DecodeSourceBytesTests, util=[frozen_util, source_util])
+
+
+class ModuleForLoaderTests:
"""Tests for importlib.util.module_for_loader."""
+ @classmethod
+ def module_for_loader(cls, func):
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ return cls.util.module_for_loader(func)
+
+ def test_warning(self):
+ # Should raise a PendingDeprecationWarning when used.
+ with warnings.catch_warnings():
+ warnings.simplefilter('error', DeprecationWarning)
+ with self.assertRaises(DeprecationWarning):
+ func = self.util.module_for_loader(lambda x: x)
+
def return_module(self, name):
- fxn = util.module_for_loader(lambda self, module: module)
+ fxn = self.module_for_loader(lambda self, module: module)
return fxn(self, name)
def raise_exception(self, name):
def to_wrap(self, module):
raise ImportError
- fxn = util.module_for_loader(to_wrap)
+ fxn = self.module_for_loader(to_wrap)
try:
fxn(self, name)
except ImportError:
@@ -35,12 +78,23 @@ class ModuleForLoaderTests(unittest.TestCase):
def test_reload(self):
# Test that a module is reused if already in sys.modules.
+ class FakeLoader:
+ def is_package(self, name):
+ return True
+ @self.module_for_loader
+ def load_module(self, module):
+ return module
name = 'a.b.c'
- module = imp.new_module('a.b.c')
+ module = types.ModuleType('a.b.c')
+ module.__loader__ = 42
+ module.__package__ = 42
with test_util.uncache(name):
sys.modules[name] = module
- returned_module = self.return_module(name)
+ loader = FakeLoader()
+ returned_module = loader.load_module(name)
self.assertIs(returned_module, sys.modules[name])
+ self.assertEqual(module.__loader__, loader)
+ self.assertEqual(module.__package__, name)
def test_new_module_failure(self):
# Test that a module is removed from sys.modules if added but an
@@ -53,7 +107,7 @@ class ModuleForLoaderTests(unittest.TestCase):
def test_reload_failure(self):
# Test that a failure on reload leaves the module in-place.
name = 'a.b.c'
- module = imp.new_module(name)
+ module = types.ModuleType(name)
with test_util.uncache(name):
sys.modules[name] = module
self.raise_exception(name)
@@ -61,7 +115,7 @@ class ModuleForLoaderTests(unittest.TestCase):
def test_decorator_attrs(self):
def fxn(self, module): pass
- wrapped = util.module_for_loader(fxn)
+ wrapped = self.module_for_loader(fxn)
self.assertEqual(wrapped.__name__, fxn.__name__)
self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
@@ -87,7 +141,7 @@ class ModuleForLoaderTests(unittest.TestCase):
self._pkg = is_package
def is_package(self, name):
return self._pkg
- @util.module_for_loader
+ @self.module_for_loader
def load_module(self, module):
return module
@@ -107,8 +161,11 @@ class ModuleForLoaderTests(unittest.TestCase):
self.assertIs(module.__loader__, loader)
self.assertEqual(module.__package__, name)
+Frozen_ModuleForLoaderTests, Source_ModuleForLoaderTests = test_util.test_both(
+ ModuleForLoaderTests, util=[frozen_util, source_util])
+
-class SetPackageTests(unittest.TestCase):
+class SetPackageTests:
"""Tests for importlib.util.set_package."""
@@ -116,34 +173,36 @@ class SetPackageTests(unittest.TestCase):
"""Verify the module has the expected value for __package__ after
passing through set_package."""
fxn = lambda: module
- wrapped = util.set_package(fxn)
- wrapped()
+ wrapped = self.util.set_package(fxn)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ wrapped()
self.assertTrue(hasattr(module, '__package__'))
self.assertEqual(expect, module.__package__)
def test_top_level(self):
# __package__ should be set to the empty string if a top-level module.
# Implicitly tests when package is set to None.
- module = imp.new_module('module')
+ module = types.ModuleType('module')
module.__package__ = None
self.verify(module, '')
def test_package(self):
# Test setting __package__ for a package.
- module = imp.new_module('pkg')
+ module = types.ModuleType('pkg')
module.__path__ = ['<path>']
module.__package__ = None
self.verify(module, 'pkg')
def test_submodule(self):
# Test __package__ for a module in a package.
- module = imp.new_module('pkg.mod')
+ module = types.ModuleType('pkg.mod')
module.__package__ = None
self.verify(module, 'pkg')
def test_setting_if_missing(self):
# __package__ should be set if it is missing.
- module = imp.new_module('mod')
+ module = types.ModuleType('mod')
if hasattr(module, '__package__'):
delattr(module, '__package__')
self.verify(module, '')
@@ -151,58 +210,383 @@ class SetPackageTests(unittest.TestCase):
def test_leaving_alone(self):
# If __package__ is set and not None then leave it alone.
for value in (True, False):
- module = imp.new_module('mod')
+ module = types.ModuleType('mod')
module.__package__ = value
self.verify(module, value)
def test_decorator_attrs(self):
def fxn(module): pass
- wrapped = util.set_package(fxn)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ wrapped = self.util.set_package(fxn)
self.assertEqual(wrapped.__name__, fxn.__name__)
self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
+Frozen_SetPackageTests, Source_SetPackageTests = test_util.test_both(
+ SetPackageTests, util=[frozen_util, source_util])
-class ResolveNameTests(unittest.TestCase):
+
+class SetLoaderTests:
+
+ """Tests importlib.util.set_loader()."""
+
+ class DummyLoader:
+ @util.set_loader
+ def load_module(self, module):
+ return self.module
+
+ def test_no_attribute(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ try:
+ del loader.module.__loader__
+ except AttributeError:
+ pass
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(loader, loader.load_module('blah').__loader__)
+
+ def test_attribute_is_None(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ loader.module.__loader__ = None
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(loader, loader.load_module('blah').__loader__)
+
+ def test_not_reset(self):
+ loader = self.DummyLoader()
+ loader.module = types.ModuleType('blah')
+ loader.module.__loader__ = 42
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', DeprecationWarning)
+ self.assertEqual(42, loader.load_module('blah').__loader__)
+
+class Frozen_SetLoaderTests(SetLoaderTests, unittest.TestCase):
+ class DummyLoader:
+ @frozen_util.set_loader
+ def load_module(self, module):
+ return self.module
+
+class Source_SetLoaderTests(SetLoaderTests, unittest.TestCase):
+ class DummyLoader:
+ @source_util.set_loader
+ def load_module(self, module):
+ return self.module
+
+
+class ResolveNameTests:
"""Tests importlib.util.resolve_name()."""
def test_absolute(self):
# bacon
- self.assertEqual('bacon', util.resolve_name('bacon', None))
+ self.assertEqual('bacon', self.util.resolve_name('bacon', None))
def test_aboslute_within_package(self):
# bacon in spam
- self.assertEqual('bacon', util.resolve_name('bacon', 'spam'))
+ self.assertEqual('bacon', self.util.resolve_name('bacon', 'spam'))
def test_no_package(self):
# .bacon in ''
with self.assertRaises(ValueError):
- util.resolve_name('.bacon', '')
+ self.util.resolve_name('.bacon', '')
def test_in_package(self):
# .bacon in spam
self.assertEqual('spam.eggs.bacon',
- util.resolve_name('.bacon', 'spam.eggs'))
+ self.util.resolve_name('.bacon', 'spam.eggs'))
def test_other_package(self):
# ..bacon in spam.bacon
self.assertEqual('spam.bacon',
- util.resolve_name('..bacon', 'spam.eggs'))
+ self.util.resolve_name('..bacon', 'spam.eggs'))
def test_escape(self):
# ..bacon in spam
with self.assertRaises(ValueError):
- util.resolve_name('..bacon', 'spam')
+ self.util.resolve_name('..bacon', 'spam')
+
+Frozen_ResolveNameTests, Source_ResolveNameTests = test_util.test_both(
+ ResolveNameTests,
+ util=[frozen_util, source_util])
+
+
+class FindSpecTests:
+
+ class FakeMetaFinder:
+ @staticmethod
+ def find_spec(name, path=None, target=None): return name, path, target
+
+ def test_sys_modules(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ loader = 'a loader!'
+ spec = self.machinery.ModuleSpec(name, loader)
+ module.__loader__ = loader
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+
+ def test_sys_modules_without___loader__(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ del module.__loader__
+ loader = 'a loader!'
+ spec = self.machinery.ModuleSpec(name, loader)
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+
+ def test_sys_modules_spec_is_None(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ module.__spec__ = None
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ self.util.find_spec(name)
+
+ def test_sys_modules_loader_is_None(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ spec = self.machinery.ModuleSpec(name, None)
+ module.__spec__ = spec
+ sys.modules[name] = module
+ found = self.util.find_spec(name)
+ self.assertEqual(found, spec)
+ def test_sys_modules_spec_is_not_set(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ module = types.ModuleType(name)
+ try:
+ del module.__spec__
+ except AttributeError:
+ pass
+ sys.modules[name] = module
+ with self.assertRaises(ValueError):
+ self.util.find_spec(name)
-def test_main():
- from test import support
- support.run_unittest(
- ModuleForLoaderTests,
- SetPackageTests,
- ResolveNameTests
- )
+ def test_success(self):
+ name = 'some_mod'
+ with test_util.uncache(name):
+ with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+ self.assertEqual((name, None, None),
+ self.util.find_spec(name))
+
+# def test_success_path(self):
+# # Searching on a path should work.
+# name = 'some_mod'
+# path = 'path to some place'
+# with test_util.uncache(name):
+# with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+# self.assertEqual((name, path, None),
+# self.util.find_spec(name, path))
+
+ def test_nothing(self):
+ # None is returned upon failure to find a loader.
+ self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule'))
+
+ def test_find_submodule(self):
+ name = 'spam'
+ subname = 'ham'
+ with test_util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = test_util.submodule(name, subname, pkg_dir)
+ spec = self.util.find_spec(fullname)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_submodule_parent_already_imported(self):
+ name = 'spam'
+ subname = 'ham'
+ with test_util.temp_module(name, pkg=True) as pkg_dir:
+ self.init.import_module(name)
+ fullname, _ = test_util.submodule(name, subname, pkg_dir)
+ spec = self.util.find_spec(fullname)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_relative_module(self):
+ name = 'spam'
+ subname = 'ham'
+ with test_util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = test_util.submodule(name, subname, pkg_dir)
+ relname = '.' + subname
+ spec = self.util.find_spec(relname, name)
+ self.assertIsNot(spec, None)
+ self.assertIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+ # Ensure successive calls behave the same.
+ spec_again = self.util.find_spec(fullname)
+ self.assertEqual(spec_again, spec)
+
+ def test_find_relative_module_missing_package(self):
+ name = 'spam'
+ subname = 'ham'
+ with test_util.temp_module(name, pkg=True) as pkg_dir:
+ fullname, _ = test_util.submodule(name, subname, pkg_dir)
+ relname = '.' + subname
+ with self.assertRaises(ValueError):
+ self.util.find_spec(relname)
+ self.assertNotIn(name, sorted(sys.modules))
+ self.assertNotIn(fullname, sorted(sys.modules))
+
+
+class Frozen_FindSpecTests(FindSpecTests, unittest.TestCase):
+ init = frozen_init
+ machinery = frozen_machinery
+ util = frozen_util
+
+class Source_FindSpecTests(FindSpecTests, unittest.TestCase):
+ init = source_init
+ machinery = source_machinery
+ util = source_util
+
+
+class MagicNumberTests:
+
+ def test_length(self):
+ # Should be 4 bytes.
+ self.assertEqual(len(self.util.MAGIC_NUMBER), 4)
+
+ def test_incorporates_rn(self):
+ # The magic number uses \r\n to come out wrong when splitting on lines.
+ self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n'))
+
+Frozen_MagicNumberTests, Source_MagicNumberTests = test_util.test_both(
+ MagicNumberTests, util=[frozen_util, source_util])
+
+
+class PEP3147Tests:
+
+ """Tests of PEP 3147-related functions: cache_from_source and source_from_cache."""
+
+ tag = sys.implementation.cache_tag
+
+ @unittest.skipUnless(sys.implementation.cache_tag is not None,
+ 'requires sys.implementation.cache_tag not be None')
+ def test_cache_from_source(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyc file (i.e. under __pycache__).
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, True), expect)
+
+ def test_cache_from_source_no_cache_tag(self):
+ # No cache tag means NotImplementedError.
+ with support.swap_attr(sys.implementation, 'cache_tag', None):
+ with self.assertRaises(NotImplementedError):
+ self.util.cache_from_source('whatever.py')
+
+ def test_cache_from_source_no_dot(self):
+ # Directory with a dot, filename without dot.
+ path = os.path.join('foo.bar', 'file')
+ expect = os.path.join('foo.bar', '__pycache__',
+ 'file{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, True), expect)
+
+ def test_cache_from_source_optimized(self):
+ # Given the path to a .py file, return the path to its PEP 3147
+ # defined .pyo file (i.e. under __pycache__).
+ path = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyo'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, False), expect)
+
+ def test_cache_from_source_cwd(self):
+ path = 'foo.py'
+ expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, True), expect)
+
+ def test_cache_from_source_override(self):
+ # When debug_override is not None, it can be any true-ish or false-ish
+ # value.
+ path = os.path.join('foo', 'bar', 'baz.py')
+ partial_expect = os.path.join('foo', 'bar', '__pycache__',
+ 'baz.{}.py'.format(self.tag))
+ self.assertEqual(self.util.cache_from_source(path, []), partial_expect + 'o')
+ self.assertEqual(self.util.cache_from_source(path, [17]),
+ partial_expect + 'c')
+ # However if the bool-ishness can't be determined, the exception
+ # propagates.
+ class Bearish:
+ def __bool__(self): raise RuntimeError
+ with self.assertRaises(RuntimeError):
+ self.util.cache_from_source('/foo/bar/baz.py', Bearish())
+
+ @unittest.skipUnless(os.sep == '\\' and os.altsep == '/',
+ 'test meaningful only where os.altsep is defined')
+ def test_sep_altsep_and_sep_cache_from_source(self):
+ # Windows path and PEP 3147 where sep is right of altsep.
+ self.assertEqual(
+ self.util.cache_from_source('\\foo\\bar\\baz/qux.py', True),
+ '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
+
+ @unittest.skipUnless(sys.implementation.cache_tag is not None,
+ 'requires sys.implementation.cache_tag to not be '
+ 'None')
+ def test_source_from_cache(self):
+ # Given the path to a PEP 3147 defined .pyc file, return the path to
+ # its source. This tests the good path.
+ path = os.path.join('foo', 'bar', 'baz', '__pycache__',
+ 'qux.{}.pyc'.format(self.tag))
+ expect = os.path.join('foo', 'bar', 'baz', 'qux.py')
+ self.assertEqual(self.util.source_from_cache(path), expect)
+
+ def test_source_from_cache_no_cache_tag(self):
+ # If sys.implementation.cache_tag is None, raise NotImplementedError.
+ path = os.path.join('blah', '__pycache__', 'whatever.pyc')
+ with support.swap_attr(sys.implementation, 'cache_tag', None):
+ with self.assertRaises(NotImplementedError):
+ self.util.source_from_cache(path)
+
+ def test_source_from_cache_bad_path(self):
+ # When the path to a pyc file is not in PEP 3147 format, a ValueError
+ # is raised.
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, '/foo/bar/bazqux.pyc')
+
+ def test_source_from_cache_no_slash(self):
+ # No slashes at all in path -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, 'foo.cpython-32.pyc')
+
+ def test_source_from_cache_too_few_dots(self):
+ # Too few dots in final path component -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache, '__pycache__/foo.pyc')
+
+ def test_source_from_cache_too_many_dots(self):
+ # Too many dots in final path component -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache,
+ '__pycache__/foo.cpython-32.foo.pyc')
+
+ def test_source_from_cache_no__pycache__(self):
+ # Another problem with the path -> ValueError
+ self.assertRaises(
+ ValueError, self.util.source_from_cache,
+ '/foo/bar/foo.cpython-32.foo.pyc')
+
+Frozen_PEP3147Tests, Source_PEP3147Tests = test_util.test_both(
+ PEP3147Tests,
+ util=[frozen_util, source_util])
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py
new file mode 100644
index 0000000..96b4adc
--- /dev/null
+++ b/Lib/test/test_importlib/test_windows.py
@@ -0,0 +1,29 @@
+from . import util
+frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
+
+import sys
+import unittest
+
+
+@unittest.skipUnless(sys.platform.startswith('win'), 'requires Windows')
+class WindowsRegistryFinderTests:
+
+ # XXX Need a test that finds the spec via the registry.
+
+ def test_find_spec_missing(self):
+ spec = self.machinery.WindowsRegistryFinder.find_spec('spam')
+ self.assertIs(spec, None)
+
+ def test_find_module_missing(self):
+ loader = self.machinery.WindowsRegistryFinder.find_module('spam')
+ self.assertIs(loader, None)
+
+
+class Frozen_WindowsRegistryFinderTests(WindowsRegistryFinderTests,
+ unittest.TestCase):
+ machinery = frozen_machinery
+
+
+class Source_WindowsRegistryFinderTests(WindowsRegistryFinderTests,
+ unittest.TestCase):
+ machinery = source_machinery
diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py
index ef32f7d..885cec3 100644
--- a/Lib/test/test_importlib/util.py
+++ b/Lib/test/test_importlib/util.py
@@ -1,9 +1,31 @@
from contextlib import contextmanager
-import imp
+from importlib import util, invalidate_caches
import os.path
from test import support
import unittest
import sys
+import types
+
+
+def import_importlib(module_name):
+ """Import a module from importlib both w/ and w/o _frozen_importlib."""
+ fresh = ('importlib',) if '.' in module_name else ()
+ frozen = support.import_fresh_module(module_name)
+ source = support.import_fresh_module(module_name, fresh=fresh,
+ blocked=('_frozen_importlib',))
+ return frozen, source
+
+
+def test_both(test_class, **kwargs):
+ frozen_tests = types.new_class('Frozen_'+test_class.__name__,
+ (test_class, unittest.TestCase))
+ source_tests = types.new_class('Source_'+test_class.__name__,
+ (test_class, unittest.TestCase))
+ frozen_tests.__module__ = source_tests.__module__ = test_class.__module__
+ for attr, (frozen_value, source_value) in kwargs.items():
+ setattr(frozen_tests, attr, frozen_value)
+ setattr(source_tests, attr, source_value)
+ return frozen_tests, source_tests
CASE_INSENSITIVE_FS = True
@@ -24,6 +46,13 @@ def case_insensitive_tests(test):
"requires a case-insensitive filesystem")(test)
+def submodule(parent, name, pkg_dir, content=''):
+ path = os.path.join(pkg_dir, name + '.py')
+ with open(path, 'w') as subfile:
+ subfile.write(content)
+ return '{}.{}'.format(parent, name), path
+
+
@contextmanager
def uncache(*names):
"""Uncache a module from sys.modules.
@@ -49,6 +78,31 @@ def uncache(*names):
except KeyError:
pass
+
+@contextmanager
+def temp_module(name, content='', *, pkg=False):
+ conflicts = [n for n in sys.modules if n.partition('.')[0] == name]
+ with support.temp_cwd(None) as cwd:
+ with uncache(name, *conflicts):
+ with support.DirsOnSysPath(cwd):
+ invalidate_caches()
+
+ location = os.path.join(cwd, name)
+ if pkg:
+ modpath = os.path.join(location, '__init__.py')
+ os.mkdir(name)
+ else:
+ modpath = location + '.py'
+ if content is None:
+ # Make sure the module file gets created.
+ content = ''
+ if content is not None:
+ # not a namespace package
+ with open(modpath, 'w') as modfile:
+ modfile.write(content)
+ yield location
+
+
@contextmanager
def import_state(**kwargs):
"""Context manager to manage the various importers and stored state in the
@@ -80,9 +134,9 @@ def import_state(**kwargs):
setattr(sys, attr, value)
-class mock_modules:
+class _ImporterMock:
- """A mock importer/loader."""
+ """Base class to help with creating importer mocks."""
def __init__(self, *names, module_code={}):
self.modules = {}
@@ -98,7 +152,7 @@ class mock_modules:
package = name.rsplit('.', 1)[0]
else:
package = import_name
- module = imp.new_module(import_name)
+ module = types.ModuleType(import_name)
module.__loader__ = self
module.__file__ = '<mock __file__>'
module.__package__ = package
@@ -112,6 +166,19 @@ class mock_modules:
def __getitem__(self, name):
return self.modules[name]
+ def __enter__(self):
+ self._uncache = uncache(*self.modules.keys())
+ self._uncache.__enter__()
+ return self
+
+ def __exit__(self, *exc_info):
+ self._uncache.__exit__(None, None, None)
+
+
+class mock_modules(_ImporterMock):
+
+ """Importer mock using PEP 302 APIs."""
+
def find_module(self, fullname, path=None):
if fullname not in self.modules:
return None
@@ -131,10 +198,28 @@ class mock_modules:
raise
return self.modules[fullname]
- def __enter__(self):
- self._uncache = uncache(*self.modules.keys())
- self._uncache.__enter__()
- return self
+class mock_spec(_ImporterMock):
- def __exit__(self, *exc_info):
- self._uncache.__exit__(None, None, None)
+ """Importer mock using PEP 451 APIs."""
+
+ def find_spec(self, fullname, path=None, parent=None):
+ try:
+ module = self.modules[fullname]
+ except KeyError:
+ return None
+ is_package = hasattr(module, '__path__')
+ spec = util.spec_from_file_location(
+ fullname, module.__file__, loader=self,
+ submodule_search_locations=getattr(module, '__path__', None))
+ return spec
+
+ def create_module(self, spec):
+ if spec.name not in self.modules:
+ raise ImportError
+ return self.modules[spec.name]
+
+ def exec_module(self, module):
+ try:
+ self.module_code[module.__spec__.name]()
+ except KeyError:
+ pass
diff --git a/Lib/test/test_index.py b/Lib/test/test_index.py
index 8785711..a2ac321 100644
--- a/Lib/test/test_index.py
+++ b/Lib/test/test_index.py
@@ -81,7 +81,8 @@ class BaseTestCase(unittest.TestCase):
return True
bad_int = BadInt()
- n = operator.index(bad_int)
+ with self.assertWarns(DeprecationWarning):
+ n = operator.index(bad_int)
self.assertEqual(n, 1)
bad_int = BadInt2()
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 5cbec9b..38367f3 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -1,22 +1,32 @@
-import re
-import sys
-import types
-import unittest
+import collections
+import datetime
+import functools
+import importlib
import inspect
+import io
import linecache
-import datetime
-import collections
import os
-import shutil
from os.path import normcase
+import _pickle
+import re
+import shutil
+import sys
+import types
+import unicodedata
+import unittest
+import unittest.mock
-from test.support import run_unittest, TESTFN, DirsOnSysPath
+try:
+ from concurrent.futures import ThreadPoolExecutor
+except ImportError:
+ ThreadPoolExecutor = None
+from test.support import run_unittest, TESTFN, DirsOnSysPath, cpython_only
+from test.support import MISSING_C_DOCSTRINGS
+from test.script_helper import assert_python_ok, assert_python_failure
from test import inspect_fodder as mod
from test import inspect_fodder2 as mod2
-# C module for test_findsource_binary
-import unicodedata
# Functions tested in this suite:
# ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode,
@@ -120,7 +130,6 @@ class TestPredicates(IsTestBase):
def test_get_slot_members(self):
class C(object):
__slots__ = ("a", "b")
-
x = C()
x.a = 42
members = dict(inspect.getmembers(x))
@@ -310,6 +319,16 @@ class TestRetrievingSourceCode(GetSourceBase):
def test_getfile(self):
self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__)
+ def test_getfile_class_without_module(self):
+ class CM(type):
+ @property
+ def __module__(cls):
+ raise AttributeError
+ class C(metaclass=CM):
+ pass
+ with self.assertRaises(TypeError):
+ inspect.getfile(C)
+
def test_getmodule_recursion(self):
from types import ModuleType
name = '__inspect_dummy'
@@ -418,14 +437,14 @@ class TestBuggyCases(GetSourceBase):
unicodedata.__file__[-4:] in (".pyc", ".pyo"),
"unicodedata is not an external binary module")
def test_findsource_binary(self):
- self.assertRaises(IOError, inspect.getsource, unicodedata)
- self.assertRaises(IOError, inspect.findsource, unicodedata)
+ self.assertRaises(OSError, inspect.getsource, unicodedata)
+ self.assertRaises(OSError, inspect.findsource, unicodedata)
def test_findsource_code_in_linecache(self):
lines = ["x=1"]
co = compile(lines[0], "_dynamically_created_file", "exec")
- self.assertRaises(IOError, inspect.findsource, co)
- self.assertRaises(IOError, inspect.getsource, co)
+ self.assertRaises(OSError, inspect.findsource, co)
+ self.assertRaises(OSError, inspect.getsource, co)
linecache.cache[co.co_filename] = (1, None, lines, co.co_filename)
try:
self.assertEqual(inspect.findsource(co), (lines,0))
@@ -463,13 +482,13 @@ class _BrokenDataDescriptor(object):
A broken data descriptor. See bug #1785.
"""
def __get__(*args):
- raise AssertionError("should not __get__ data descriptors")
+ raise AttributeError("broken data descriptor")
def __set__(*args):
raise RuntimeError
def __getattr__(*args):
- raise AssertionError("should not __getattr__ data descriptors")
+ raise AttributeError("broken data descriptor")
class _BrokenMethodDescriptor(object):
@@ -477,10 +496,10 @@ class _BrokenMethodDescriptor(object):
A broken method descriptor. See bug #1785.
"""
def __get__(*args):
- raise AssertionError("should not __get__ method descriptors")
+ raise AttributeError("broken method descriptor")
def __getattr__(*args):
- raise AssertionError("should not __getattr__ method descriptors")
+ raise AttributeError("broken method descriptor")
# Helper for testing classify_class_attrs.
@@ -559,6 +578,96 @@ class TestClassesAndFunctions(unittest.TestCase):
kwonlyargs_e=['arg'],
formatted='(*, arg)')
+ def test_argspec_api_ignores_wrapped(self):
+ # Issue 20684: low level introspection API must ignore __wrapped__
+ @functools.wraps(mod.spam)
+ def ham(x, y):
+ pass
+ # Basic check
+ self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)')
+ self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)')
+ self.assertFullArgSpecEquals(functools.partial(ham),
+ ['x', 'y'], formatted='(x, y)')
+ # Other variants
+ def check_method(f):
+ self.assertArgSpecEquals(f, ['self', 'x', 'y'],
+ formatted='(self, x, y)')
+ class C:
+ @functools.wraps(mod.spam)
+ def ham(self, x, y):
+ pass
+ pham = functools.partialmethod(ham)
+ @functools.wraps(mod.spam)
+ def __call__(self, x, y):
+ pass
+ check_method(C())
+ check_method(C.ham)
+ check_method(C().ham)
+ check_method(C.pham)
+ check_method(C().pham)
+
+ class C_new:
+ @functools.wraps(mod.spam)
+ def __new__(self, x, y):
+ pass
+ check_method(C_new)
+
+ class C_init:
+ @functools.wraps(mod.spam)
+ def __init__(self, x, y):
+ pass
+ check_method(C_init)
+
+ def test_getfullargspec_signature_attr(self):
+ def test():
+ pass
+ spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY)
+ test.__signature__ = inspect.Signature(parameters=(spam_param,))
+
+ self.assertFullArgSpecEquals(test, args_e=['spam'], formatted='(spam)')
+
+ def test_getfullargspec_signature_annos(self):
+ def test(a:'spam') -> 'ham': pass
+ spec = inspect.getfullargspec(test)
+ self.assertEqual(test.__annotations__, spec.annotations)
+
+ def test(): pass
+ spec = inspect.getfullargspec(test)
+ self.assertEqual(test.__annotations__, spec.annotations)
+
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_getfullargspec_builtin_methods(self):
+ self.assertFullArgSpecEquals(_pickle.Pickler.dump,
+ args_e=['self', 'obj'], formatted='(self, obj)')
+
+ self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump,
+ args_e=['self', 'obj'], formatted='(self, obj)')
+
+ self.assertFullArgSpecEquals(
+ os.stat,
+ args_e=['path'],
+ kwonlyargs_e=['dir_fd', 'follow_symlinks'],
+ kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True},
+ formatted='(path, *, dir_fd=None, follow_symlinks=True)')
+
+ @cpython_only
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_getfullagrspec_builtin_func(self):
+ import _testcapi
+ builtin = _testcapi.docstring_with_signature_with_defaults
+ spec = inspect.getfullargspec(builtin)
+ self.assertEqual(spec.defaults[0], 'avocado')
+
+ @cpython_only
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_getfullagrspec_builtin_func_no_signature(self):
+ import _testcapi
+ builtin = _testcapi.docstring_no_signature
+ with self.assertRaises(TypeError):
+ inspect.getfullargspec(builtin)
def test_getargspec_method(self):
class A(object):
@@ -588,6 +697,10 @@ class TestClassesAndFunctions(unittest.TestCase):
md = _BrokenMethodDescriptor()
attrs = attrs_wo_objs(A)
+
+ self.assertIn(('__new__', 'method', object), attrs, 'missing __new__')
+ self.assertIn(('__init__', 'method', object), attrs, 'missing __init__')
+
self.assertIn(('s', 'static method', A), attrs, 'missing static method')
self.assertIn(('c', 'class method', A), attrs, 'missing class method')
self.assertIn(('p', 'property', A), attrs, 'missing property')
@@ -650,6 +763,88 @@ class TestClassesAndFunctions(unittest.TestCase):
if isinstance(builtin, type):
inspect.classify_class_attrs(builtin)
+ def test_classify_DynamicClassAttribute(self):
+ class Meta(type):
+ def __getattr__(self, name):
+ if name == 'ham':
+ return 'spam'
+ return super().__getattr__(name)
+ class VA(metaclass=Meta):
+ @types.DynamicClassAttribute
+ def ham(self):
+ return 'eggs'
+ should_find_dca = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham'])
+ self.assertIn(should_find_dca, inspect.classify_class_attrs(VA))
+ should_find_ga = inspect.Attribute('ham', 'data', Meta, 'spam')
+ self.assertIn(should_find_ga, inspect.classify_class_attrs(VA))
+
+ def test_classify_metaclass_class_attribute(self):
+ class Meta(type):
+ fish = 'slap'
+ def __dir__(self):
+ return ['__class__', '__modules__', '__name__', 'fish']
+ class Class(metaclass=Meta):
+ pass
+ should_find = inspect.Attribute('fish', 'data', Meta, 'slap')
+ self.assertIn(should_find, inspect.classify_class_attrs(Class))
+
+ def test_classify_VirtualAttribute(self):
+ class Meta(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'BOOM']
+ def __getattr__(self, name):
+ if name =='BOOM':
+ return 42
+ return super().__getattr(name)
+ class Class(metaclass=Meta):
+ pass
+ should_find = inspect.Attribute('BOOM', 'data', Meta, 42)
+ self.assertIn(should_find, inspect.classify_class_attrs(Class))
+
+ def test_classify_VirtualAttribute_multi_classes(self):
+ class Meta1(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'one']
+ def __getattr__(self, name):
+ if name =='one':
+ return 1
+ return super().__getattr__(name)
+ class Meta2(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'two']
+ def __getattr__(self, name):
+ if name =='two':
+ return 2
+ return super().__getattr__(name)
+ class Meta3(Meta1, Meta2):
+ def __dir__(cls):
+ return list(sorted(set(['__class__', '__module__', '__name__', 'three'] +
+ Meta1.__dir__(cls) + Meta2.__dir__(cls))))
+ def __getattr__(self, name):
+ if name =='three':
+ return 3
+ return super().__getattr__(name)
+ class Class1(metaclass=Meta1):
+ pass
+ class Class2(Class1, metaclass=Meta3):
+ pass
+
+ should_find1 = inspect.Attribute('one', 'data', Meta1, 1)
+ should_find2 = inspect.Attribute('two', 'data', Meta2, 2)
+ should_find3 = inspect.Attribute('three', 'data', Meta3, 3)
+ cca = inspect.classify_class_attrs(Class2)
+ for sf in (should_find1, should_find2, should_find3):
+ self.assertIn(sf, cca)
+
+ def test_classify_class_attrs_with_buggy_dir(self):
+ class M(type):
+ def __dir__(cls):
+ return ['__class__', '__name__', 'missing']
+ class C(metaclass=M):
+ pass
+ attrs = [a[0] for a in inspect.classify_class_attrs(C)]
+ self.assertNotIn('missing', attrs)
+
def test_getmembers_descriptors(self):
class A(object):
dd = _BrokenDataDescriptor()
@@ -693,6 +888,28 @@ class TestClassesAndFunctions(unittest.TestCase):
self.assertIn(('f', b.f), inspect.getmembers(b))
self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod))
+ def test_getmembers_VirtualAttribute(self):
+ class M(type):
+ def __getattr__(cls, name):
+ if name == 'eggs':
+ return 'scrambled'
+ return super().__getattr__(name)
+ class A(metaclass=M):
+ @types.DynamicClassAttribute
+ def eggs(self):
+ return 'spam'
+ self.assertIn(('eggs', 'scrambled'), inspect.getmembers(A))
+ self.assertIn(('eggs', 'spam'), inspect.getmembers(A()))
+
+ def test_getmembers_with_buggy_dir(self):
+ class M(type):
+ def __dir__(cls):
+ return ['__class__', '__name__', 'missing']
+ class C(metaclass=M):
+ pass
+ attrs = [a[0] for a in inspect.getmembers(C)]
+ self.assertNotIn('missing', attrs)
+
_global_ref = object()
class TestGetClosureVars(unittest.TestCase):
@@ -989,6 +1206,20 @@ class TestGetcallargsFunctions(unittest.TestCase):
self.assertEqualException(f3, '1, 2')
self.assertEqualException(f3, '1, 2, a=1, b=2')
+ # issue #20816: getcallargs() fails to iterate over non-existent
+ # kwonlydefaults and raises a wrong TypeError
+ def f5(*, a): pass
+ with self.assertRaisesRegex(TypeError,
+ 'missing 1 required keyword-only'):
+ inspect.getcallargs(f5)
+
+
+ # issue20817:
+ def f6(a, b, c):
+ pass
+ with self.assertRaisesRegex(TypeError, "'a', 'b' and 'c'"):
+ inspect.getcallargs(f6)
+
class TestGetcallargsMethods(TestGetcallargsFunctions):
def setUp(self):
@@ -1080,6 +1311,15 @@ class TestGetattrStatic(unittest.TestCase):
self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.x)
+ def test_classVirtualAttribute(self):
+ class Thing(object):
+ @types.DynamicClassAttribute
+ def x(self):
+ return self._x
+ _x = object()
+
+ self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.__dict__['x'])
+
def test_inherited_classattribute(self):
class Thing(object):
x = object()
@@ -1390,11 +1630,13 @@ class TestSignatureObject(unittest.TestCase):
self.assertEqual(str(S()), '()')
- def test(po, pk, *args, ko, **kwargs):
+ def test(po, pk, pod=42, pkd=100, *args, ko, **kwargs):
pass
sig = inspect.signature(test)
po = sig.parameters['po'].replace(kind=P.POSITIONAL_ONLY)
+ pod = sig.parameters['pod'].replace(kind=P.POSITIONAL_ONLY)
pk = sig.parameters['pk']
+ pkd = sig.parameters['pkd']
args = sig.parameters['args']
ko = sig.parameters['ko']
kwargs = sig.parameters['kwargs']
@@ -1417,6 +1659,15 @@ class TestSignatureObject(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'duplicate parameter name'):
S((po, pk, args, kwargs2, ko))
+ with self.assertRaisesRegex(ValueError, 'follows default argument'):
+ S((pod, po))
+
+ with self.assertRaisesRegex(ValueError, 'follows default argument'):
+ S((po, pkd, pk))
+
+ with self.assertRaisesRegex(ValueError, 'follows default argument'):
+ S((pkd, pk))
+
def test_signature_immutability(self):
def test(a):
pass
@@ -1461,19 +1712,100 @@ class TestSignatureObject(unittest.TestCase):
('kwargs', ..., int, "var_keyword")),
...))
- def test_signature_on_builtin_function(self):
- with self.assertRaisesRegex(ValueError, 'not supported by signature'):
- inspect.signature(type)
- with self.assertRaisesRegex(ValueError, 'not supported by signature'):
- # support for 'wrapper_descriptor'
- inspect.signature(type.__call__)
- with self.assertRaisesRegex(ValueError, 'not supported by signature'):
- # support for 'method-wrapper'
- inspect.signature(min.__call__)
- with self.assertRaisesRegex(ValueError,
- 'no signature found for builtin function'):
- # support for 'method-wrapper'
- inspect.signature(min)
+ @cpython_only
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_signature_on_builtins(self):
+ import _testcapi
+
+ def test_unbound_method(o):
+ """Use this to test unbound methods (things that should have a self)"""
+ signature = inspect.signature(o)
+ self.assertTrue(isinstance(signature, inspect.Signature))
+ self.assertEqual(list(signature.parameters.values())[0].name, 'self')
+ return signature
+
+ def test_callable(o):
+ """Use this to test bound methods or normal callables (things that don't expect self)"""
+ signature = inspect.signature(o)
+ self.assertTrue(isinstance(signature, inspect.Signature))
+ if signature.parameters:
+ self.assertNotEqual(list(signature.parameters.values())[0].name, 'self')
+ return signature
+
+ signature = test_callable(_testcapi.docstring_with_signature_with_defaults)
+ def p(name): return signature.parameters[name].default
+ self.assertEqual(p('s'), 'avocado')
+ self.assertEqual(p('b'), b'bytes')
+ self.assertEqual(p('d'), 3.14)
+ self.assertEqual(p('i'), 35)
+ self.assertEqual(p('n'), None)
+ self.assertEqual(p('t'), True)
+ self.assertEqual(p('f'), False)
+ self.assertEqual(p('local'), 3)
+ self.assertEqual(p('sys'), sys.maxsize)
+ self.assertEqual(p('exp'), sys.maxsize - 1)
+
+ test_callable(object)
+
+ # normal method
+ # (PyMethodDescr_Type, "method_descriptor")
+ test_unbound_method(_pickle.Pickler.dump)
+ d = _pickle.Pickler(io.StringIO())
+ test_callable(d.dump)
+
+ # static method
+ test_callable(str.maketrans)
+ test_callable('abc'.maketrans)
+
+ # class method
+ test_callable(dict.fromkeys)
+ test_callable({}.fromkeys)
+
+ # wrapper around slot (PyWrapperDescr_Type, "wrapper_descriptor")
+ test_unbound_method(type.__call__)
+ test_unbound_method(int.__add__)
+ test_callable((3).__add__)
+
+ # _PyMethodWrapper_Type
+ # support for 'method-wrapper'
+ test_callable(min.__call__)
+
+ # This doesn't work now.
+ # (We don't have a valid signature for "type" in 3.4)
+ with self.assertRaisesRegex(ValueError, "no signature found"):
+ class ThisWorksNow:
+ __call__ = type
+ test_callable(ThisWorksNow())
+
+ # Regression test for issue #20786
+ test_unbound_method(dict.__delitem__)
+ test_unbound_method(property.__delete__)
+
+
+ @cpython_only
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_signature_on_decorated_builtins(self):
+ import _testcapi
+ func = _testcapi.docstring_with_signature_with_defaults
+
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs) -> int:
+ return func(*args, **kwargs)
+ return wrapper
+
+ decorated_func = decorator(func)
+
+ self.assertEqual(inspect.signature(func),
+ inspect.signature(decorated_func))
+
+ @cpython_only
+ def test_signature_on_builtins_no_signature(self):
+ import _testcapi
+ with self.assertRaisesRegex(ValueError, 'no signature found for builtin'):
+ inspect.signature(_testcapi.docstring_no_signature)
def test_signature_on_non_function(self):
with self.assertRaisesRegex(TypeError, 'is not a callable object'):
@@ -1482,18 +1814,112 @@ class TestSignatureObject(unittest.TestCase):
with self.assertRaisesRegex(TypeError, 'is not a Python function'):
inspect.Signature.from_function(42)
+ def test_signature_from_builtin_errors(self):
+ with self.assertRaisesRegex(TypeError, 'is not a Python builtin'):
+ inspect.Signature.from_builtin(42)
+
+ def test_signature_from_functionlike_object(self):
+ def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs):
+ pass
+
+ class funclike:
+ # Has to be callable, and have correct
+ # __code__, __annotations__, __defaults__, __name__,
+ # and __kwdefaults__ attributes
+
+ def __init__(self, func):
+ self.__name__ = func.__name__
+ self.__code__ = func.__code__
+ self.__annotations__ = func.__annotations__
+ self.__defaults__ = func.__defaults__
+ self.__kwdefaults__ = func.__kwdefaults__
+ self.func = func
+
+ def __call__(self, *args, **kwargs):
+ return self.func(*args, **kwargs)
+
+ sig_func = inspect.Signature.from_function(func)
+
+ sig_funclike = inspect.Signature.from_function(funclike(func))
+ self.assertEqual(sig_funclike, sig_func)
+
+ sig_funclike = inspect.signature(funclike(func))
+ self.assertEqual(sig_funclike, sig_func)
+
+ # If object is not a duck type of function, then
+ # signature will try to get a signature for its '__call__'
+ # method
+ fl = funclike(func)
+ del fl.__defaults__
+ self.assertEqual(self.signature(fl),
+ ((('args', ..., ..., "var_positional"),
+ ('kwargs', ..., ..., "var_keyword")),
+ ...))
+
+ # Test with cython-like builtins:
+ _orig_isdesc = inspect.ismethoddescriptor
+ def _isdesc(obj):
+ if hasattr(obj, '_builtinmock'):
+ return True
+ return _orig_isdesc(obj)
+
+ with unittest.mock.patch('inspect.ismethoddescriptor', _isdesc):
+ builtin_func = funclike(func)
+ # Make sure that our mock setup is working
+ self.assertFalse(inspect.ismethoddescriptor(builtin_func))
+ builtin_func._builtinmock = True
+ self.assertTrue(inspect.ismethoddescriptor(builtin_func))
+ self.assertEqual(inspect.signature(builtin_func), sig_func)
+
+ def test_signature_functionlike_class(self):
+ # We only want to duck type function-like objects,
+ # not classes.
+
+ def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs):
+ pass
+
+ class funclike:
+ def __init__(self, marker):
+ pass
+
+ __name__ = func.__name__
+ __code__ = func.__code__
+ __annotations__ = func.__annotations__
+ __defaults__ = func.__defaults__
+ __kwdefaults__ = func.__kwdefaults__
+
+ with self.assertRaisesRegex(TypeError, 'is not a Python function'):
+ inspect.Signature.from_function(funclike)
+
+ self.assertEqual(str(inspect.signature(funclike)), '(marker)')
+
def test_signature_on_method(self):
class Test:
- def foo(self, arg1, arg2=1) -> int:
+ def __init__(*args):
+ pass
+ def m1(self, arg1, arg2=1) -> int:
+ pass
+ def m2(*args):
+ pass
+ def __call__(*, a):
pass
- meth = Test().foo
-
- self.assertEqual(self.signature(meth),
+ self.assertEqual(self.signature(Test().m1),
((('arg1', ..., ..., "positional_or_keyword"),
('arg2', 1, ..., "positional_or_keyword")),
int))
+ self.assertEqual(self.signature(Test().m2),
+ ((('args', ..., ..., "var_positional"),),
+ ...))
+
+ self.assertEqual(self.signature(Test),
+ ((('args', ..., ..., "var_positional"),),
+ ...))
+
+ with self.assertRaisesRegex(ValueError, 'invalid method signature'):
+ self.signature(Test())
+
def test_signature_on_classmethod(self):
class Test:
@classmethod
@@ -1687,6 +2113,38 @@ class TestSignatureObject(unittest.TestCase):
ba = inspect.signature(_foo).bind(12, 14)
self.assertEqual(_foo(*ba.args, **ba.kwargs), (12, 14, 13))
+ def test_signature_on_partialmethod(self):
+ from functools import partialmethod
+
+ class Spam:
+ def test():
+ pass
+ ham = partialmethod(test)
+
+ with self.assertRaisesRegex(ValueError, "has incorrect arguments"):
+ inspect.signature(Spam.ham)
+
+ class Spam:
+ def test(it, a, *, c) -> 'spam':
+ pass
+ ham = partialmethod(test, c=1)
+
+ self.assertEqual(self.signature(Spam.ham),
+ ((('it', ..., ..., 'positional_or_keyword'),
+ ('a', ..., ..., 'positional_or_keyword'),
+ ('c', 1, ..., 'keyword_only')),
+ 'spam'))
+
+ self.assertEqual(self.signature(Spam().ham),
+ ((('a', ..., ..., 'positional_or_keyword'),
+ ('c', 1, ..., 'keyword_only')),
+ 'spam'))
+
+ def test_signature_on_fake_partialmethod(self):
+ def foo(a): pass
+ foo._partialmethod = 'spam'
+ self.assertEqual(str(inspect.signature(foo)), '(a)')
+
def test_signature_on_decorated(self):
import functools
@@ -1736,6 +2194,17 @@ class TestSignatureObject(unittest.TestCase):
((('b', ..., ..., "positional_or_keyword"),),
...))
+ # Test we handle __signature__ partway down the wrapper stack
+ def wrapped_foo_call():
+ pass
+ wrapped_foo_call.__wrapped__ = Foo.__call__
+
+ self.assertEqual(self.signature(wrapped_foo_call),
+ ((('a', ..., ..., "positional_or_keyword"),
+ ('b', ..., ..., "positional_or_keyword")),
+ ...))
+
+
def test_signature_on_class(self):
class C:
def __init__(self, a):
@@ -1817,6 +2286,49 @@ class TestSignatureObject(unittest.TestCase):
('bar', 2, ..., "keyword_only")),
...))
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_signature_on_class_without_init(self):
+ # Test classes without user-defined __init__ or __new__
+ class C: pass
+ self.assertEqual(str(inspect.signature(C)), '()')
+ class D(C): pass
+ self.assertEqual(str(inspect.signature(D)), '()')
+
+ # Test meta-classes without user-defined __init__ or __new__
+ class C(type): pass
+ class D(C): pass
+ with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
+ self.assertEqual(inspect.signature(C), None)
+ with self.assertRaisesRegex(ValueError, "callable.*is not supported"):
+ self.assertEqual(inspect.signature(D), None)
+
+ @unittest.skipIf(MISSING_C_DOCSTRINGS,
+ "Signature information for builtins requires docstrings")
+ def test_signature_on_builtin_class(self):
+ self.assertEqual(str(inspect.signature(_pickle.Pickler)),
+ '(file, protocol=None, fix_imports=True)')
+
+ class P(_pickle.Pickler): pass
+ class EmptyTrait: pass
+ class P2(EmptyTrait, P): pass
+ self.assertEqual(str(inspect.signature(P)),
+ '(file, protocol=None, fix_imports=True)')
+ self.assertEqual(str(inspect.signature(P2)),
+ '(file, protocol=None, fix_imports=True)')
+
+ class P3(P2):
+ def __init__(self, spam):
+ pass
+ self.assertEqual(str(inspect.signature(P3)), '(spam)')
+
+ class MetaP(type):
+ def __call__(cls, foo, bar):
+ pass
+ class P4(P2, metaclass=MetaP):
+ pass
+ self.assertEqual(str(inspect.signature(P4)), '(foo, bar)')
+
def test_signature_on_callable_objects(self):
class Foo:
def __call__(self, a):
@@ -1838,18 +2350,16 @@ class TestSignatureObject(unittest.TestCase):
((('a', ..., ..., "positional_or_keyword"),),
...))
- class ToFail:
- __call__ = type
- with self.assertRaisesRegex(ValueError, "not supported by signature"):
- inspect.signature(ToFail())
-
-
class Wrapped:
pass
Wrapped.__wrapped__ = lambda a: None
self.assertEqual(self.signature(Wrapped),
((('a', ..., ..., "positional_or_keyword"),),
...))
+ # wrapper loop:
+ Wrapped.__wrapped__ = Wrapped
+ with self.assertRaisesRegex(ValueError, 'wrapper loop'):
+ self.signature(Wrapped)
def test_signature_on_lambdas(self):
self.assertEqual(self.signature((lambda a=10: a)),
@@ -1923,6 +2433,7 @@ class TestSignatureObject(unittest.TestCase):
def test_signature_str_positional_only(self):
P = inspect.Parameter
+ S = inspect.Signature
def test(a_po, *, b, **kwargs):
return a_po, kwargs
@@ -1933,14 +2444,20 @@ class TestSignatureObject(unittest.TestCase):
test.__signature__ = sig.replace(parameters=new_params)
self.assertEqual(str(inspect.signature(test)),
- '(<a_po>, *, b, **kwargs)')
+ '(a_po, /, *, b, **kwargs)')
- sig = inspect.signature(test)
- new_params = list(sig.parameters.values())
- new_params[0] = new_params[0].replace(name=None)
- test.__signature__ = sig.replace(parameters=new_params)
- self.assertEqual(str(inspect.signature(test)),
- '(<0>, *, b, **kwargs)')
+ self.assertEqual(str(S(parameters=[P('foo', P.POSITIONAL_ONLY)])),
+ '(foo, /)')
+
+ self.assertEqual(str(S(parameters=[
+ P('foo', P.POSITIONAL_ONLY),
+ P('bar', P.VAR_KEYWORD)])),
+ '(foo, /, **bar)')
+
+ self.assertEqual(str(S(parameters=[
+ P('foo', P.POSITIONAL_ONLY),
+ P('bar', P.VAR_POSITIONAL)])),
+ '(foo, /, *bar)')
def test_signature_replace_anno(self):
def test() -> 42:
@@ -1955,6 +2472,22 @@ class TestSignatureObject(unittest.TestCase):
self.assertEqual(sig.return_annotation, 42)
self.assertEqual(sig, inspect.signature(test))
+ def test_signature_on_mangled_parameters(self):
+ class Spam:
+ def foo(self, __p1:1=2, *, __p2:2=3):
+ pass
+ class Ham(Spam):
+ pass
+
+ self.assertEqual(self.signature(Spam.foo),
+ ((('self', ..., ..., "positional_or_keyword"),
+ ('_Spam__p1', 2, 1, "positional_or_keyword"),
+ ('_Spam__p2', 3, 2, "keyword_only")),
+ ...))
+
+ self.assertEqual(self.signature(Spam.foo),
+ self.signature(Ham.foo))
+
class TestParameterObject(unittest.TestCase):
def test_signature_parameter_kinds(self):
@@ -1979,10 +2512,13 @@ class TestParameterObject(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'not a valid parameter name'):
inspect.Parameter('1', kind=inspect.Parameter.VAR_KEYWORD)
- with self.assertRaisesRegex(ValueError,
- 'non-positional-only parameter'):
+ with self.assertRaisesRegex(TypeError, 'name must be a str'):
inspect.Parameter(None, kind=inspect.Parameter.VAR_KEYWORD)
+ with self.assertRaisesRegex(ValueError,
+ 'is not a valid parameter name'):
+ inspect.Parameter('$', kind=inspect.Parameter.VAR_KEYWORD)
+
with self.assertRaisesRegex(ValueError, 'cannot have default values'):
inspect.Parameter('a', default=42,
kind=inspect.Parameter.VAR_KEYWORD)
@@ -2031,7 +2567,8 @@ class TestParameterObject(unittest.TestCase):
self.assertEqual(p2.name, 'bar')
self.assertNotEqual(p2, p)
- with self.assertRaisesRegex(ValueError, 'not a valid parameter name'):
+ with self.assertRaisesRegex(ValueError,
+ 'name is a required attribute'):
p2 = p2.replace(name=p2.empty)
p2 = p2.replace(name='foo', default=None)
@@ -2053,14 +2590,11 @@ class TestParameterObject(unittest.TestCase):
self.assertEqual(p2, p)
def test_signature_parameter_positional_only(self):
- p = inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY)
- self.assertEqual(str(p), '<>')
-
- p = p.replace(name='1')
- self.assertEqual(str(p), '<1>')
+ with self.assertRaisesRegex(TypeError, 'name must be a str'):
+ inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY)
def test_signature_parameter_immutability(self):
- p = inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY)
+ p = inspect.Parameter('spam', kind=inspect.Parameter.KEYWORD_ONLY)
with self.assertRaises(AttributeError):
p.foo = 'bar'
@@ -2258,6 +2792,15 @@ class TestSignatureBind(unittest.TestCase):
self.assertEqual(self.call(test, 1, 2, 4, 5, bar=6),
(1, 2, 4, 5, 6, {}))
+ self.assertEqual(self.call(test, 1, 2),
+ (1, 2, 3, 42, 50, {}))
+
+ self.assertEqual(self.call(test, 1, 2, foo=4, bar=5),
+ (1, 2, 3, 4, 5, {}))
+
+ with self.assertRaisesRegex(TypeError, "but was passed as a keyword"):
+ self.call(test, 1, 2, foo=4, bar=5, c_po=10)
+
with self.assertRaisesRegex(TypeError, "parameter is positional only"):
self.call(test, 1, 2, c_po=4)
@@ -2274,6 +2817,22 @@ class TestSignatureBind(unittest.TestCase):
ba = sig.bind(1, self=2, b=3)
self.assertEqual(ba.args, (1, 2, 3))
+ def test_signature_bind_vararg_name(self):
+ def test(a, *args):
+ return a, args
+ sig = inspect.signature(test)
+
+ with self.assertRaisesRegex(TypeError, "too many keyword arguments"):
+ sig.bind(a=0, args=1)
+
+ def test(*args, **kwargs):
+ return args, kwargs
+ self.assertEqual(self.call(test, args=1), ((), {'args': 1}))
+
+ sig = inspect.signature(test)
+ ba = sig.bind(args=1)
+ self.assertEqual(ba.arguments, {'kwargs': {'args': 1}})
+
class TestBoundArguments(unittest.TestCase):
def test_signature_bound_arguments_unhashable(self):
@@ -2301,6 +2860,168 @@ class TestBoundArguments(unittest.TestCase):
self.assertNotEqual(ba, ba4)
+class TestSignaturePrivateHelpers(unittest.TestCase):
+ def test_signature_get_bound_param(self):
+ getter = inspect._signature_get_bound_param
+
+ self.assertEqual(getter('($self)'), 'self')
+ self.assertEqual(getter('($self, obj)'), 'self')
+ self.assertEqual(getter('($cls, /, obj)'), 'cls')
+
+ def _strip_non_python_syntax(self, input,
+ clean_signature, self_parameter, last_positional_only):
+ computed_clean_signature, \
+ computed_self_parameter, \
+ computed_last_positional_only = \
+ inspect._signature_strip_non_python_syntax(input)
+ self.assertEqual(computed_clean_signature, clean_signature)
+ self.assertEqual(computed_self_parameter, self_parameter)
+ self.assertEqual(computed_last_positional_only, last_positional_only)
+
+ def test_signature_strip_non_python_syntax(self):
+ self._strip_non_python_syntax(
+ "($module, /, path, mode, *, dir_fd=None, " +
+ "effective_ids=False,\n follow_symlinks=True)",
+ "(module, path, mode, *, dir_fd=None, " +
+ "effective_ids=False, follow_symlinks=True)",
+ 0,
+ 0)
+
+ self._strip_non_python_syntax(
+ "($module, word, salt, /)",
+ "(module, word, salt)",
+ 0,
+ 2)
+
+ self._strip_non_python_syntax(
+ "(x, y=None, z=None, /)",
+ "(x, y=None, z=None)",
+ None,
+ 2)
+
+ self._strip_non_python_syntax(
+ "(x, y=None, z=None)",
+ "(x, y=None, z=None)",
+ None,
+ None)
+
+ self._strip_non_python_syntax(
+ "(x,\n y=None,\n z = None )",
+ "(x, y=None, z=None)",
+ None,
+ None)
+
+ self._strip_non_python_syntax(
+ "",
+ "",
+ None,
+ None)
+
+ self._strip_non_python_syntax(
+ None,
+ None,
+ None,
+ None)
+
+
+class TestUnwrap(unittest.TestCase):
+
+ def test_unwrap_one(self):
+ def func(a, b):
+ return a + b
+ wrapper = functools.lru_cache(maxsize=20)(func)
+ self.assertIs(inspect.unwrap(wrapper), func)
+
+ def test_unwrap_several(self):
+ def func(a, b):
+ return a + b
+ wrapper = func
+ for __ in range(10):
+ @functools.wraps(wrapper)
+ def wrapper():
+ pass
+ self.assertIsNot(wrapper.__wrapped__, func)
+ self.assertIs(inspect.unwrap(wrapper), func)
+
+ def test_stop(self):
+ def func1(a, b):
+ return a + b
+ @functools.wraps(func1)
+ def func2():
+ pass
+ @functools.wraps(func2)
+ def wrapper():
+ pass
+ func2.stop_here = 1
+ unwrapped = inspect.unwrap(wrapper,
+ stop=(lambda f: hasattr(f, "stop_here")))
+ self.assertIs(unwrapped, func2)
+
+ def test_cycle(self):
+ def func1(): pass
+ func1.__wrapped__ = func1
+ with self.assertRaisesRegex(ValueError, 'wrapper loop'):
+ inspect.unwrap(func1)
+
+ def func2(): pass
+ func2.__wrapped__ = func1
+ func1.__wrapped__ = func2
+ with self.assertRaisesRegex(ValueError, 'wrapper loop'):
+ inspect.unwrap(func1)
+ with self.assertRaisesRegex(ValueError, 'wrapper loop'):
+ inspect.unwrap(func2)
+
+ def test_unhashable(self):
+ def func(): pass
+ func.__wrapped__ = None
+ class C:
+ __hash__ = None
+ __wrapped__ = func
+ self.assertIsNone(inspect.unwrap(C()))
+
+class TestMain(unittest.TestCase):
+ def test_only_source(self):
+ module = importlib.import_module('unittest')
+ rc, out, err = assert_python_ok('-m', 'inspect',
+ 'unittest')
+ lines = out.decode().splitlines()
+ # ignore the final newline
+ self.assertEqual(lines[:-1], inspect.getsource(module).splitlines())
+ self.assertEqual(err, b'')
+
+ @unittest.skipIf(ThreadPoolExecutor is None,
+ 'threads required to test __qualname__ for source files')
+ def test_qualname_source(self):
+ rc, out, err = assert_python_ok('-m', 'inspect',
+ 'concurrent.futures:ThreadPoolExecutor')
+ lines = out.decode().splitlines()
+ # ignore the final newline
+ self.assertEqual(lines[:-1],
+ inspect.getsource(ThreadPoolExecutor).splitlines())
+ self.assertEqual(err, b'')
+
+ def test_builtins(self):
+ module = importlib.import_module('unittest')
+ _, out, err = assert_python_failure('-m', 'inspect',
+ 'sys')
+ lines = err.decode().splitlines()
+ self.assertEqual(lines, ["Can't get info for builtin modules."])
+
+ def test_details(self):
+ module = importlib.import_module('unittest')
+ rc, out, err = assert_python_ok('-m', 'inspect',
+ 'unittest', '--details')
+ output = out.decode()
+ # Just a quick sanity check on the output
+ self.assertIn(module.__name__, output)
+ self.assertIn(module.__file__, output)
+ if not sys.flags.optimize:
+ self.assertIn(module.__cached__, output)
+ self.assertEqual(err, b'')
+
+
+
+
def test_main():
run_unittest(
TestDecorators, TestRetrievingSourceCode, TestOneliners, TestBuggyCases,
@@ -2308,7 +3029,8 @@ def test_main():
TestGetcallargsFunctions, TestGetcallargsMethods,
TestGetcallargsUnboundMethods, TestGetattrStatic, TestGetGeneratorState,
TestNoEOL, TestSignatureObject, TestSignatureBind, TestParameterObject,
- TestBoundArguments, TestGetClosureVars
+ TestBoundArguments, TestSignaturePrivateHelpers, TestGetClosureVars,
+ TestUnwrap, TestMain
)
if __name__ == "__main__":
diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py
index c489510..e94602e 100644
--- a/Lib/test/test_int.py
+++ b/Lib/test/test_int.py
@@ -228,6 +228,47 @@ class IntTestCases(unittest.TestCase):
self.assertRaises(TypeError, int, base=10)
self.assertRaises(TypeError, int, base=0)
+ def test_int_base_limits(self):
+ """Testing the supported limits of the int() base parameter."""
+ self.assertEqual(int('0', 5), 0)
+ with self.assertRaises(ValueError):
+ int('0', 1)
+ with self.assertRaises(ValueError):
+ int('0', 37)
+ with self.assertRaises(ValueError):
+ int('0', -909) # An old magic value base from Python 2.
+ with self.assertRaises(ValueError):
+ int('0', base=0-(2**234))
+ with self.assertRaises(ValueError):
+ int('0', base=2**234)
+ # Bases 2 through 36 are supported.
+ for base in range(2,37):
+ self.assertEqual(int('0', base=base), 0)
+
+ def test_int_base_bad_types(self):
+ """Not integer types are not valid bases; issue16772."""
+ with self.assertRaises(TypeError):
+ int('0', 5.5)
+ with self.assertRaises(TypeError):
+ int('0', 5.0)
+
+ def test_int_base_indexable(self):
+ class MyIndexable(object):
+ def __init__(self, value):
+ self.value = value
+ def __index__(self):
+ return self.value
+
+ # Check out of range bases.
+ for base in 2**100, -2**100, 1, 37:
+ with self.assertRaises(ValueError):
+ int('43', base)
+
+ # Check in-range bases.
+ self.assertEqual(int('101', base=MyIndexable(2)), 5)
+ self.assertEqual(int('101', base=MyIndexable(10)), 101)
+ self.assertEqual(int('101', base=MyIndexable(36)), 1 + 36**2)
+
def test_non_numeric_input_types(self):
# Test possible non-numeric types for the argument x, including
# subclasses of the explicitly documented accepted types.
@@ -359,15 +400,18 @@ class IntTestCases(unittest.TestCase):
return True
bad_int = BadInt()
- n = int(bad_int)
+ with self.assertWarns(DeprecationWarning):
+ n = int(bad_int)
self.assertEqual(n, 1)
bad_int = BadInt2()
- n = int(bad_int)
+ with self.assertWarns(DeprecationWarning):
+ n = int(bad_int)
self.assertEqual(n, 1)
bad_int = TruncReturnsBadInt()
- n = int(bad_int)
+ with self.assertWarns(DeprecationWarning):
+ n = int(bad_int)
self.assertEqual(n, 1)
good_int = TruncReturnsIntSubclass()
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index 2a96b7b..267537f 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -35,6 +35,7 @@ import weakref
from collections import deque, UserList
from itertools import cycle, count
from test import support
+from test.script_helper import assert_python_ok
import codecs
import io # C implementation of io
@@ -164,7 +165,7 @@ class CloseFailureIO(MockRawIO):
def close(self):
if not self.closed:
self.closed = 1
- raise IOError
+ raise OSError
class CCloseFailureIO(CloseFailureIO, io.RawIOBase):
pass
@@ -595,9 +596,9 @@ class IOTest(unittest.TestCase):
def test_flush_error_on_close(self):
f = self.open(support.TESTFN, "wb", buffering=0)
def bad_flush():
- raise IOError()
+ raise OSError()
f.flush = bad_flush
- self.assertRaises(IOError, f.close) # exception not swallowed
+ self.assertRaises(OSError, f.close) # exception not swallowed
self.assertTrue(f.closed)
def test_multi_close(self):
@@ -757,7 +758,7 @@ class CommonBufferedTests:
if s:
# The destructor *may* have printed an unraisable error, check it
self.assertEqual(len(s.splitlines()), 1)
- self.assertTrue(s.startswith("Exception IOError: "), s)
+ self.assertTrue(s.startswith("Exception OSError: "), s)
self.assertTrue(s.endswith(" ignored"), s)
def test_repr(self):
@@ -773,22 +774,22 @@ class CommonBufferedTests:
def test_flush_error_on_close(self):
raw = self.MockRawIO()
def bad_flush():
- raise IOError()
+ raise OSError()
raw.flush = bad_flush
b = self.tp(raw)
- self.assertRaises(IOError, b.close) # exception not swallowed
+ self.assertRaises(OSError, b.close) # exception not swallowed
self.assertTrue(b.closed)
def test_close_error_on_close(self):
raw = self.MockRawIO()
def bad_flush():
- raise IOError('flush')
+ raise OSError('flush')
def bad_close():
- raise IOError('close')
+ raise OSError('close')
raw.close = bad_close
b = self.tp(raw)
b.flush = bad_flush
- with self.assertRaises(IOError) as err: # exception not swallowed
+ with self.assertRaises(OSError) as err: # exception not swallowed
b.close()
self.assertEqual(err.exception.args, ('close',))
self.assertEqual(err.exception.__context__.args, ('flush',))
@@ -828,6 +829,14 @@ class SizeofTest:
bufio = self.tp(rawio, buffer_size=bufsize2)
self.assertEqual(sys.getsizeof(bufio), size + bufsize2)
+ @support.cpython_only
+ def test_buffer_freeing(self) :
+ bufsize = 4096
+ rawio = self.MockRawIO()
+ bufio = self.tp(rawio, buffer_size=bufsize)
+ size = sys.getsizeof(bufio) - bufsize
+ bufio.close()
+ self.assertEqual(sys.getsizeof(bufio), size)
class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
read_mode = "rb"
@@ -1012,8 +1021,8 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
def test_misbehaved_io(self):
rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg"))
bufio = self.tp(rawio)
- self.assertRaises(IOError, bufio.seek, 0)
- self.assertRaises(IOError, bufio.tell)
+ self.assertRaises(OSError, bufio.seek, 0)
+ self.assertRaises(OSError, bufio.tell)
def test_no_extraneous_read(self):
# Issue #9550; when the raw IO object has satisfied the read request,
@@ -1064,17 +1073,18 @@ class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
bufio = self.tp(rawio)
# _pyio.BufferedReader seems to implement reading different, so that
# checking this is not so easy.
- self.assertRaises(IOError, bufio.read, 10)
+ self.assertRaises(OSError, bufio.read, 10)
def test_garbage_collection(self):
# C BufferedReader objects are collected.
# The Python version has __del__, so it ends into gc.garbage instead
- rawio = self.FileIO(support.TESTFN, "w+b")
- f = self.tp(rawio)
- f.f = f
- wr = weakref.ref(f)
- del f
- support.gc_collect()
+ with support.check_warnings(('', ResourceWarning)):
+ rawio = self.FileIO(support.TESTFN, "w+b")
+ f = self.tp(rawio)
+ f.f = f
+ wr = weakref.ref(f)
+ del f
+ support.gc_collect()
self.assertTrue(wr() is None, wr)
def test_args_error(self):
@@ -1327,9 +1337,9 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
def test_misbehaved_io(self):
rawio = self.MisbehavedRawIO()
bufio = self.tp(rawio, 5)
- self.assertRaises(IOError, bufio.seek, 0)
- self.assertRaises(IOError, bufio.tell)
- self.assertRaises(IOError, bufio.write, b"abcdef")
+ self.assertRaises(OSError, bufio.seek, 0)
+ self.assertRaises(OSError, bufio.tell)
+ self.assertRaises(OSError, bufio.write, b"abcdef")
def test_max_buffer_size_removal(self):
with self.assertRaises(TypeError):
@@ -1338,11 +1348,11 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
def test_write_error_on_close(self):
raw = self.MockRawIO()
def bad_write(b):
- raise IOError()
+ raise OSError()
raw.write = bad_write
b = self.tp(raw)
b.write(b'spam')
- self.assertRaises(IOError, b.close) # exception not swallowed
+ self.assertRaises(OSError, b.close) # exception not swallowed
self.assertTrue(b.closed)
@@ -1373,13 +1383,14 @@ class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
# C BufferedWriter objects are collected, and collecting them flushes
# all data to disk.
# The Python version has __del__, so it ends into gc.garbage instead
- rawio = self.FileIO(support.TESTFN, "w+b")
- f = self.tp(rawio)
- f.write(b"123xxx")
- f.x = f
- wr = weakref.ref(f)
- del f
- support.gc_collect()
+ with support.check_warnings(('', ResourceWarning)):
+ rawio = self.FileIO(support.TESTFN, "w+b")
+ f = self.tp(rawio)
+ f.write(b"123xxx")
+ f.x = f
+ wr = weakref.ref(f)
+ del f
+ support.gc_collect()
self.assertTrue(wr() is None, wr)
with self.open(support.TESTFN, "rb") as f:
self.assertEqual(f.read(), b"123xxx")
@@ -1426,14 +1437,14 @@ class BufferedRWPairTest(unittest.TestCase):
def readable(self):
return False
- self.assertRaises(IOError, self.tp, NotReadable(), self.MockRawIO())
+ self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO())
def test_constructor_with_not_writeable(self):
class NotWriteable(MockRawIO):
def writable(self):
return False
- self.assertRaises(IOError, self.tp, self.MockRawIO(), NotWriteable())
+ self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable())
def test_read(self):
pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
@@ -1962,7 +1973,7 @@ class TextIOWrapperTest(unittest.TestCase):
r = self.BytesIO()
b = self.BufferedWriter(r)
with self.assertRaisesRegex(LookupError, "is not a text encoding"):
- self.TextIOWrapper(b, encoding="hex_codec")
+ self.TextIOWrapper(b, encoding="hex")
def test_detach(self):
r = self.BytesIO()
@@ -2209,7 +2220,7 @@ class TextIOWrapperTest(unittest.TestCase):
if s:
# The destructor *may* have printed an unraisable error, check it
self.assertEqual(len(s.splitlines()), 1)
- self.assertTrue(s.startswith("Exception IOError: "), s)
+ self.assertTrue(s.startswith("Exception OSError: "), s)
self.assertTrue(s.endswith(" ignored"), s)
# Systematic tests of the text I/O API
@@ -2281,7 +2292,7 @@ class TextIOWrapperTest(unittest.TestCase):
f.seek(0)
for line in f:
self.assertEqual(line, "\xff\n")
- self.assertRaises(IOError, f.tell)
+ self.assertRaises(OSError, f.tell)
self.assertEqual(f.tell(), p2)
f.close()
@@ -2385,7 +2396,7 @@ class TextIOWrapperTest(unittest.TestCase):
def readable(self):
return False
txt = self.TextIOWrapper(UnReadable())
- self.assertRaises(IOError, txt.read)
+ self.assertRaises(OSError, txt.read)
def test_read_one_by_one(self):
txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"))
@@ -2560,9 +2571,9 @@ class TextIOWrapperTest(unittest.TestCase):
def test_flush_error_on_close(self):
txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
def bad_flush():
- raise IOError()
+ raise OSError()
txt.flush = bad_flush
- self.assertRaises(IOError, txt.close) # exception not swallowed
+ self.assertRaises(OSError, txt.close) # exception not swallowed
self.assertTrue(txt.closed)
def test_multi_close(self):
@@ -2618,11 +2629,11 @@ class TextIOWrapperTest(unittest.TestCase):
# Issue #17106
# Bypass the early encoding check added in issue 20404
def _make_illegal_wrapper():
- quopri = codecs.lookup("quopri_codec")
+ quopri = codecs.lookup("quopri")
quopri._is_text_encoding = True
try:
t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'),
- newline='\n', encoding="quopri_codec")
+ newline='\n', encoding="quopri")
finally:
quopri._is_text_encoding = False
return t
@@ -2634,8 +2645,46 @@ class TextIOWrapperTest(unittest.TestCase):
t = _make_illegal_wrapper()
self.assertRaises(TypeError, t.read)
+ def _check_create_at_shutdown(self, **kwargs):
+ # Issue #20037: creating a TextIOWrapper at shutdown
+ # shouldn't crash the interpreter.
+ iomod = self.io.__name__
+ code = """if 1:
+ import codecs
+ import {iomod} as io
+
+ # Avoid looking up codecs at shutdown
+ codecs.lookup('utf-8')
+
+ class C:
+ def __init__(self):
+ self.buf = io.BytesIO()
+ def __del__(self):
+ io.TextIOWrapper(self.buf, **{kwargs})
+ print("ok")
+ c = C()
+ """.format(iomod=iomod, kwargs=kwargs)
+ return assert_python_ok("-c", code)
+
+ def test_create_at_shutdown_without_encoding(self):
+ rc, out, err = self._check_create_at_shutdown()
+ if err:
+ # Can error out with a RuntimeError if the module state
+ # isn't found.
+ self.assertIn(self.shutdown_error, err.decode())
+ else:
+ self.assertEqual("ok", out.decode().strip())
+
+ def test_create_at_shutdown_with_encoding(self):
+ rc, out, err = self._check_create_at_shutdown(encoding='utf-8',
+ errors='strict')
+ self.assertFalse(err)
+ self.assertEqual("ok", out.decode().strip())
+
class CTextIOWrapperTest(TextIOWrapperTest):
+ io = io
+ shutdown_error = "RuntimeError: could not find io module state"
def test_initialization(self):
r = self.BytesIO(b"\xc3\xa9\n\n")
@@ -2650,14 +2699,15 @@ class CTextIOWrapperTest(TextIOWrapperTest):
# C TextIOWrapper objects are collected, and collecting them flushes
# all data to disk.
# The Python version has __del__, so it ends in gc.garbage instead.
- rawio = io.FileIO(support.TESTFN, "wb")
- b = self.BufferedWriter(rawio)
- t = self.TextIOWrapper(b, encoding="ascii")
- t.write("456def")
- t.x = t
- wr = weakref.ref(t)
- del t
- support.gc_collect()
+ with support.check_warnings(('', ResourceWarning)):
+ rawio = io.FileIO(support.TESTFN, "wb")
+ b = self.BufferedWriter(rawio)
+ t = self.TextIOWrapper(b, encoding="ascii")
+ t.write("456def")
+ t.x = t
+ wr = weakref.ref(t)
+ del t
+ support.gc_collect()
self.assertTrue(wr() is None, wr)
with self.open(support.TESTFN, "rb") as f:
self.assertEqual(f.read(), b"456def")
@@ -2678,7 +2728,9 @@ class CTextIOWrapperTest(TextIOWrapperTest):
class PyTextIOWrapperTest(TextIOWrapperTest):
- pass
+ io = pyio
+ #shutdown_error = "LookupError: unknown encoding: ascii"
+ shutdown_error = "TypeError: 'NoneType' object is not iterable"
class IncrementalNewlineDecoderTest(unittest.TestCase):
@@ -2817,7 +2869,8 @@ class MiscIOTest(unittest.TestCase):
self.assertEqual(f.mode, "wb")
f.close()
- f = self.open(support.TESTFN, "U")
+ with support.check_warnings(('', DeprecationWarning)):
+ f = self.open(support.TESTFN, "U")
self.assertEqual(f.name, support.TESTFN)
self.assertEqual(f.buffer.name, support.TESTFN)
self.assertEqual(f.buffer.raw.name, support.TESTFN)
@@ -2947,7 +3000,7 @@ class MiscIOTest(unittest.TestCase):
for fd in fds:
try:
os.close(fd)
- except EnvironmentError as e:
+ except OSError as e:
if e.errno != errno.EBADF:
raise
self.addCleanup(cleanup_fds)
@@ -3069,7 +3122,6 @@ class MiscIOTest(unittest.TestCase):
class CMiscIOTest(MiscIOTest):
io = io
- shutdown_error = "RuntimeError: could not find io module state"
def test_readinto_buffer_overflow(self):
# Issue #18025
@@ -3082,7 +3134,6 @@ class CMiscIOTest(MiscIOTest):
class PyMiscIOTest(MiscIOTest):
io = pyio
- shutdown_error = "LookupError: unknown encoding: ascii"
@unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.')
@@ -3115,15 +3166,18 @@ class SignalsTest(unittest.TestCase):
try:
wio = self.io.open(w, **fdopen_kwargs)
t.start()
- signal.alarm(1)
# Fill the pipe enough that the write will be blocking.
# It will be interrupted by the timer armed above. Since the
# other thread has read one byte, the low-level write will
# return with a successful (partial) result rather than an EINTR.
# The buffered IO layer must check for pending signal
# handlers, which in this case will invoke alarm_interrupt().
- self.assertRaises(ZeroDivisionError,
- wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1))
+ signal.alarm(1)
+ try:
+ self.assertRaises(ZeroDivisionError,
+ wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1))
+ finally:
+ signal.alarm(0)
t.join()
# We got one byte, get another one and check that it isn't a
# repeat of the first one.
@@ -3137,7 +3191,7 @@ class SignalsTest(unittest.TestCase):
# buffer, and block again.
try:
wio.close()
- except IOError as e:
+ except OSError as e:
if e.errno != errno.EBADF:
raise
@@ -3265,7 +3319,7 @@ class SignalsTest(unittest.TestCase):
# buffer, and could block (in case of failure).
try:
wio.close()
- except IOError as e:
+ except OSError as e:
if e.errno != errno.EBADF:
raise
diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py
index 531c9af..efe9f51 100644
--- a/Lib/test/test_ioctl.py
+++ b/Lib/test/test_ioctl.py
@@ -8,7 +8,7 @@ get_attribute(termios, 'TIOCGPGRP') #Can't run tests without this feature
try:
tty = open("/dev/tty", "rb")
-except IOError:
+except OSError:
raise unittest.SkipTest("Unable to open /dev/tty")
else:
# Skip if another process is in foreground
@@ -86,8 +86,6 @@ class IoctlTests(unittest.TestCase):
os.close(mfd)
os.close(sfd)
-def test_main():
- run_unittest(IoctlTests)
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index 5bfcf41..f2947b9 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -1358,6 +1358,14 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(True, ipaddress.ip_network(
'127.42.0.0/16').is_loopback)
self.assertEqual(False, ipaddress.ip_network('128.0.0.0').is_loopback)
+ self.assertEqual(False,
+ ipaddress.ip_network('100.64.0.0/10').is_private)
+ self.assertEqual(False, ipaddress.ip_network('100.64.0.0/10').is_global)
+
+ self.assertEqual(True,
+ ipaddress.ip_network('192.0.2.128/25').is_private)
+ self.assertEqual(True,
+ ipaddress.ip_network('192.0.3.0/24').is_global)
# test addresses
self.assertEqual(True, ipaddress.ip_address('0.0.0.0').is_unspecified)
@@ -1423,6 +1431,10 @@ class IpaddrUnitTest(unittest.TestCase):
self.assertEqual(False, ipaddress.ip_network('::1').is_unspecified)
self.assertEqual(False, ipaddress.ip_network('::/127').is_unspecified)
+ self.assertEqual(True,
+ ipaddress.ip_network('2001::1/128').is_private)
+ self.assertEqual(True,
+ ipaddress.ip_network('200::1/128').is_global)
# test addresses
self.assertEqual(True, ipaddress.ip_address('ffff::').is_multicast)
self.assertEqual(True, ipaddress.ip_address(2**128 - 1).is_multicast)
diff --git a/Lib/test/test_iterlen.py b/Lib/test/test_iterlen.py
index 9101f6c..152f5fc 100644
--- a/Lib/test/test_iterlen.py
+++ b/Lib/test/test_iterlen.py
@@ -45,31 +45,21 @@ import unittest
from test import support
from itertools import repeat
from collections import deque
-from builtins import len as _len
+from operator import length_hint
n = 10
-def len(obj):
- try:
- return _len(obj)
- except TypeError:
- try:
- # note: this is an internal undocumented API,
- # don't rely on it in your own programs
- return obj.__length_hint__()
- except AttributeError:
- raise TypeError
class TestInvariantWithoutMutations:
def test_invariant(self):
it = self.it
for i in reversed(range(1, n+1)):
- self.assertEqual(len(it), i)
+ self.assertEqual(length_hint(it), i)
next(it)
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
self.assertRaises(StopIteration, next, it)
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
class TestTemporarilyImmutable(TestInvariantWithoutMutations):
@@ -78,12 +68,12 @@ class TestTemporarilyImmutable(TestInvariantWithoutMutations):
# length immutability during iteration
it = self.it
- self.assertEqual(len(it), n)
+ self.assertEqual(length_hint(it), n)
next(it)
- self.assertEqual(len(it), n-1)
+ self.assertEqual(length_hint(it), n-1)
self.mutate()
self.assertRaises(RuntimeError, next, it)
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
## ------- Concrete Type Tests -------
@@ -92,10 +82,6 @@ class TestRepeat(TestInvariantWithoutMutations, unittest.TestCase):
def setUp(self):
self.it = repeat(None, n)
- def test_no_len_for_infinite_repeat(self):
- # The repeat() object can also be infinite
- self.assertRaises(TypeError, len, repeat(None))
-
class TestXrange(TestInvariantWithoutMutations, unittest.TestCase):
def setUp(self):
@@ -167,14 +153,15 @@ class TestList(TestInvariantWithoutMutations, unittest.TestCase):
it = iter(d)
next(it)
next(it)
- self.assertEqual(len(it), n-2)
+ self.assertEqual(length_hint(it), n - 2)
d.append(n)
- self.assertEqual(len(it), n-1) # grow with append
+ self.assertEqual(length_hint(it), n - 1) # grow with append
d[1:] = []
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
self.assertEqual(list(it), [])
d.extend(range(20))
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
+
class TestListReversed(TestInvariantWithoutMutations, unittest.TestCase):
@@ -186,32 +173,41 @@ class TestListReversed(TestInvariantWithoutMutations, unittest.TestCase):
it = reversed(d)
next(it)
next(it)
- self.assertEqual(len(it), n-2)
+ self.assertEqual(length_hint(it), n - 2)
d.append(n)
- self.assertEqual(len(it), n-2) # ignore append
+ self.assertEqual(length_hint(it), n - 2) # ignore append
d[1:] = []
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
self.assertEqual(list(it), []) # confirm invariant
d.extend(range(20))
- self.assertEqual(len(it), 0)
+ self.assertEqual(length_hint(it), 0)
## -- Check to make sure exceptions are not suppressed by __length_hint__()
class BadLen(object):
- def __iter__(self): return iter(range(10))
+ def __iter__(self):
+ return iter(range(10))
+
def __len__(self):
raise RuntimeError('hello')
+
class BadLengthHint(object):
- def __iter__(self): return iter(range(10))
+ def __iter__(self):
+ return iter(range(10))
+
def __length_hint__(self):
raise RuntimeError('hello')
+
class NoneLengthHint(object):
- def __iter__(self): return iter(range(10))
+ def __iter__(self):
+ return iter(range(10))
+
def __length_hint__(self):
- return None
+ return NotImplemented
+
class TestLengthHintExceptions(unittest.TestCase):
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 514a6b7..21f1bde 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -10,6 +10,8 @@ import random
import copy
import pickle
from functools import reduce
+import sys
+import struct
maxsize = support.MAX_Py_ssize_t
minsize = -maxsize-1
@@ -1729,9 +1731,8 @@ class TestVariousIteratorArgs(unittest.TestCase):
class LengthTransparency(unittest.TestCase):
def test_repeat(self):
- from test.test_iterlen import len
- self.assertEqual(len(repeat(None, 50)), 50)
- self.assertRaises(TypeError, len, repeat(None))
+ self.assertEqual(operator.length_hint(repeat(None, 50)), 50)
+ self.assertEqual(operator.length_hint(repeat(None), 12), 12)
class RegressionTests(unittest.TestCase):
@@ -1808,6 +1809,44 @@ class SubclassWithKwargsTest(unittest.TestCase):
# we expect type errors because of wrong argument count
self.assertNotIn("does not take keyword arguments", err.args[0])
+@support.cpython_only
+class SizeofTest(unittest.TestCase):
+ def setUp(self):
+ self.ssize_t = struct.calcsize('n')
+
+ check_sizeof = support.check_sizeof
+
+ def test_product_sizeof(self):
+ basesize = support.calcobjsize('3Pi')
+ check = self.check_sizeof
+ check(product('ab', '12'), basesize + 2 * self.ssize_t)
+ check(product(*(('abc',) * 10)), basesize + 10 * self.ssize_t)
+
+ def test_combinations_sizeof(self):
+ basesize = support.calcobjsize('3Pni')
+ check = self.check_sizeof
+ check(combinations('abcd', 3), basesize + 3 * self.ssize_t)
+ check(combinations(range(10), 4), basesize + 4 * self.ssize_t)
+
+ def test_combinations_with_replacement_sizeof(self):
+ cwr = combinations_with_replacement
+ basesize = support.calcobjsize('3Pni')
+ check = self.check_sizeof
+ check(cwr('abcd', 3), basesize + 3 * self.ssize_t)
+ check(cwr(range(10), 4), basesize + 4 * self.ssize_t)
+
+ def test_permutations_sizeof(self):
+ basesize = support.calcobjsize('4Pni')
+ check = self.check_sizeof
+ check(permutations('abcd'),
+ basesize + 4 * self.ssize_t + 4 * self.ssize_t)
+ check(permutations('abcd', 3),
+ basesize + 4 * self.ssize_t + 3 * self.ssize_t)
+ check(permutations('abcde', 3),
+ basesize + 5 * self.ssize_t + 3 * self.ssize_t)
+ check(permutations(range(10), 4),
+ basesize + 10 * self.ssize_t + 4 * self.ssize_t)
+
libreftest = """ Doctest for examples in the library reference: libitertools.tex
@@ -2043,7 +2082,8 @@ __test__ = {'libreftest' : libreftest}
def test_main(verbose=None):
test_classes = (TestBasicOps, TestVariousIteratorArgs, TestGC,
RegressionTests, LengthTransparency,
- SubclassWithKwargsTest, TestExamples)
+ SubclassWithKwargsTest, TestExamples,
+ SizeofTest)
support.run_unittest(*test_classes)
# verify reference counting
diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py
index d23e285..35c02de 100644
--- a/Lib/test/test_json/test_decode.py
+++ b/Lib/test/test_json/test_decode.py
@@ -1,5 +1,5 @@
import decimal
-from io import StringIO
+from io import StringIO, BytesIO
from collections import OrderedDict
from test.test_json import PyTest, CTest
@@ -70,5 +70,26 @@ class TestDecode:
msg = 'escape'
self.assertRaisesRegex(ValueError, msg, self.loads, s)
+ def test_invalid_input_type(self):
+ msg = 'the JSON object must be str'
+ for value in [1, 3.14, b'bytes', b'\xff\x00', [], {}, None]:
+ self.assertRaisesRegex(TypeError, msg, self.loads, value)
+ with self.assertRaisesRegex(TypeError, msg):
+ self.json.load(BytesIO(b'[1,2,3]'))
+
+ def test_string_with_utf8_bom(self):
+ # see #18958
+ bom_json = "[1,2,3]".encode('utf-8-sig').decode('utf-8')
+ with self.assertRaises(ValueError) as cm:
+ self.loads(bom_json)
+ self.assertIn('BOM', str(cm.exception))
+ with self.assertRaises(ValueError) as cm:
+ self.json.load(StringIO(bom_json))
+ self.assertIn('BOM', str(cm.exception))
+ # make sure that the BOM is not detected in the middle of a string
+ bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8'))
+ self.assertEqual(self.loads(bom_in_str), '\ufeff')
+ self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff')
+
class TestPyDecode(TestDecode, PyTest): pass
class TestCDecode(TestDecode, CTest): pass
diff --git a/Lib/test/test_json/test_enum.py b/Lib/test/test_json/test_enum.py
new file mode 100644
index 0000000..10f4148
--- /dev/null
+++ b/Lib/test/test_json/test_enum.py
@@ -0,0 +1,120 @@
+from enum import Enum, IntEnum
+from math import isnan
+from test.test_json import PyTest, CTest
+
+SMALL = 1
+BIG = 1<<32
+HUGE = 1<<64
+REALLY_HUGE = 1<<96
+
+class BigNum(IntEnum):
+ small = SMALL
+ big = BIG
+ huge = HUGE
+ really_huge = REALLY_HUGE
+
+E = 2.718281
+PI = 3.141593
+TAU = 2 * PI
+
+class FloatNum(float, Enum):
+ e = E
+ pi = PI
+ tau = TAU
+
+INF = float('inf')
+NEG_INF = float('-inf')
+NAN = float('nan')
+
+class WierdNum(float, Enum):
+ inf = INF
+ neg_inf = NEG_INF
+ nan = NAN
+
+class TestEnum:
+
+ def test_floats(self):
+ for enum in FloatNum:
+ self.assertEqual(self.dumps(enum), repr(enum.value))
+ self.assertEqual(float(self.dumps(enum)), enum)
+ self.assertEqual(self.loads(self.dumps(enum)), enum)
+
+ def test_weird_floats(self):
+ for enum, expected in zip(WierdNum, ('Infinity', '-Infinity', 'NaN')):
+ self.assertEqual(self.dumps(enum), expected)
+ if not isnan(enum):
+ self.assertEqual(float(self.dumps(enum)), enum)
+ self.assertEqual(self.loads(self.dumps(enum)), enum)
+ else:
+ self.assertTrue(isnan(float(self.dumps(enum))))
+ self.assertTrue(isnan(self.loads(self.dumps(enum))))
+
+ def test_ints(self):
+ for enum in BigNum:
+ self.assertEqual(self.dumps(enum), str(enum.value))
+ self.assertEqual(int(self.dumps(enum)), enum)
+ self.assertEqual(self.loads(self.dumps(enum)), enum)
+
+ def test_list(self):
+ self.assertEqual(self.dumps(list(BigNum)),
+ str([SMALL, BIG, HUGE, REALLY_HUGE]))
+ self.assertEqual(self.loads(self.dumps(list(BigNum))),
+ list(BigNum))
+ self.assertEqual(self.dumps(list(FloatNum)),
+ str([E, PI, TAU]))
+ self.assertEqual(self.loads(self.dumps(list(FloatNum))),
+ list(FloatNum))
+ self.assertEqual(self.dumps(list(WierdNum)),
+ '[Infinity, -Infinity, NaN]')
+ self.assertEqual(self.loads(self.dumps(list(WierdNum)))[:2],
+ list(WierdNum)[:2])
+ self.assertTrue(isnan(self.loads(self.dumps(list(WierdNum)))[2]))
+
+ def test_dict_keys(self):
+ s, b, h, r = BigNum
+ e, p, t = FloatNum
+ i, j, n = WierdNum
+ d = {
+ s:'tiny', b:'large', h:'larger', r:'largest',
+ e:"Euler's number", p:'pi', t:'tau',
+ i:'Infinity', j:'-Infinity', n:'NaN',
+ }
+ nd = self.loads(self.dumps(d))
+ self.assertEqual(nd[str(SMALL)], 'tiny')
+ self.assertEqual(nd[str(BIG)], 'large')
+ self.assertEqual(nd[str(HUGE)], 'larger')
+ self.assertEqual(nd[str(REALLY_HUGE)], 'largest')
+ self.assertEqual(nd[repr(E)], "Euler's number")
+ self.assertEqual(nd[repr(PI)], 'pi')
+ self.assertEqual(nd[repr(TAU)], 'tau')
+ self.assertEqual(nd['Infinity'], 'Infinity')
+ self.assertEqual(nd['-Infinity'], '-Infinity')
+ self.assertEqual(nd['NaN'], 'NaN')
+
+ def test_dict_values(self):
+ d = dict(
+ tiny=BigNum.small,
+ large=BigNum.big,
+ larger=BigNum.huge,
+ largest=BigNum.really_huge,
+ e=FloatNum.e,
+ pi=FloatNum.pi,
+ tau=FloatNum.tau,
+ i=WierdNum.inf,
+ j=WierdNum.neg_inf,
+ n=WierdNum.nan,
+ )
+ nd = self.loads(self.dumps(d))
+ self.assertEqual(nd['tiny'], SMALL)
+ self.assertEqual(nd['large'], BIG)
+ self.assertEqual(nd['larger'], HUGE)
+ self.assertEqual(nd['largest'], REALLY_HUGE)
+ self.assertEqual(nd['e'], E)
+ self.assertEqual(nd['pi'], PI)
+ self.assertEqual(nd['tau'], TAU)
+ self.assertEqual(nd['i'], INF)
+ self.assertEqual(nd['j'], NEG_INF)
+ self.assertTrue(isnan(nd['n']))
+
+class TestPyEnum(TestEnum, PyTest): pass
+class TestCEnum(TestEnum, CTest): pass
diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py
index 3dd877a..7caafdb 100644
--- a/Lib/test/test_json/test_fail.py
+++ b/Lib/test/test_json/test_fail.py
@@ -1,4 +1,5 @@
from test.test_json import PyTest, CTest
+import re
# 2007-10-05
JSONDOCS = [
@@ -100,6 +101,94 @@ class TestFail:
#This is for python encoder
self.assertRaises(TypeError, self.dumps, data, indent=True)
+ def test_truncated_input(self):
+ test_cases = [
+ ('', 'Expecting value', 0),
+ ('[', 'Expecting value', 1),
+ ('[42', "Expecting ',' delimiter", 3),
+ ('[42,', 'Expecting value', 4),
+ ('["', 'Unterminated string starting at', 1),
+ ('["spam', 'Unterminated string starting at', 1),
+ ('["spam"', "Expecting ',' delimiter", 7),
+ ('["spam",', 'Expecting value', 8),
+ ('{', 'Expecting property name enclosed in double quotes', 1),
+ ('{"', 'Unterminated string starting at', 1),
+ ('{"spam', 'Unterminated string starting at', 1),
+ ('{"spam"', "Expecting ':' delimiter", 7),
+ ('{"spam":', 'Expecting value', 8),
+ ('{"spam":42', "Expecting ',' delimiter", 10),
+ ('{"spam":42,', 'Expecting property name enclosed in double quotes', 11),
+ ]
+ test_cases += [
+ ('"', 'Unterminated string starting at', 0),
+ ('"spam', 'Unterminated string starting at', 0),
+ ]
+ for data, msg, idx in test_cases:
+ self.assertRaisesRegex(ValueError,
+ r'^{0}: line 1 column {1} \(char {2}\)'.format(
+ re.escape(msg), idx + 1, idx),
+ self.loads, data)
+
+ def test_unexpected_data(self):
+ test_cases = [
+ ('[,', 'Expecting value', 1),
+ ('{"spam":[}', 'Expecting value', 9),
+ ('[42:', "Expecting ',' delimiter", 3),
+ ('[42 "spam"', "Expecting ',' delimiter", 4),
+ ('[42,]', 'Expecting value', 4),
+ ('{"spam":[42}', "Expecting ',' delimiter", 11),
+ ('["]', 'Unterminated string starting at', 1),
+ ('["spam":', "Expecting ',' delimiter", 7),
+ ('["spam",]', 'Expecting value', 8),
+ ('{:', 'Expecting property name enclosed in double quotes', 1),
+ ('{,', 'Expecting property name enclosed in double quotes', 1),
+ ('{42', 'Expecting property name enclosed in double quotes', 1),
+ ('[{]', 'Expecting property name enclosed in double quotes', 2),
+ ('{"spam",', "Expecting ':' delimiter", 7),
+ ('{"spam"}', "Expecting ':' delimiter", 7),
+ ('[{"spam"]', "Expecting ':' delimiter", 8),
+ ('{"spam":}', 'Expecting value', 8),
+ ('[{"spam":]', 'Expecting value', 9),
+ ('{"spam":42 "ham"', "Expecting ',' delimiter", 11),
+ ('[{"spam":42]', "Expecting ',' delimiter", 11),
+ ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11),
+ ]
+ for data, msg, idx in test_cases:
+ self.assertRaisesRegex(ValueError,
+ r'^{0}: line 1 column {1} \(char {2}\)'.format(
+ re.escape(msg), idx + 1, idx),
+ self.loads, data)
+
+ def test_extra_data(self):
+ test_cases = [
+ ('[]]', 'Extra data', 2),
+ ('{}}', 'Extra data', 2),
+ ('[],[]', 'Extra data', 2),
+ ('{},{}', 'Extra data', 2),
+ ]
+ test_cases += [
+ ('42,"spam"', 'Extra data', 2),
+ ('"spam",42', 'Extra data', 6),
+ ]
+ for data, msg, idx in test_cases:
+ self.assertRaisesRegex(ValueError,
+ r'^{0}: line 1 column {1} - line 1 column {2}'
+ r' \(char {3} - {4}\)'.format(
+ re.escape(msg), idx + 1, len(data) + 1, idx, len(data)),
+ self.loads, data)
+
+ def test_linecol(self):
+ test_cases = [
+ ('!', 1, 1, 0),
+ (' !', 1, 2, 1),
+ ('\n!', 2, 1, 1),
+ ('\n \n\n !', 4, 6, 10),
+ ]
+ for data, line, col, idx in test_cases:
+ self.assertRaisesRegex(ValueError,
+ r'^Expecting value: line {0} column {1}'
+ r' \(char {2}\)$'.format(line, col, idx),
+ self.loads, data)
class TestPyFail(TestFail, PyTest): pass
class TestCFail(TestFail, CTest): pass
diff --git a/Lib/test/test_json/test_indent.py b/Lib/test/test_json/test_indent.py
index a4d4d20..e07856f 100644
--- a/Lib/test/test_json/test_indent.py
+++ b/Lib/test/test_json/test_indent.py
@@ -32,6 +32,8 @@ class TestIndent:
d1 = self.dumps(h)
d2 = self.dumps(h, indent=2, sort_keys=True, separators=(',', ': '))
d3 = self.dumps(h, indent='\t', sort_keys=True, separators=(',', ': '))
+ d4 = self.dumps(h, indent=2, sort_keys=True)
+ d5 = self.dumps(h, indent='\t', sort_keys=True)
h1 = self.loads(d1)
h2 = self.loads(d2)
@@ -42,6 +44,8 @@ class TestIndent:
self.assertEqual(h3, h)
self.assertEqual(d2, expect.expandtabs(2))
self.assertEqual(d3, expect)
+ self.assertEqual(d4, d2)
+ self.assertEqual(d5, d3)
def test_indent0(self):
h = {3: 1}
diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py
new file mode 100644
index 0000000..af99f52
--- /dev/null
+++ b/Lib/test/test_keyword.py
@@ -0,0 +1,138 @@
+import keyword
+import unittest
+from test import support
+import filecmp
+import os
+import sys
+import subprocess
+import shutil
+import textwrap
+
+KEYWORD_FILE = support.findfile('keyword.py')
+GRAMMAR_FILE = os.path.join(os.path.split(__file__)[0],
+ '..', '..', 'Python', 'graminit.c')
+TEST_PY_FILE = 'keyword_test.py'
+GRAMMAR_TEST_FILE = 'graminit_test.c'
+PY_FILE_WITHOUT_KEYWORDS = 'minimal_keyword.py'
+NONEXISTENT_FILE = 'not_here.txt'
+
+
+class Test_iskeyword(unittest.TestCase):
+ def test_true_is_a_keyword(self):
+ self.assertTrue(keyword.iskeyword('True'))
+
+ def test_uppercase_true_is_not_a_keyword(self):
+ self.assertFalse(keyword.iskeyword('TRUE'))
+
+ def test_none_value_is_not_a_keyword(self):
+ self.assertFalse(keyword.iskeyword(None))
+
+ # This is probably an accident of the current implementation, but should be
+ # preserved for backward compatibility.
+ def test_changing_the_kwlist_does_not_affect_iskeyword(self):
+ oldlist = keyword.kwlist
+ self.addCleanup(setattr, keyword, 'kwlist', oldlist)
+ keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice']
+ self.assertFalse(keyword.iskeyword('eggs'))
+
+
+class TestKeywordGeneration(unittest.TestCase):
+
+ def _copy_file_without_generated_keywords(self, source_file, dest_file):
+ with open(source_file, 'rb') as fp:
+ lines = fp.readlines()
+ nl = lines[0][len(lines[0].strip()):]
+ with open(dest_file, 'wb') as fp:
+ fp.writelines(lines[:lines.index(b"#--start keywords--" + nl) + 1])
+ fp.writelines(lines[lines.index(b"#--end keywords--" + nl):])
+
+ def _generate_keywords(self, grammar_file, target_keyword_py_file):
+ proc = subprocess.Popen([sys.executable,
+ KEYWORD_FILE,
+ grammar_file,
+ target_keyword_py_file], stderr=subprocess.PIPE)
+ stderr = proc.communicate()[1]
+ return proc.returncode, stderr
+
+ @unittest.skipIf(not os.path.exists(GRAMMAR_FILE),
+ 'test only works from source build directory')
+ def test_real_grammar_and_keyword_file(self):
+ self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE)
+ self.addCleanup(support.unlink, TEST_PY_FILE)
+ self.assertFalse(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE))
+ self.assertEqual((0, b''), self._generate_keywords(GRAMMAR_FILE,
+ TEST_PY_FILE))
+ self.assertTrue(filecmp.cmp(KEYWORD_FILE, TEST_PY_FILE))
+
+ def test_grammar(self):
+ self._copy_file_without_generated_keywords(KEYWORD_FILE, TEST_PY_FILE)
+ self.addCleanup(support.unlink, TEST_PY_FILE)
+ with open(GRAMMAR_TEST_FILE, 'w') as fp:
+ # Some of these are probably implementation accidents.
+ fp.writelines(textwrap.dedent("""\
+ {2, 1},
+ {11, "encoding_decl", 0, 2, states_79,
+ "\000\000\040\000\000\000\000\000\000\000\000\000"
+ "\000\000\000\000\000\000\000\000\000"},
+ {1, "jello"},
+ {326, 0},
+ {1, "turnip"},
+ \t{1, "This one is tab indented"
+ {278, 0},
+ {1, "crazy but legal"
+ "also legal" {1, "
+ {1, "continue"},
+ {1, "lemon"},
+ {1, "tomato"},
+ {1, "wigii"},
+ {1, 'no good'}
+ {283, 0},
+ {1, "too many spaces"}"""))
+ self.addCleanup(support.unlink, GRAMMAR_TEST_FILE)
+ self._generate_keywords(GRAMMAR_TEST_FILE, TEST_PY_FILE)
+ expected = [
+ " 'This one is tab indented',",
+ " 'also legal',",
+ " 'continue',",
+ " 'crazy but legal',",
+ " 'jello',",
+ " 'lemon',",
+ " 'tomato',",
+ " 'turnip',",
+ " 'wigii',",
+ ]
+ with open(TEST_PY_FILE) as fp:
+ lines = fp.read().splitlines()
+ start = lines.index("#--start keywords--") + 1
+ end = lines.index("#--end keywords--")
+ actual = lines[start:end]
+ self.assertEqual(actual, expected)
+
+ def test_empty_grammar_results_in_no_keywords(self):
+ self._copy_file_without_generated_keywords(KEYWORD_FILE,
+ PY_FILE_WITHOUT_KEYWORDS)
+ self.addCleanup(support.unlink, PY_FILE_WITHOUT_KEYWORDS)
+ shutil.copyfile(KEYWORD_FILE, TEST_PY_FILE)
+ self.addCleanup(support.unlink, TEST_PY_FILE)
+ self.assertEqual((0, b''), self._generate_keywords(os.devnull,
+ TEST_PY_FILE))
+ self.assertTrue(filecmp.cmp(TEST_PY_FILE, PY_FILE_WITHOUT_KEYWORDS))
+
+ def test_keywords_py_without_markers_produces_error(self):
+ rc, stderr = self._generate_keywords(os.devnull, os.devnull)
+ self.assertNotEqual(rc, 0)
+ self.assertRegex(stderr, b'does not contain format markers')
+
+ def test_missing_grammar_file_produces_error(self):
+ rc, stderr = self._generate_keywords(NONEXISTENT_FILE, KEYWORD_FILE)
+ self.assertNotEqual(rc, 0)
+ self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode())
+
+ def test_missing_keywords_py_file_produces_error(self):
+ rc, stderr = self._generate_keywords(os.devnull, NONEXISTENT_FILE)
+ self.assertNotEqual(rc, 0)
+ self.assertRegex(stderr, b'(?ms)' + NONEXISTENT_FILE.encode())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py
index 6c2ff00..7f315d4 100644
--- a/Lib/test/test_keywordonlyarg.py
+++ b/Lib/test/test_keywordonlyarg.py
@@ -174,6 +174,18 @@ class KeywordOnlyArgTestCase(unittest.TestCase):
return __a
self.assertEqual(X().f(), 42)
+ def test_default_evaluation_order(self):
+ # See issue 16967
+ a = 42
+ with self.assertRaises(NameError) as err:
+ def f(v=a, x=b, *, y=c, z=d):
+ pass
+ self.assertEqual(str(err.exception), "name 'b' is not defined")
+ with self.assertRaises(NameError) as err:
+ f = lambda v=a, x=b, *, y=c, z=d: None
+ self.assertEqual(str(err.exception), "name 'b' is not defined")
+
+
def test_main():
run_unittest(KeywordOnlyArgTestCase)
diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py
index e5e6058..bafdeba 100644
--- a/Lib/test/test_kqueue.py
+++ b/Lib/test/test_kqueue.py
@@ -94,7 +94,7 @@ class TestKQueue(unittest.TestCase):
client.setblocking(False)
try:
client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
- except socket.error as e:
+ except OSError as e:
self.assertEqual(e.args[0], errno.EINPROGRESS)
else:
#raise AssertionError("Connect should have raised EINPROGRESS")
@@ -185,6 +185,33 @@ class TestKQueue(unittest.TestCase):
b.close()
kq.close()
+ def test_close(self):
+ open_file = open(__file__, "rb")
+ self.addCleanup(open_file.close)
+ fd = open_file.fileno()
+ kqueue = select.kqueue()
+
+ # test fileno() method and closed attribute
+ self.assertIsInstance(kqueue.fileno(), int)
+ self.assertFalse(kqueue.closed)
+
+ # test close()
+ kqueue.close()
+ self.assertTrue(kqueue.closed)
+ self.assertRaises(ValueError, kqueue.fileno)
+
+ # close() can be called more than once
+ kqueue.close()
+
+ # operations must fail with ValueError("I/O operation on closed ...")
+ self.assertRaises(ValueError, kqueue.control, None, 4)
+
+ def test_fd_non_inheritable(self):
+ kqueue = select.kqueue()
+ self.addCleanup(kqueue.close)
+ self.assertEqual(os.get_inheritable(kqueue.fileno()), False)
+
+
def test_main():
support.run_unittest(TestKQueue)
diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py
index 63ee697..5b276e7 100644
--- a/Lib/test/test_largefile.py
+++ b/Lib/test/test_largefile.py
@@ -159,7 +159,7 @@ def setUpModule():
# Seeking is not enough of a test: you must write and flush, too!
f.write(b'x')
f.flush()
- except (IOError, OverflowError):
+ except (OSError, OverflowError):
raise unittest.SkipTest("filesystem does not have "
"largefile support")
finally:
diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py
index 76347b7..ab990ae 100644
--- a/Lib/test/test_locale.py
+++ b/Lib/test/test_locale.py
@@ -371,7 +371,8 @@ class NormalizeTest(unittest.TestCase):
def test_locale_alias(self):
for localename, alias in locale.locale_alias.items():
- self.check(localename, alias)
+ with self.subTest(locale=(localename, alias)):
+ self.check(localename, alias)
def test_empty(self):
self.check('', '')
@@ -383,6 +384,7 @@ class NormalizeTest(unittest.TestCase):
def test_english(self):
self.check('en', 'en_US.ISO8859-1')
self.check('EN', 'en_US.ISO8859-1')
+ self.check('en.iso88591', 'en_US.ISO8859-1')
self.check('en_US', 'en_US.ISO8859-1')
self.check('en_us', 'en_US.ISO8859-1')
self.check('en_GB', 'en_GB.ISO8859-1')
@@ -391,7 +393,10 @@ class NormalizeTest(unittest.TestCase):
self.check('en_US:UTF-8', 'en_US.UTF-8')
self.check('en_US.ISO8859-1', 'en_US.ISO8859-1')
self.check('en_US.US-ASCII', 'en_US.ISO8859-1')
+ self.check('en_US.88591', 'en_US.ISO8859-1')
+ self.check('en_US.885915', 'en_US.ISO8859-15')
self.check('english', 'en_EN.ISO8859-1')
+ self.check('english_uk.ascii', 'en_GB.ISO8859-1')
def test_hyphenated_encoding(self):
self.check('az_AZ.iso88599e', 'az_AZ.ISO8859-9E')
@@ -411,10 +416,12 @@ class NormalizeTest(unittest.TestCase):
def test_euro_modifier(self):
self.check('de_DE@euro', 'de_DE.ISO8859-15')
self.check('en_US.ISO8859-15@euro', 'en_US.ISO8859-15')
+ self.check('de_DE.utf8@euro', 'de_DE.UTF-8')
def test_latin_modifier(self):
self.check('be_BY.UTF-8@latin', 'be_BY.UTF-8@latin')
self.check('sr_RS.UTF-8@latin', 'sr_RS.UTF-8@latin')
+ self.check('sr_RS.UTF-8@latn', 'sr_RS.UTF-8@latin')
def test_valencia_modifier(self):
self.check('ca_ES.UTF-8@valencia', 'ca_ES.UTF-8@valencia')
@@ -435,6 +442,39 @@ class NormalizeTest(unittest.TestCase):
self.check('sd_IN', 'sd_IN.UTF-8')
self.check('sd', 'sd_IN.UTF-8')
+ def test_euc_encoding(self):
+ self.check('ja_jp.euc', 'ja_JP.eucJP')
+ self.check('ja_jp.eucjp', 'ja_JP.eucJP')
+ self.check('ko_kr.euc', 'ko_KR.eucKR')
+ self.check('ko_kr.euckr', 'ko_KR.eucKR')
+ self.check('zh_cn.euc', 'zh_CN.eucCN')
+ self.check('zh_tw.euc', 'zh_TW.eucTW')
+ self.check('zh_tw.euctw', 'zh_TW.eucTW')
+
+ def test_japanese(self):
+ self.check('ja', 'ja_JP.eucJP')
+ self.check('ja.jis', 'ja_JP.JIS7')
+ self.check('ja.sjis', 'ja_JP.SJIS')
+ self.check('ja_jp', 'ja_JP.eucJP')
+ self.check('ja_jp.ajec', 'ja_JP.eucJP')
+ self.check('ja_jp.euc', 'ja_JP.eucJP')
+ self.check('ja_jp.eucjp', 'ja_JP.eucJP')
+ self.check('ja_jp.iso-2022-jp', 'ja_JP.JIS7')
+ self.check('ja_jp.iso2022jp', 'ja_JP.JIS7')
+ self.check('ja_jp.jis', 'ja_JP.JIS7')
+ self.check('ja_jp.jis7', 'ja_JP.JIS7')
+ self.check('ja_jp.mscode', 'ja_JP.SJIS')
+ self.check('ja_jp.pck', 'ja_JP.SJIS')
+ self.check('ja_jp.sjis', 'ja_JP.SJIS')
+ self.check('ja_jp.ujis', 'ja_JP.eucJP')
+ self.check('ja_jp.utf8', 'ja_JP.UTF-8')
+ self.check('japan', 'ja_JP.eucJP')
+ self.check('japanese', 'ja_JP.eucJP')
+ self.check('japanese-euc', 'ja_JP.eucJP')
+ self.check('japanese.euc', 'ja_JP.eucJP')
+ self.check('japanese.sjis', 'ja_JP.SJIS')
+ self.check('jp_jp', 'ja_JP.eucJP')
+
class TestMiscellaneous(unittest.TestCase):
def test_getpreferredencoding(self):
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
index f34172a..e72b036 100644
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -24,6 +24,7 @@ import logging.handlers
import logging.config
import codecs
+import configparser
import datetime
import pickle
import io
@@ -38,6 +39,7 @@ import socket
import struct
import sys
import tempfile
+from test.script_helper import assert_python_ok
from test.support import (captured_stdout, run_with_locale, run_unittest,
patch, requires_zlib, TestHandler, Matcher)
import textwrap
@@ -56,7 +58,9 @@ try:
import smtpd
from urllib.parse import urlparse, parse_qs
from socketserver import (ThreadingUDPServer, DatagramRequestHandler,
- ThreadingTCPServer, StreamRequestHandler)
+ ThreadingTCPServer, StreamRequestHandler,
+ ThreadingUnixStreamServer,
+ ThreadingUnixDatagramServer)
except ImportError:
threading = None
try:
@@ -73,13 +77,12 @@ try:
except ImportError:
pass
-
class BaseTest(unittest.TestCase):
"""Base class for logging tests."""
log_format = "%(name)s -> %(levelname)s: %(message)s"
- expected_log_pat = r"^([\w.]+) -> ([\w]+): ([\d]+)$"
+ expected_log_pat = r"^([\w.]+) -> (\w+): (\d+)$"
message_num = 0
def setUp(self):
@@ -91,7 +94,8 @@ class BaseTest(unittest.TestCase):
self.saved_handlers = logging._handlers.copy()
self.saved_handler_list = logging._handlerList[:]
self.saved_loggers = saved_loggers = logger_dict.copy()
- self.saved_level_names = logging._levelNames.copy()
+ self.saved_name_to_level = logging._nameToLevel.copy()
+ self.saved_level_to_name = logging._levelToName.copy()
self.logger_states = logger_states = {}
for name in saved_loggers:
logger_states[name] = getattr(saved_loggers[name],
@@ -133,8 +137,10 @@ class BaseTest(unittest.TestCase):
self.root_logger.setLevel(self.original_logging_level)
logging._acquireLock()
try:
- logging._levelNames.clear()
- logging._levelNames.update(self.saved_level_names)
+ logging._levelToName.clear()
+ logging._levelToName.update(self.saved_level_to_name)
+ logging._nameToLevel.clear()
+ logging._nameToLevel.update(self.saved_name_to_level)
logging._handlers.clear()
logging._handlers.update(self.saved_handlers)
logging._handlerList[:] = self.saved_handler_list
@@ -148,12 +154,12 @@ class BaseTest(unittest.TestCase):
finally:
logging._releaseLock()
- def assert_log_lines(self, expected_values, stream=None):
+ def assert_log_lines(self, expected_values, stream=None, pat=None):
"""Match the collected log lines against the regular expression
self.expected_log_pat, and compare the extracted group values to
the expected_values list of tuples."""
stream = stream or self.stream
- pat = re.compile(self.expected_log_pat)
+ pat = re.compile(pat or self.expected_log_pat)
actual_lines = stream.getvalue().splitlines()
self.assertEqual(len(actual_lines), len(expected_values))
for actual, expected in zip(actual_lines, expected_values):
@@ -428,7 +434,7 @@ class CustomLevelsAndFiltersTest(BaseTest):
"""Test various filtering possibilities with custom logging levels."""
# Skip the logger name group.
- expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$"
+ expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$"
def setUp(self):
BaseTest.setUp(self)
@@ -558,7 +564,7 @@ class HandlerTest(BaseTest):
self.assertEqual(h.facility, h.LOG_USER)
self.assertTrue(h.unixsocket)
h.close()
- except socket.error: # syslogd might not be available
+ except OSError: # syslogd might not be available
pass
for method in ('GET', 'POST', 'PUT'):
if method == 'PUT':
@@ -583,6 +589,7 @@ class HandlerTest(BaseTest):
for _ in range(tries):
try:
os.unlink(fname)
+ self.deletion_time = time.time()
except OSError:
pass
time.sleep(0.004 * random.randint(0, 4))
@@ -590,6 +597,9 @@ class HandlerTest(BaseTest):
del_count = 500
log_count = 500
+ self.handle_time = None
+ self.deletion_time = None
+
for delay in (False, True):
fd, fn = tempfile.mkstemp('.log', 'test_logging-3-')
os.close(fd)
@@ -603,7 +613,14 @@ class HandlerTest(BaseTest):
for _ in range(log_count):
time.sleep(0.005)
r = logging.makeLogRecord({'msg': 'testing' })
- h.handle(r)
+ try:
+ self.handle_time = time.time()
+ h.handle(r)
+ except Exception:
+ print('Deleted at %s, '
+ 'opened at %s' % (self.deletion_time,
+ self.handle_time))
+ raise
finally:
remover.join()
h.close()
@@ -645,41 +662,6 @@ class StreamHandlerTest(BaseTest):
# -- if it proves to be of wider utility than just test_logging
if threading:
- class TestSMTPChannel(smtpd.SMTPChannel):
- """
- This derived class has had to be created because smtpd does not
- support use of custom channel maps, although they are allowed by
- asyncore's design. Issue #11959 has been raised to address this,
- and if resolved satisfactorily, some of this code can be removed.
- """
- def __init__(self, server, conn, addr, sockmap):
- asynchat.async_chat.__init__(self, conn, sockmap)
- self.smtp_server = server
- self.conn = conn
- self.addr = addr
- self.data_size_limit = None
- self.received_lines = []
- self.smtp_state = self.COMMAND
- self.seen_greeting = ''
- self.mailfrom = None
- self.rcpttos = []
- self.received_data = ''
- self.fqdn = socket.getfqdn()
- self.num_bytes = 0
- try:
- self.peer = conn.getpeername()
- except socket.error as err:
- # a race condition may occur if the other end is closing
- # before we can get the peername
- self.close()
- if err.args[0] != errno.ENOTCONN:
- raise
- return
- self.push('220 %s %s' % (self.fqdn, smtpd.__version__))
- self.set_terminator(b'\r\n')
- self.extended_smtp = False
-
-
class TestSMTPServer(smtpd.SMTPServer):
"""
This class implements a test SMTP server.
@@ -700,37 +682,14 @@ if threading:
:func:`asyncore.loop`. This avoids changing the
:mod:`asyncore` module's global state.
"""
- channel_class = TestSMTPChannel
def __init__(self, addr, handler, poll_interval, sockmap):
- self._localaddr = addr
- self._remoteaddr = None
- self.data_size_limit = None
- self.sockmap = sockmap
- asyncore.dispatcher.__init__(self, map=sockmap)
- try:
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.setblocking(0)
- self.set_socket(sock, map=sockmap)
- # try to re-use a server port if possible
- self.set_reuse_addr()
- self.bind(addr)
- self.port = sock.getsockname()[1]
- self.listen(5)
- except:
- self.close()
- raise
+ smtpd.SMTPServer.__init__(self, addr, None, map=sockmap)
+ self.port = self.socket.getsockname()[1]
self._handler = handler
self._thread = None
self.poll_interval = poll_interval
- def handle_accepted(self, conn, addr):
- """
- Redefined only because the base class does not pass in a
- map, forcing use of a global in :mod:`asyncore`.
- """
- channel = self.channel_class(self, conn, addr, self.sockmap)
-
def process_message(self, peer, mailfrom, rcpttos, data):
"""
Delegates to the handler passed in to the server's constructor.
@@ -761,8 +720,8 @@ if threading:
:func:`asyncore.loop`.
"""
try:
- asyncore.loop(poll_interval, map=self.sockmap)
- except select.error:
+ asyncore.loop(poll_interval, map=self._map)
+ except OSError:
# On FreeBSD 8, closing the server repeatably
# raises this error. We swallow it if the
# server has been closed.
@@ -869,7 +828,7 @@ if threading:
sock, addr = self.socket.accept()
if self.sslctx:
sock = self.sslctx.wrap_socket(sock, server_side=True)
- except socket.error as e:
+ except OSError as e:
# socket errors are silenced by the caller, print them here
sys.stderr.write("Got an error:\n%s\n" % e)
raise
@@ -906,6 +865,9 @@ if threading:
super(TestTCPServer, self).server_bind()
self.port = self.socket.getsockname()[1]
+ class TestUnixStreamServer(TestTCPServer):
+ address_family = socket.AF_UNIX
+
class TestUDPServer(ControlMixin, ThreadingUDPServer):
"""
A UDP server which is controllable using :class:`ControlMixin`.
@@ -935,7 +897,7 @@ if threading:
if data:
try:
super(DelegatingUDPRequestHandler, self).finish()
- except socket.error:
+ except OSError:
if not self.server._closed:
raise
@@ -953,6 +915,9 @@ if threading:
super(TestUDPServer, self).server_close()
self._closed = True
+ class TestUnixDatagramServer(TestUDPServer):
+ address_family = socket.AF_UNIX
+
# - end of server_helper section
@unittest.skipUnless(threading, 'Threading required for this test.')
@@ -991,7 +956,7 @@ class MemoryHandlerTest(BaseTest):
"""Tests for the MemoryHandler."""
# Do not bother with a logger name group.
- expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$"
+ expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$"
def setUp(self):
BaseTest.setUp(self)
@@ -1044,7 +1009,7 @@ class ConfigFileTest(BaseTest):
"""Reading logging config from a .ini-style config file."""
- expected_log_pat = r"^([\w]+) \+\+ ([\w]+)$"
+ expected_log_pat = r"^(\w+) \+\+ (\w+)$"
# config0 is a standard configuration.
config0 = """
@@ -1292,6 +1257,24 @@ class ConfigFileTest(BaseTest):
# Original logger output is empty.
self.assert_log_lines([])
+ def test_config0_using_cp_ok(self):
+ # A simple config file which overrides the default settings.
+ with captured_stdout() as output:
+ file = io.StringIO(textwrap.dedent(self.config0))
+ cp = configparser.ConfigParser()
+ cp.read_file(file)
+ logging.config.fileConfig(cp)
+ logger = logging.getLogger()
+ # Won't output anything
+ logger.info(self.next_message())
+ # Outputs a message
+ logger.error(self.next_message())
+ self.assert_log_lines([
+ ('ERROR', '2'),
+ ], stream=output)
+ # Original logger output is empty.
+ self.assert_log_lines([])
+
def test_config1_ok(self, config=config1):
# A config file defining a sub-parser as well.
with captured_stdout() as output:
@@ -1381,7 +1364,7 @@ class ConfigFileTest(BaseTest):
def test_logger_disabling(self):
self.apply_config(self.disable_test)
- logger = logging.getLogger('foo')
+ logger = logging.getLogger('some_pristine_logger')
self.assertFalse(logger.disabled)
self.apply_config(self.disable_test)
self.assertTrue(logger.disabled)
@@ -1394,17 +1377,23 @@ class SocketHandlerTest(BaseTest):
"""Test for SocketHandler objects."""
+ if threading:
+ server_class = TestTCPServer
+ address = ('localhost', 0)
+
def setUp(self):
"""Set up a TCP server to receive log messages, and a SocketHandler
pointing to that server's address and port."""
BaseTest.setUp(self)
- addr = ('localhost', 0)
- self.server = server = TestTCPServer(addr, self.handle_socket,
- 0.01)
+ self.server = server = self.server_class(self.address,
+ self.handle_socket, 0.01)
server.start()
server.ready.wait()
- self.sock_hdlr = logging.handlers.SocketHandler('localhost',
- server.port)
+ hcls = logging.handlers.SocketHandler
+ if isinstance(server.server_address, tuple):
+ self.sock_hdlr = hcls('localhost', server.port)
+ else:
+ self.sock_hdlr = hcls(server.server_address, None)
self.log_output = ''
self.root_logger.removeHandler(self.root_logger.handlers[0])
self.root_logger.addHandler(self.sock_hdlr)
@@ -1444,35 +1433,69 @@ class SocketHandlerTest(BaseTest):
self.assertEqual(self.log_output, "spam\neggs\n")
def test_noserver(self):
+ # Avoid timing-related failures due to SocketHandler's own hard-wired
+ # one-second timeout on socket.create_connection() (issue #16264).
+ self.sock_hdlr.retryStart = 2.5
# Kill the server
self.server.stop(2.0)
- #The logging call should try to connect, which should fail
+ # The logging call should try to connect, which should fail
try:
raise RuntimeError('Deliberate mistake')
except RuntimeError:
self.root_logger.exception('Never sent')
self.root_logger.error('Never sent, either')
now = time.time()
- self.assertTrue(self.sock_hdlr.retryTime > now)
+ self.assertGreater(self.sock_hdlr.retryTime, now)
time.sleep(self.sock_hdlr.retryTime - now + 0.001)
self.root_logger.error('Nor this')
+def _get_temp_domain_socket():
+ fd, fn = tempfile.mkstemp(prefix='test_logging_', suffix='.sock')
+ os.close(fd)
+ # just need a name - file can't be present, or we'll get an
+ # 'address already in use' error.
+ os.remove(fn)
+ return fn
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class UnixSocketHandlerTest(SocketHandlerTest):
+
+ """Test for SocketHandler with unix sockets."""
+
+ if threading:
+ server_class = TestUnixStreamServer
+
+ def setUp(self):
+ # override the definition in the base class
+ self.address = _get_temp_domain_socket()
+ SocketHandlerTest.setUp(self)
+
+ def tearDown(self):
+ SocketHandlerTest.tearDown(self)
+ os.remove(self.address)
@unittest.skipUnless(threading, 'Threading required for this test.')
class DatagramHandlerTest(BaseTest):
"""Test for DatagramHandler."""
+ if threading:
+ server_class = TestUDPServer
+ address = ('localhost', 0)
+
def setUp(self):
"""Set up a UDP server to receive log messages, and a DatagramHandler
pointing to that server's address and port."""
BaseTest.setUp(self)
- addr = ('localhost', 0)
- self.server = server = TestUDPServer(addr, self.handle_datagram, 0.01)
+ self.server = server = self.server_class(self.address,
+ self.handle_datagram, 0.01)
server.start()
server.ready.wait()
- self.sock_hdlr = logging.handlers.DatagramHandler('localhost',
- server.port)
+ hcls = logging.handlers.DatagramHandler
+ if isinstance(server.server_address, tuple):
+ self.sock_hdlr = hcls('localhost', server.port)
+ else:
+ self.sock_hdlr = hcls(server.server_address, None)
self.log_output = ''
self.root_logger.removeHandler(self.root_logger.handlers[0])
self.root_logger.addHandler(self.sock_hdlr)
@@ -1507,21 +1530,44 @@ class DatagramHandlerTest(BaseTest):
@unittest.skipUnless(threading, 'Threading required for this test.')
+class UnixDatagramHandlerTest(DatagramHandlerTest):
+
+ """Test for DatagramHandler using Unix sockets."""
+
+ if threading:
+ server_class = TestUnixDatagramServer
+
+ def setUp(self):
+ # override the definition in the base class
+ self.address = _get_temp_domain_socket()
+ DatagramHandlerTest.setUp(self)
+
+ def tearDown(self):
+ DatagramHandlerTest.tearDown(self)
+ os.remove(self.address)
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
class SysLogHandlerTest(BaseTest):
"""Test for SysLogHandler using UDP."""
+ if threading:
+ server_class = TestUDPServer
+ address = ('localhost', 0)
+
def setUp(self):
"""Set up a UDP server to receive log messages, and a SysLogHandler
pointing to that server's address and port."""
BaseTest.setUp(self)
- addr = ('localhost', 0)
- self.server = server = TestUDPServer(addr, self.handle_datagram,
- 0.01)
+ self.server = server = self.server_class(self.address,
+ self.handle_datagram, 0.01)
server.start()
server.ready.wait()
- self.sl_hdlr = logging.handlers.SysLogHandler(('localhost',
- server.port))
+ hcls = logging.handlers.SysLogHandler
+ if isinstance(server.server_address, tuple):
+ self.sl_hdlr = hcls(('localhost', server.port))
+ else:
+ self.sl_hdlr = hcls(server.server_address)
self.log_output = ''
self.root_logger.removeHandler(self.root_logger.handlers[0])
self.root_logger.addHandler(self.sl_hdlr)
@@ -1559,6 +1605,23 @@ class SysLogHandlerTest(BaseTest):
@unittest.skipUnless(threading, 'Threading required for this test.')
+class UnixSysLogHandlerTest(SysLogHandlerTest):
+
+ """Test for SysLogHandler with Unix sockets."""
+
+ if threading:
+ server_class = TestUnixDatagramServer
+
+ def setUp(self):
+ # override the definition in the base class
+ self.address = _get_temp_domain_socket()
+ SysLogHandlerTest.setUp(self)
+
+ def tearDown(self):
+ SysLogHandlerTest.tearDown(self)
+ os.remove(self.address)
+
+@unittest.skipUnless(threading, 'Threading required for this test.')
class HTTPHandlerTest(BaseTest):
"""Test for HTTPHandler."""
@@ -1779,7 +1842,7 @@ class WarningsTest(BaseTest):
logger.removeHandler(h)
s = stream.getvalue()
h.close()
- self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0)
+ self.assertGreater(s.find("UserWarning: I'm warning you...\n"), 0)
#See if an explicit file uses the original implementation
a_file = io.StringIO()
@@ -1817,7 +1880,7 @@ class ConfigDictTest(BaseTest):
"""Reading logging config from a dictionary."""
- expected_log_pat = r"^([\w]+) \+\+ ([\w]+)$"
+ expected_log_pat = r"^(\w+) \+\+ (\w+)$"
# config0 is a standard configuration.
config0 = {
@@ -2389,6 +2452,32 @@ class ConfigDictTest(BaseTest):
},
}
+ # As config0, but with properties
+ config14 = {
+ 'version': 1,
+ 'formatters': {
+ 'form1' : {
+ 'format' : '%(levelname)s ++ %(message)s',
+ },
+ },
+ 'handlers' : {
+ 'hand1' : {
+ 'class' : 'logging.StreamHandler',
+ 'formatter' : 'form1',
+ 'level' : 'NOTSET',
+ 'stream' : 'ext://sys.stdout',
+ '.': {
+ 'foo': 'bar',
+ 'terminator': '!\n',
+ }
+ },
+ },
+ 'root' : {
+ 'level' : 'WARNING',
+ 'handlers' : ['hand1'],
+ },
+ }
+
out_of_order = {
"version": 1,
"formatters": {
@@ -2656,11 +2745,20 @@ class ConfigDictTest(BaseTest):
def test_config13_failure(self):
self.assertRaises(Exception, self.apply_config, self.config13)
+ def test_config14_ok(self):
+ with captured_stdout() as output:
+ self.apply_config(self.config14)
+ h = logging._handlers['hand1']
+ self.assertEqual(h.foo, 'bar')
+ self.assertEqual(h.terminator, '!\n')
+ logging.warning('Exclamation')
+ self.assertTrue(output.getvalue().endswith('Exclamation!\n'))
+
@unittest.skipUnless(threading, 'listen() needs threading to work')
- def setup_via_listener(self, text):
+ def setup_via_listener(self, text, verify=None):
text = text.encode("utf-8")
# Ask for a randomly assigned port (by using port 0)
- t = logging.config.listen(0)
+ t = logging.config.listen(0, verify)
t.start()
t.ready.wait()
# Now get the port allocated
@@ -2720,6 +2818,69 @@ class ConfigDictTest(BaseTest):
# Original logger output is empty.
self.assert_log_lines([])
+ @unittest.skipUnless(threading, 'Threading required for this test.')
+ def test_listen_verify(self):
+
+ def verify_fail(stuff):
+ return None
+
+ def verify_reverse(stuff):
+ return stuff[::-1]
+
+ logger = logging.getLogger("compiler.parser")
+ to_send = textwrap.dedent(ConfigFileTest.config1)
+ # First, specify a verification function that will fail.
+ # We expect to see no output, since our configuration
+ # never took effect.
+ with captured_stdout() as output:
+ self.setup_via_listener(to_send, verify_fail)
+ # Both will output a message
+ logger.info(self.next_message())
+ logger.error(self.next_message())
+ self.assert_log_lines([], stream=output)
+ # Original logger output has the stuff we logged.
+ self.assert_log_lines([
+ ('INFO', '1'),
+ ('ERROR', '2'),
+ ], pat=r"^[\w.]+ -> (\w+): (\d+)$")
+
+ # Now, perform no verification. Our configuration
+ # should take effect.
+
+ with captured_stdout() as output:
+ self.setup_via_listener(to_send) # no verify callable specified
+ logger = logging.getLogger("compiler.parser")
+ # Both will output a message
+ logger.info(self.next_message())
+ logger.error(self.next_message())
+ self.assert_log_lines([
+ ('INFO', '3'),
+ ('ERROR', '4'),
+ ], stream=output)
+ # Original logger output still has the stuff we logged before.
+ self.assert_log_lines([
+ ('INFO', '1'),
+ ('ERROR', '2'),
+ ], pat=r"^[\w.]+ -> (\w+): (\d+)$")
+
+ # Now, perform verification which transforms the bytes.
+
+ with captured_stdout() as output:
+ self.setup_via_listener(to_send[::-1], verify_reverse)
+ logger = logging.getLogger("compiler.parser")
+ # Both will output a message
+ logger.info(self.next_message())
+ logger.error(self.next_message())
+ self.assert_log_lines([
+ ('INFO', '5'),
+ ('ERROR', '6'),
+ ], stream=output)
+ # Original logger output still has the stuff we logged before.
+ self.assert_log_lines([
+ ('INFO', '1'),
+ ('ERROR', '2'),
+ ], pat=r"^[\w.]+ -> (\w+): (\d+)$")
+
def test_out_of_order(self):
self.apply_config(self.out_of_order)
handler = logging.getLogger('mymodule').handlers[0]
@@ -2779,14 +2940,14 @@ class ChildLoggerTest(BaseTest):
l2 = logging.getLogger('def.ghi')
c1 = r.getChild('xyz')
c2 = r.getChild('uvw.xyz')
- self.assertTrue(c1 is logging.getLogger('xyz'))
- self.assertTrue(c2 is logging.getLogger('uvw.xyz'))
+ self.assertIs(c1, logging.getLogger('xyz'))
+ self.assertIs(c2, logging.getLogger('uvw.xyz'))
c1 = l1.getChild('def')
c2 = c1.getChild('ghi')
c3 = l1.getChild('def.ghi')
- self.assertTrue(c1 is logging.getLogger('abc.def'))
- self.assertTrue(c2 is logging.getLogger('abc.def.ghi'))
- self.assertTrue(c2 is c3)
+ self.assertIs(c1, logging.getLogger('abc.def'))
+ self.assertIs(c2, logging.getLogger('abc.def.ghi'))
+ self.assertIs(c2, c3)
class DerivedLogRecord(logging.LogRecord):
@@ -2829,7 +2990,7 @@ class LogRecordFactoryTest(BaseTest):
class QueueHandlerTest(BaseTest):
# Do not bother with a logger name group.
- expected_log_pat = r"^[\w.]+ -> ([\w]+): ([\d]+)$"
+ expected_log_pat = r"^[\w.]+ -> (\w+): (\d+)$"
def setUp(self):
BaseTest.setUp(self)
@@ -3123,13 +3284,13 @@ class ShutdownTest(BaseTest):
self.assertEqual('0 - release', self.called[-1])
def test_with_ioerror_in_acquire(self):
- self._test_with_failure_in_method('acquire', IOError)
+ self._test_with_failure_in_method('acquire', OSError)
def test_with_ioerror_in_flush(self):
- self._test_with_failure_in_method('flush', IOError)
+ self._test_with_failure_in_method('flush', OSError)
def test_with_ioerror_in_close(self):
- self._test_with_failure_in_method('close', IOError)
+ self._test_with_failure_in_method('close', OSError)
def test_with_valueerror_in_acquire(self):
self._test_with_failure_in_method('acquire', ValueError)
@@ -3235,6 +3396,25 @@ class ModuleLevelMiscTest(BaseTest):
logging.setLoggerClass(logging.Logger)
self.assertEqual(logging.getLoggerClass(), logging.Logger)
+ def test_logging_at_shutdown(self):
+ # Issue #20037
+ code = """if 1:
+ import logging
+
+ class A:
+ def __del__(self):
+ try:
+ raise ValueError("some error")
+ except Exception:
+ logging.exception("exception in __del__")
+
+ a = A()"""
+ rc, out, err = assert_python_ok("-c", code)
+ err = err.decode()
+ self.assertIn("exception in __del__", err)
+ self.assertIn("ValueError: some error", err)
+
+
class LogRecordTest(BaseTest):
def test_str_rep(self):
r = logging.makeLogRecord({})
@@ -3352,6 +3532,12 @@ class BasicConfigTest(unittest.TestCase):
"ERROR:root:Log an error")
def test_filename(self):
+
+ def cleanup(h1, h2, fn):
+ h1.close()
+ h2.close()
+ os.remove(fn)
+
logging.basicConfig(filename='test.log')
self.assertEqual(len(logging.root.handlers), 1)
@@ -3359,17 +3545,23 @@ class BasicConfigTest(unittest.TestCase):
self.assertIsInstance(handler, logging.FileHandler)
expected = logging.FileHandler('test.log', 'a')
- self.addCleanup(expected.close)
self.assertEqual(handler.stream.mode, expected.stream.mode)
self.assertEqual(handler.stream.name, expected.stream.name)
+ self.addCleanup(cleanup, handler, expected, 'test.log')
def test_filemode(self):
+
+ def cleanup(h1, h2, fn):
+ h1.close()
+ h2.close()
+ os.remove(fn)
+
logging.basicConfig(filename='test.log', filemode='wb')
handler = logging.root.handlers[0]
expected = logging.FileHandler('test.log', 'wb')
- self.addCleanup(expected.close)
self.assertEqual(handler.stream.mode, expected.stream.mode)
+ self.addCleanup(cleanup, handler, expected, 'test.log')
def test_stream(self):
stream = io.StringIO()
@@ -3825,6 +4017,63 @@ class TimedRotatingFileHandlerTest(BaseFileTest):
assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler,
self.fn, 'W7', delay=True)
+ def test_compute_rollover_daily_attime(self):
+ currentTime = 0
+ atTime = datetime.time(12, 0, 0)
+ rh = logging.handlers.TimedRotatingFileHandler(
+ self.fn, when='MIDNIGHT', interval=1, backupCount=0, utc=True,
+ atTime=atTime)
+ try:
+ actual = rh.computeRollover(currentTime)
+ self.assertEqual(actual, currentTime + 12 * 60 * 60)
+
+ actual = rh.computeRollover(currentTime + 13 * 60 * 60)
+ self.assertEqual(actual, currentTime + 36 * 60 * 60)
+ finally:
+ rh.close()
+
+ #@unittest.skipIf(True, 'Temporarily skipped while failures investigated.')
+ def test_compute_rollover_weekly_attime(self):
+ currentTime = int(time.time())
+ today = currentTime - currentTime % 86400
+
+ atTime = datetime.time(12, 0, 0)
+
+ wday = time.gmtime(today).tm_wday
+ for day in range(7):
+ rh = logging.handlers.TimedRotatingFileHandler(
+ self.fn, when='W%d' % day, interval=1, backupCount=0, utc=True,
+ atTime=atTime)
+ try:
+ if wday > day:
+ # The rollover day has already passed this week, so we
+ # go over into next week
+ expected = (7 - wday + day)
+ else:
+ expected = (day - wday)
+ # At this point expected is in days from now, convert to seconds
+ expected *= 24 * 60 * 60
+ # Add in the rollover time
+ expected += 12 * 60 * 60
+ # Add in adjustment for today
+ expected += today
+ actual = rh.computeRollover(today)
+ if actual != expected:
+ print('failed in timezone: %d' % time.timezone)
+ print('local vars: %s' % locals())
+ self.assertEqual(actual, expected)
+ if day == wday:
+ # goes into following week
+ expected += 7 * 24 * 60 * 60
+ actual = rh.computeRollover(today + 13 * 60 * 60)
+ if actual != expected:
+ print('failed in timezone: %d' % time.timezone)
+ print('local vars: %s' % locals())
+ self.assertEqual(actual, expected)
+ finally:
+ rh.close()
+
+
def secs(**kw):
return datetime.timedelta(**kw) // datetime.timedelta(seconds=1)
@@ -3883,7 +4132,7 @@ class NTEventLogHandlerTest(BaseTest):
h.handle(r)
h.close()
# Now see if the event is recorded
- self.assertTrue(num_recs < win32evtlog.GetNumberOfEventLogRecords(elh))
+ self.assertLess(num_recs, win32evtlog.GetNumberOfEventLogRecords(elh))
flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \
win32evtlog.EVENTLOG_SEQUENTIAL_READ
found = False
@@ -3916,7 +4165,8 @@ def test_main():
SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest,
LastResortTest, LogRecordTest, ExceptionTest,
SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest,
- TimedRotatingFileHandlerTest
+ TimedRotatingFileHandlerTest, UnixSocketHandlerTest,
+ UnixDatagramHandlerTest, UnixSysLogHandlerTest
)
if __name__ == "__main__":
diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py
index 3f9c1e2..13152ec 100644
--- a/Lib/test/test_long.py
+++ b/Lib/test/test_long.py
@@ -1079,7 +1079,7 @@ class LongTest(unittest.TestCase):
self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True)
self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False)
self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True)
- self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False),
+ self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False)
self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False)
self.assertEqual((0).to_bytes(0, 'big'), b'')
self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01')
diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py
index 160b0e8..3e73d7f 100644
--- a/Lib/test/test_lzma.py
+++ b/Lib/test/test_lzma.py
@@ -383,6 +383,8 @@ class FileTestCase(unittest.TestCase):
pass
with LZMAFile(BytesIO(), "w") as f:
pass
+ with LZMAFile(BytesIO(), "x") as f:
+ pass
with LZMAFile(BytesIO(), "a") as f:
pass
@@ -410,13 +412,29 @@ class FileTestCase(unittest.TestCase):
with LZMAFile(TESTFN, "ab"):
pass
+ def test_init_with_x_mode(self):
+ self.addCleanup(unlink, TESTFN)
+ for mode in ("x", "xb"):
+ unlink(TESTFN)
+ with LZMAFile(TESTFN, mode):
+ pass
+ with self.assertRaises(FileExistsError):
+ with LZMAFile(TESTFN, mode):
+ pass
+
def test_init_bad_mode(self):
with self.assertRaises(ValueError):
LZMAFile(BytesIO(COMPRESSED_XZ), (3, "x"))
with self.assertRaises(ValueError):
LZMAFile(BytesIO(COMPRESSED_XZ), "")
with self.assertRaises(ValueError):
- LZMAFile(BytesIO(COMPRESSED_XZ), "x")
+ LZMAFile(BytesIO(COMPRESSED_XZ), "xt")
+ with self.assertRaises(ValueError):
+ LZMAFile(BytesIO(COMPRESSED_XZ), "x+")
+ with self.assertRaises(ValueError):
+ LZMAFile(BytesIO(COMPRESSED_XZ), "rx")
+ with self.assertRaises(ValueError):
+ LZMAFile(BytesIO(COMPRESSED_XZ), "wx")
with self.assertRaises(ValueError):
LZMAFile(BytesIO(COMPRESSED_XZ), "rt")
with self.assertRaises(ValueError):
@@ -698,6 +716,20 @@ class FileTestCase(unittest.TestCase):
with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f:
self.assertRaises(EOFError, f.read)
+ def test_read_truncated(self):
+ # Drop stream footer: CRC (4 bytes), index size (4 bytes),
+ # flags (2 bytes) and magic number (2 bytes).
+ truncated = COMPRESSED_XZ[:-12]
+ with LZMAFile(BytesIO(truncated)) as f:
+ self.assertRaises(EOFError, f.read)
+ with LZMAFile(BytesIO(truncated)) as f:
+ self.assertEqual(f.read(len(INPUT)), INPUT)
+ self.assertRaises(EOFError, f.read, 1)
+ # Incomplete 12-byte header.
+ for i in range(12):
+ with LZMAFile(BytesIO(truncated[:i])) as f:
+ self.assertRaises(EOFError, f.read, 1)
+
def test_read_bad_args(self):
f = LZMAFile(BytesIO(COMPRESSED_XZ))
f.close()
@@ -1041,8 +1073,6 @@ class OpenTestCase(unittest.TestCase):
with self.assertRaises(ValueError):
lzma.open(TESTFN, "")
with self.assertRaises(ValueError):
- lzma.open(TESTFN, "x")
- with self.assertRaises(ValueError):
lzma.open(TESTFN, "rbt")
with self.assertRaises(ValueError):
lzma.open(TESTFN, "rb", encoding="utf-8")
@@ -1091,6 +1121,16 @@ class OpenTestCase(unittest.TestCase):
with lzma.open(bio, "rt", newline="\r") as f:
self.assertEqual(f.readlines(), [text])
+ def test_x_mode(self):
+ self.addCleanup(unlink, TESTFN)
+ for mode in ("x", "xb", "xt"):
+ unlink(TESTFN)
+ with lzma.open(TESTFN, mode):
+ pass
+ with self.assertRaises(FileExistsError):
+ with lzma.open(TESTFN, mode):
+ pass
+
class MiscellaneousTestCase(unittest.TestCase):
diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py
index 5e9e30a..d405093 100644
--- a/Lib/test/test_mailbox.py
+++ b/Lib/test/test_mailbox.py
@@ -596,7 +596,7 @@ class TestMaildir(TestMailbox, unittest.TestCase):
def setUp(self):
TestMailbox.setUp(self)
- if os.name in ('nt', 'os2') or sys.platform == 'cygwin':
+ if (os.name == 'nt') or (sys.platform == 'cygwin'):
self._box.colon = '!'
def assertMailboxEmpty(self):
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 0b14137..068c471 100644
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -22,37 +22,13 @@ class HelperMixin:
class IntTestCase(unittest.TestCase, HelperMixin):
def test_ints(self):
- # Test the full range of Python ints.
- n = sys.maxsize
+ # Test a range of Python ints larger than the machine word size.
+ n = sys.maxsize ** 2
while n:
for expected in (-n, n):
self.helper(expected)
n = n >> 1
- def test_int64(self):
- # Simulate int marshaling on a 64-bit box. This is most interesting if
- # we're running the test on a 32-bit box, of course.
-
- def to_little_endian_string(value, nbytes):
- b = bytearray()
- for i in range(nbytes):
- b.append(value & 0xff)
- value >>= 8
- return b
-
- maxint64 = (1 << 63) - 1
- minint64 = -maxint64-1
-
- for base in maxint64, minint64, -maxint64, -(minint64 >> 1):
- while base:
- s = b'I' + to_little_endian_string(base, 8)
- got = marshal.loads(s)
- self.assertEqual(base, got)
- if base == -1: # a fixed-point for shifting right 1
- base = 0
- else:
- base >>= 1
-
def test_bool(self):
for b in (True, False):
self.helper(b)
@@ -199,10 +175,14 @@ class BugsTestCase(unittest.TestCase):
except Exception:
pass
- def test_loads_recursion(self):
+ def test_loads_2x_code(self):
s = b'c' + (b'X' * 4*4) + b'{' * 2**20
self.assertRaises(ValueError, marshal.loads, s)
+ def test_loads_recursion(self):
+ s = b'c' + (b'X' * 4*5) + b'{' * 2**20
+ self.assertRaises(ValueError, marshal.loads, s)
+
def test_recursion_limit(self):
# Create a deeply nested structure.
head = last = []
@@ -280,15 +260,20 @@ class BugsTestCase(unittest.TestCase):
def test_bad_reader(self):
class BadReader(io.BytesIO):
- def read(self, n=-1):
- b = super().read(n)
+ def readinto(self, buf):
+ n = super().readinto(buf)
if n is not None and n > 4:
- b += b' ' * 10**6
- return b
+ n += 10**6
+ return n
for value in (1.0, 1j, b'0123456789', '0123456789'):
self.assertRaises(ValueError, marshal.load,
BadReader(marshal.dumps(value)))
+ def _test_eof(self):
+ data = marshal.dumps(("hello", "dolly", None))
+ for i in range(len(data)):
+ self.assertRaises(EOFError, marshal.loads, data[0: i])
+
LARGE_SIZE = 2**31
pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
@@ -333,6 +318,122 @@ class LargeValuesTestCase(unittest.TestCase):
def test_bytearray(self, size):
self.check_unmarshallable(bytearray(size))
+def CollectObjectIDs(ids, obj):
+ """Collect object ids seen in a structure"""
+ if id(obj) in ids:
+ return
+ ids.add(id(obj))
+ if isinstance(obj, (list, tuple, set, frozenset)):
+ for e in obj:
+ CollectObjectIDs(ids, e)
+ elif isinstance(obj, dict):
+ for k, v in obj.items():
+ CollectObjectIDs(ids, k)
+ CollectObjectIDs(ids, v)
+ return len(ids)
+
+class InstancingTestCase(unittest.TestCase, HelperMixin):
+ intobj = 123321
+ floatobj = 1.2345
+ strobj = "abcde"*3
+ dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
+
+ def helper3(self, rsample, recursive=False, simple=False):
+ #we have two instances
+ sample = (rsample, rsample)
+
+ n0 = CollectObjectIDs(set(), sample)
+
+ s3 = marshal.dumps(sample, 3)
+ n3 = CollectObjectIDs(set(), marshal.loads(s3))
+
+ #same number of instances generated
+ self.assertEqual(n3, n0)
+
+ if not recursive:
+ #can compare with version 2
+ s2 = marshal.dumps(sample, 2)
+ n2 = CollectObjectIDs(set(), marshal.loads(s2))
+ #old format generated more instances
+ self.assertGreater(n2, n0)
+
+ #if complex objects are in there, old format is larger
+ if not simple:
+ self.assertGreater(len(s2), len(s3))
+ else:
+ self.assertGreaterEqual(len(s2), len(s3))
+
+ def testInt(self):
+ self.helper(self.intobj)
+ self.helper3(self.intobj, simple=True)
+
+ def testFloat(self):
+ self.helper(self.floatobj)
+ self.helper3(self.floatobj)
+
+ def testStr(self):
+ self.helper(self.strobj)
+ self.helper3(self.strobj)
+
+ def testDict(self):
+ self.helper(self.dictobj)
+ self.helper3(self.dictobj)
+
+ def testModule(self):
+ with open(__file__, "rb") as f:
+ code = f.read()
+ if __file__.endswith(".py"):
+ code = compile(code, __file__, "exec")
+ self.helper(code)
+ self.helper3(code)
+
+ def testRecursion(self):
+ d = dict(self.dictobj)
+ d["self"] = d
+ self.helper3(d, recursive=True)
+ l = [self.dictobj]
+ l.append(l)
+ self.helper3(l, recursive=True)
+
+class CompatibilityTestCase(unittest.TestCase):
+ def _test(self, version):
+ with open(__file__, "rb") as f:
+ code = f.read()
+ if __file__.endswith(".py"):
+ code = compile(code, __file__, "exec")
+ data = marshal.dumps(code, version)
+ marshal.loads(data)
+
+ def test0To3(self):
+ self._test(0)
+
+ def test1To3(self):
+ self._test(1)
+
+ def test2To3(self):
+ self._test(2)
+
+ def test3To3(self):
+ self._test(3)
+
+class InterningTestCase(unittest.TestCase, HelperMixin):
+ strobj = "this is an interned string"
+ strobj = sys.intern(strobj)
+
+ def testIntern(self):
+ s = marshal.loads(marshal.dumps(self.strobj))
+ self.assertEqual(s, self.strobj)
+ self.assertEqual(id(s), id(self.strobj))
+ s2 = sys.intern(s)
+ self.assertEqual(id(s2), id(s))
+
+ def testNoIntern(self):
+ s = marshal.loads(marshal.dumps(self.strobj, 2))
+ self.assertEqual(s, self.strobj)
+ self.assertNotEqual(id(s), id(self.strobj))
+ s2 = sys.intern(s)
+ self.assertNotEqual(id(s2), id(s))
+
def test_main():
support.run_unittest(IntTestCase,
diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py
index 4fa9a19..9ef293e 100644
--- a/Lib/test/test_memoryio.py
+++ b/Lib/test/test_memoryio.py
@@ -520,12 +520,12 @@ class TextIOTestMixin:
def test_relative_seek(self):
memio = self.ioclass()
- self.assertRaises(IOError, memio.seek, -1, 1)
- self.assertRaises(IOError, memio.seek, 3, 1)
- self.assertRaises(IOError, memio.seek, -3, 1)
- self.assertRaises(IOError, memio.seek, -1, 2)
- self.assertRaises(IOError, memio.seek, 1, 1)
- self.assertRaises(IOError, memio.seek, 1, 2)
+ self.assertRaises(OSError, memio.seek, -1, 1)
+ self.assertRaises(OSError, memio.seek, 3, 1)
+ self.assertRaises(OSError, memio.seek, -3, 1)
+ self.assertRaises(OSError, memio.seek, -1, 2)
+ self.assertRaises(OSError, memio.seek, 1, 1)
+ self.assertRaises(OSError, memio.seek, 1, 2)
def test_textio_properties(self):
memio = self.ioclass()
diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py
index bf0eaad..e7df8a7 100644
--- a/Lib/test/test_memoryview.py
+++ b/Lib/test/test_memoryview.py
@@ -352,6 +352,15 @@ class AbstractMemoryTests:
self.assertIs(wr(), None)
self.assertIs(L[0], b)
+ def test_reversed(self):
+ for tp in self._types:
+ b = tp(self._source)
+ m = self._view(b)
+ aslist = list(reversed(m.tolist()))
+ self.assertEqual(list(reversed(m)), aslist)
+ self.assertEqual(list(reversed(m)), list(m[::-1]))
+
+
# Variations on source objects for the buffer: bytes-like objects, then arrays
# with itemsize > 1.
# NOTE: support for multi-dimensional objects is unimplemented.
diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py
index 113c7fd..a1516c2 100644
--- a/Lib/test/test_minidom.py
+++ b/Lib/test/test_minidom.py
@@ -360,6 +360,15 @@ class MinidomTest(unittest.TestCase):
and el.getAttribute("spam2") == "bam2")
dom.unlink()
+ def testGetAttrList(self):
+ pass
+
+ def testGetAttrValues(self):
+ pass
+
+ def testGetAttrLength(self):
+ pass
+
def testGetAttribute(self):
dom = Document()
child = dom.appendChild(
@@ -380,6 +389,8 @@ class MinidomTest(unittest.TestCase):
self.assertEqual(child2.getAttributeNS("http://www.python.org", "missing"),
'')
+ def testGetAttributeNode(self): pass
+
def testGetElementsByTagNameNS(self):
d="""<foo xmlns:minidom='http://pyxml.sf.net/minidom'>
<minidom:myelem/>
@@ -450,6 +461,8 @@ class MinidomTest(unittest.TestCase):
self.confirm(str(node) == repr(node))
dom.unlink()
+ def testTextNodeRepr(self): pass
+
def testWriteXML(self):
str = '<?xml version="1.0" ?><a b="c"/>'
dom = parseString(str)
@@ -513,6 +526,14 @@ class MinidomTest(unittest.TestCase):
and pi.localName is None
and pi.namespaceURI == xml.dom.EMPTY_NAMESPACE)
+ def testProcessingInstructionRepr(self): pass
+
+ def testTextRepr(self): pass
+
+ def testWriteText(self): pass
+
+ def testDocumentElement(self): pass
+
def testTooManyDocumentElements(self):
doc = parseString("<doc/>")
elem = doc.createElement("extra")
@@ -521,6 +542,26 @@ class MinidomTest(unittest.TestCase):
elem.unlink()
doc.unlink()
+ def testCreateElementNS(self): pass
+
+ def testCreateAttributeNS(self): pass
+
+ def testParse(self): pass
+
+ def testParseString(self): pass
+
+ def testComment(self): pass
+
+ def testAttrListItem(self): pass
+
+ def testAttrListItems(self): pass
+
+ def testAttrListItemNS(self): pass
+
+ def testAttrListKeys(self): pass
+
+ def testAttrListKeysNS(self): pass
+
def testRemoveNamedItem(self):
doc = parseString("<doc a=''/>")
e = doc.documentElement
@@ -540,6 +581,30 @@ class MinidomTest(unittest.TestCase):
self.assertRaises(xml.dom.NotFoundErr, attrs.removeNamedItemNS,
"http://xml.python.org/", "b")
+ def testAttrListValues(self): pass
+
+ def testAttrListLength(self): pass
+
+ def testAttrList__getitem__(self): pass
+
+ def testAttrList__setitem__(self): pass
+
+ def testSetAttrValueandNodeValue(self): pass
+
+ def testParseElement(self): pass
+
+ def testParseAttributes(self): pass
+
+ def testParseElementNamespaces(self): pass
+
+ def testParseAttributeNamespaces(self): pass
+
+ def testParseProcessingInstructions(self): pass
+
+ def testChildNodes(self): pass
+
+ def testFirstChild(self): pass
+
def testHasChildNodes(self):
dom = parseString("<doc><foo/></doc>")
doc = dom.documentElement
diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py
index 899df8d..6ca5e1b 100644
--- a/Lib/test/test_mmap.py
+++ b/Lib/test/test_mmap.py
@@ -1,11 +1,12 @@
from test.support import (TESTFN, run_unittest, import_module, unlink,
- requires, _2G, _4G)
+ requires, _2G, _4G, gc_collect)
import unittest
import os
import re
import itertools
import socket
import sys
+import weakref
# Skip test if we can't import mmap.
mmap = import_module('mmap')
@@ -245,7 +246,7 @@ class MmapTests(unittest.TestCase):
def test_bad_file_desc(self):
# Try opening a bad file descriptor...
- self.assertRaises(mmap.error, mmap.mmap, -2, 4096)
+ self.assertRaises(OSError, mmap.mmap, -2, 4096)
def test_tougher_find(self):
# Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2,
@@ -655,7 +656,7 @@ class MmapTests(unittest.TestCase):
m = mmap.mmap(f.fileno(), 0)
f.close()
try:
- m.resize(0) # will raise WindowsError
+ m.resize(0) # will raise OSError
except:
pass
try:
@@ -671,7 +672,7 @@ class MmapTests(unittest.TestCase):
# parameters to _get_osfhandle.
s = socket.socket()
try:
- with self.assertRaises(mmap.error):
+ with self.assertRaises(OSError):
m = mmap.mmap(s.fileno(), 10)
finally:
s.close()
@@ -682,14 +683,23 @@ class MmapTests(unittest.TestCase):
self.assertTrue(m.closed)
def test_context_manager_exception(self):
- # Test that the IOError gets passed through
+ # Test that the OSError gets passed through
with self.assertRaises(Exception) as exc:
with mmap.mmap(-1, 10) as m:
- raise IOError
- self.assertIsInstance(exc.exception, IOError,
+ raise OSError
+ self.assertIsInstance(exc.exception, OSError,
"wrong exception raised in context manager")
self.assertTrue(m.closed, "context manager failed")
+ def test_weakref(self):
+ # Check mmap objects are weakrefable
+ mm = mmap.mmap(-1, 16)
+ wr = weakref.ref(mm)
+ self.assertIs(wr(), mm)
+ del mm
+ gc_collect()
+ self.assertIs(wr(), None)
+
class LargeMmapTests(unittest.TestCase):
def setUp(self):
@@ -707,7 +717,7 @@ class LargeMmapTests(unittest.TestCase):
f.seek(num_zeroes)
f.write(tail)
f.flush()
- except (IOError, OverflowError):
+ except (OSError, OverflowError):
f.close()
raise unittest.SkipTest("filesystem does not have largefile support")
return f
diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py
index e5a2525..1230293 100644
--- a/Lib/test/test_module.py
+++ b/Lib/test/test_module.py
@@ -1,6 +1,8 @@
# Test the module type
import unittest
+import weakref
from test.support import run_unittest, gc_collect
+from test.script_helper import assert_python_ok
import sys
ModuleType = type(sys)
@@ -33,7 +35,12 @@ class ModuleTests(unittest.TestCase):
foo = ModuleType("foo")
self.assertEqual(foo.__name__, "foo")
self.assertEqual(foo.__doc__, None)
- self.assertEqual(foo.__dict__, {"__name__": "foo", "__doc__": None})
+ self.assertIs(foo.__loader__, None)
+ self.assertIs(foo.__package__, None)
+ self.assertIs(foo.__spec__, None)
+ self.assertEqual(foo.__dict__, {"__name__": "foo", "__doc__": None,
+ "__loader__": None, "__package__": None,
+ "__spec__": None})
def test_ascii_docstring(self):
# ASCII docstring
@@ -41,7 +48,9 @@ class ModuleTests(unittest.TestCase):
self.assertEqual(foo.__name__, "foo")
self.assertEqual(foo.__doc__, "foodoc")
self.assertEqual(foo.__dict__,
- {"__name__": "foo", "__doc__": "foodoc"})
+ {"__name__": "foo", "__doc__": "foodoc",
+ "__loader__": None, "__package__": None,
+ "__spec__": None})
def test_unicode_docstring(self):
# Unicode docstring
@@ -49,7 +58,9 @@ class ModuleTests(unittest.TestCase):
self.assertEqual(foo.__name__, "foo")
self.assertEqual(foo.__doc__, "foodoc\u1234")
self.assertEqual(foo.__dict__,
- {"__name__": "foo", "__doc__": "foodoc\u1234"})
+ {"__name__": "foo", "__doc__": "foodoc\u1234",
+ "__loader__": None, "__package__": None,
+ "__spec__": None})
def test_reinit(self):
# Reinitialization should not replace the __dict__
@@ -61,10 +72,10 @@ class ModuleTests(unittest.TestCase):
self.assertEqual(foo.__doc__, "foodoc")
self.assertEqual(foo.bar, 42)
self.assertEqual(foo.__dict__,
- {"__name__": "foo", "__doc__": "foodoc", "bar": 42})
+ {"__name__": "foo", "__doc__": "foodoc", "bar": 42,
+ "__loader__": None, "__package__": None, "__spec__": None})
self.assertTrue(foo.__dict__ is d)
- @unittest.expectedFailure
def test_dont_clear_dict(self):
# See issue 7140.
def f():
@@ -89,6 +100,14 @@ a = A(destroyed)"""
gc_collect()
self.assertEqual(destroyed, [1])
+ def test_weakref(self):
+ m = ModuleType("foo")
+ wr = weakref.ref(m)
+ self.assertIs(wr(), m)
+ del m
+ gc_collect()
+ self.assertIs(wr(), None)
+
def test_module_repr_minimal(self):
# reprs when modules have no __file__, __name__, or __loader__
m = ModuleType('foo')
@@ -110,13 +129,19 @@ a = A(destroyed)"""
m.__file__ = '/tmp/foo.py'
self.assertEqual(repr(m), "<module '?' from '/tmp/foo.py'>")
+ def test_module_repr_with_loader_as_None(self):
+ m = ModuleType('foo')
+ assert m.__loader__ is None
+ self.assertEqual(repr(m), "<module 'foo'>")
+
def test_module_repr_with_bare_loader_but_no_name(self):
m = ModuleType('foo')
del m.__name__
# Yes, a class not an instance.
m.__loader__ = BareLoader
+ loader_repr = repr(BareLoader)
self.assertEqual(
- repr(m), "<module '?' (<class 'test.test_module.BareLoader'>)>")
+ repr(m), "<module '?' ({})>".format(loader_repr))
def test_module_repr_with_full_loader_but_no_name(self):
# m.__loader__.module_repr() will fail because the module has no
@@ -126,15 +151,17 @@ a = A(destroyed)"""
del m.__name__
# Yes, a class not an instance.
m.__loader__ = FullLoader
+ loader_repr = repr(FullLoader)
self.assertEqual(
- repr(m), "<module '?' (<class 'test.test_module.FullLoader'>)>")
+ repr(m), "<module '?' ({})>".format(loader_repr))
def test_module_repr_with_bare_loader(self):
m = ModuleType('foo')
# Yes, a class not an instance.
m.__loader__ = BareLoader
+ module_repr = repr(BareLoader)
self.assertEqual(
- repr(m), "<module 'foo' (<class 'test.test_module.BareLoader'>)>")
+ repr(m), "<module 'foo' ({})>".format(module_repr))
def test_module_repr_with_full_loader(self):
m = ModuleType('foo')
@@ -164,8 +191,25 @@ a = A(destroyed)"""
def test_module_repr_source(self):
r = repr(unittest)
- self.assertEqual(r[:25], "<module 'unittest' from '")
- self.assertEqual(r[-13:], "__init__.py'>")
+ starts_with = "<module 'unittest' from '"
+ ends_with = "__init__.py'>"
+ self.assertEqual(r[:len(starts_with)], starts_with,
+ '{!r} does not start with {!r}'.format(r, starts_with))
+ self.assertEqual(r[-len(ends_with):], ends_with,
+ '{!r} does not end with {!r}'.format(r, ends_with))
+
+ def test_module_finalization_at_shutdown(self):
+ # Module globals and builtins should still be available during shutdown
+ rc, out, err = assert_python_ok("-c", "from test import final_a")
+ self.assertFalse(err)
+ lines = out.splitlines()
+ self.assertEqual(set(lines), {
+ b"x = a",
+ b"x = b",
+ b"final_a.x = a",
+ b"final_b.x = b",
+ b"len = len",
+ b"shutil.rmtree = rmtree"})
# frozen and namespace module reprs are tested in importlib.
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index 6889184..ea592c7 100644
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -175,57 +175,28 @@ class Test_StreamReader(unittest.TestCase):
support.unlink(TESTFN)
class Test_StreamWriter(unittest.TestCase):
- if len('\U00012345') == 2: # UCS2
- def test_gb18030(self):
- s= io.BytesIO()
- c = codecs.getwriter('gb18030')(s)
- c.write('123')
- self.assertEqual(s.getvalue(), b'123')
- c.write('\U00012345')
- self.assertEqual(s.getvalue(), b'123\x907\x959')
- c.write('\U00012345'[0])
- self.assertEqual(s.getvalue(), b'123\x907\x959')
- c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac')
- self.assertEqual(s.getvalue(),
- b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
- c.write('\U00012345'[0])
- self.assertEqual(s.getvalue(),
- b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
- self.assertRaises(UnicodeError, c.reset)
- self.assertEqual(s.getvalue(),
- b'123\x907\x959\x907\x959\x907\x959\x827\xcf5\x810\x851')
-
- def test_utf_8(self):
- s= io.BytesIO()
- c = codecs.getwriter('utf-8')(s)
- c.write('123')
- self.assertEqual(s.getvalue(), b'123')
- c.write('\U00012345')
- self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
-
- # Python utf-8 codec can't buffer surrogate pairs yet.
- if 0:
- c.write('\U00012345'[0])
- self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
- c.write('\U00012345'[1] + '\U00012345' + '\uac00\u00ac')
- self.assertEqual(s.getvalue(),
- b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
- b'\xea\xb0\x80\xc2\xac')
- c.write('\U00012345'[0])
- self.assertEqual(s.getvalue(),
- b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
- b'\xea\xb0\x80\xc2\xac')
- c.reset()
- self.assertEqual(s.getvalue(),
- b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
- b'\xea\xb0\x80\xc2\xac\xed\xa0\x88')
- c.write('\U00012345'[1])
- self.assertEqual(s.getvalue(),
- b'123\xf0\x92\x8d\x85\xf0\x92\x8d\x85\xf0\x92\x8d\x85'
- b'\xea\xb0\x80\xc2\xac\xed\xa0\x88\xed\xbd\x85')
-
- else: # UCS4
- pass
+ def test_gb18030(self):
+ s= io.BytesIO()
+ c = codecs.getwriter('gb18030')(s)
+ c.write('123')
+ self.assertEqual(s.getvalue(), b'123')
+ c.write('\U00012345')
+ self.assertEqual(s.getvalue(), b'123\x907\x959')
+ c.write('\uac00\u00ac')
+ self.assertEqual(s.getvalue(),
+ b'123\x907\x959\x827\xcf5\x810\x851')
+
+ def test_utf_8(self):
+ s= io.BytesIO()
+ c = codecs.getwriter('utf-8')(s)
+ c.write('123')
+ self.assertEqual(s.getvalue(), b'123')
+ c.write('\U00012345')
+ self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
+ c.write('\uac00\u00ac')
+ self.assertEqual(s.getvalue(),
+ b'123\xf0\x92\x8d\x85'
+ b'\xea\xb0\x80\xc2\xac')
def test_streamwriter_strwrite(self):
s = io.BytesIO()
diff --git a/Lib/test/test_multiprocessing_fork.py b/Lib/test/test_multiprocessing_fork.py
new file mode 100644
index 0000000..2bf4e75
--- /dev/null
+++ b/Lib/test/test_multiprocessing_fork.py
@@ -0,0 +1,7 @@
+import unittest
+import test._test_multiprocessing
+
+test._test_multiprocessing.install_tests_in_module_dict(globals(), 'fork')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_multiprocessing_forkserver.py b/Lib/test/test_multiprocessing_forkserver.py
new file mode 100644
index 0000000..193a04a
--- /dev/null
+++ b/Lib/test/test_multiprocessing_forkserver.py
@@ -0,0 +1,7 @@
+import unittest
+import test._test_multiprocessing
+
+test._test_multiprocessing.install_tests_in_module_dict(globals(), 'forkserver')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_multiprocessing_main_handling.py b/Lib/test/test_multiprocessing_main_handling.py
new file mode 100644
index 0000000..7ea8e78
--- /dev/null
+++ b/Lib/test/test_multiprocessing_main_handling.py
@@ -0,0 +1,285 @@
+# tests __main__ module handling in multiprocessing
+
+import importlib
+import importlib.machinery
+import zipimport
+import unittest
+import sys
+import os
+import os.path
+import py_compile
+
+from test import support
+from test.script_helper import (
+ make_pkg, make_script, make_zip_pkg, make_zip_script,
+ assert_python_ok, assert_python_failure, temp_dir,
+ spawn_python, kill_python)
+
+# Skip tests if _multiprocessing wasn't built.
+_multiprocessing = support.import_module('_multiprocessing')
+# Look up which start methods are available to test
+import multiprocessing
+AVAILABLE_START_METHODS = set(multiprocessing.get_all_start_methods())
+
+verbose = support.verbose
+
+test_source = """\
+# multiprocessing includes all sorts of shenanigans to make __main__
+# attributes accessible in the subprocess in a pickle compatible way.
+
+# We run the "doesn't work in the interactive interpreter" example from
+# the docs to make sure it *does* work from an executed __main__,
+# regardless of the invocation mechanism
+
+import sys
+import time
+from multiprocessing import Pool, set_start_method
+
+# We use this __main__ defined function in the map call below in order to
+# check that multiprocessing in correctly running the unguarded
+# code in child processes and then making it available as __main__
+def f(x):
+ return x*x
+
+# Check explicit relative imports
+if "check_sibling" in __file__:
+ # We're inside a package and not in a __main__.py file
+ # so make sure explicit relative imports work correctly
+ from . import sibling
+
+if __name__ == '__main__':
+ start_method = sys.argv[1]
+ set_start_method(start_method)
+ p = Pool(5)
+ results = []
+ p.map_async(f, [1, 2, 3], callback=results.extend)
+ deadline = time.time() + 10 # up to 10 s to report the results
+ while not results:
+ time.sleep(0.05)
+ if time.time() > deadline:
+ raise RuntimeError("Timed out waiting for results")
+ results.sort()
+ print(start_method, "->", results)
+"""
+
+test_source_main_skipped_in_children = """\
+# __main__.py files have an implied "if __name__ == '__main__'" so
+# multiprocessing should always skip running them in child processes
+
+# This means we can't use __main__ defined functions in child processes,
+# so we just use "int" as a passthrough operation below
+
+if __name__ != "__main__":
+ raise RuntimeError("Should only be called as __main__!")
+
+import sys
+import time
+from multiprocessing import Pool, set_start_method
+
+start_method = sys.argv[1]
+set_start_method(start_method)
+p = Pool(5)
+results = []
+p.map_async(int, [1, 4, 9], callback=results.extend)
+deadline = time.time() + 10 # up to 10 s to report the results
+while not results:
+ time.sleep(0.05)
+ if time.time() > deadline:
+ raise RuntimeError("Timed out waiting for results")
+results.sort()
+print(start_method, "->", results)
+"""
+
+# These helpers were copied from test_cmd_line_script & tweaked a bit...
+
+def _make_test_script(script_dir, script_basename,
+ source=test_source, omit_suffix=False):
+ to_return = make_script(script_dir, script_basename,
+ source, omit_suffix)
+ # Hack to check explicit relative imports
+ if script_basename == "check_sibling":
+ make_script(script_dir, "sibling", "")
+ importlib.invalidate_caches()
+ return to_return
+
+def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
+ source=test_source, depth=1):
+ to_return = make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
+ source, depth)
+ importlib.invalidate_caches()
+ return to_return
+
+# There's no easy way to pass the script directory in to get
+# -m to work (avoiding that is the whole point of making
+# directories and zipfiles executable!)
+# So we fake it for testing purposes with a custom launch script
+launch_source = """\
+import sys, os.path, runpy
+sys.path.insert(0, %s)
+runpy._run_module_as_main(%r)
+"""
+
+def _make_launch_script(script_dir, script_basename, module_name, path=None):
+ if path is None:
+ path = "os.path.dirname(__file__)"
+ else:
+ path = repr(path)
+ source = launch_source % (path, module_name)
+ to_return = make_script(script_dir, script_basename, source)
+ importlib.invalidate_caches()
+ return to_return
+
+class MultiProcessingCmdLineMixin():
+ maxDiff = None # Show full tracebacks on subprocess failure
+
+ def setUp(self):
+ if self.start_method not in AVAILABLE_START_METHODS:
+ self.skipTest("%r start method not available" % self.start_method)
+
+ def _check_output(self, script_name, exit_code, out, err):
+ if verbose > 1:
+ print("Output from test script %r:" % script_name)
+ print(out)
+ self.assertEqual(exit_code, 0)
+ self.assertEqual(err.decode('utf-8'), '')
+ expected_results = "%s -> [1, 4, 9]" % self.start_method
+ self.assertEqual(out.decode('utf-8').strip(), expected_results)
+
+ def _check_script(self, script_name, *cmd_line_switches):
+ if not __debug__:
+ cmd_line_switches += ('-' + 'O' * sys.flags.optimize,)
+ run_args = cmd_line_switches + (script_name, self.start_method)
+ rc, out, err = assert_python_ok(*run_args, __isolated=False)
+ self._check_output(script_name, rc, out, err)
+
+ def test_basic_script(self):
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, 'script')
+ self._check_script(script_name)
+
+ def test_basic_script_no_suffix(self):
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, 'script',
+ omit_suffix=True)
+ self._check_script(script_name)
+
+ def test_ipython_workaround(self):
+ # Some versions of the IPython launch script are missing the
+ # __name__ = "__main__" guard, and multiprocessing has long had
+ # a workaround for that case
+ # See https://github.com/ipython/ipython/issues/4698
+ source = test_source_main_skipped_in_children
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, 'ipython',
+ source=source)
+ self._check_script(script_name)
+ script_no_suffix = _make_test_script(script_dir, 'ipython',
+ source=source,
+ omit_suffix=True)
+ self._check_script(script_no_suffix)
+
+ def test_script_compiled(self):
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, 'script')
+ py_compile.compile(script_name, doraise=True)
+ os.remove(script_name)
+ pyc_file = support.make_legacy_pyc(script_name)
+ self._check_script(pyc_file)
+
+ def test_directory(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, '__main__',
+ source=source)
+ self._check_script(script_dir)
+
+ def test_directory_compiled(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, '__main__',
+ source=source)
+ py_compile.compile(script_name, doraise=True)
+ os.remove(script_name)
+ pyc_file = support.make_legacy_pyc(script_name)
+ self._check_script(script_dir)
+
+ def test_zipfile(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, '__main__',
+ source=source)
+ zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name)
+ self._check_script(zip_name)
+
+ def test_zipfile_compiled(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ script_name = _make_test_script(script_dir, '__main__',
+ source=source)
+ compiled_name = py_compile.compile(script_name, doraise=True)
+ zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
+ self._check_script(zip_name)
+
+ def test_module_in_package(self):
+ with temp_dir() as script_dir:
+ pkg_dir = os.path.join(script_dir, 'test_pkg')
+ make_pkg(pkg_dir)
+ script_name = _make_test_script(pkg_dir, 'check_sibling')
+ launch_name = _make_launch_script(script_dir, 'launch',
+ 'test_pkg.check_sibling')
+ self._check_script(launch_name)
+
+ def test_module_in_package_in_zipfile(self):
+ with temp_dir() as script_dir:
+ zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
+ launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
+ self._check_script(launch_name)
+
+ def test_module_in_subpackage_in_zipfile(self):
+ with temp_dir() as script_dir:
+ zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
+ launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
+ self._check_script(launch_name)
+
+ def test_package(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ pkg_dir = os.path.join(script_dir, 'test_pkg')
+ make_pkg(pkg_dir)
+ script_name = _make_test_script(pkg_dir, '__main__',
+ source=source)
+ launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
+ self._check_script(launch_name)
+
+ def test_package_compiled(self):
+ source = self.main_in_children_source
+ with temp_dir() as script_dir:
+ pkg_dir = os.path.join(script_dir, 'test_pkg')
+ make_pkg(pkg_dir)
+ script_name = _make_test_script(pkg_dir, '__main__',
+ source=source)
+ compiled_name = py_compile.compile(script_name, doraise=True)
+ os.remove(script_name)
+ pyc_file = support.make_legacy_pyc(script_name)
+ launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
+ self._check_script(launch_name)
+
+# Test all supported start methods (setupClass skips as appropriate)
+
+class SpawnCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase):
+ start_method = 'spawn'
+ main_in_children_source = test_source_main_skipped_in_children
+
+class ForkCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase):
+ start_method = 'fork'
+ main_in_children_source = test_source
+
+class ForkServerCmdLineTest(MultiProcessingCmdLineMixin, unittest.TestCase):
+ start_method = 'forkserver'
+ main_in_children_source = test_source_main_skipped_in_children
+
+def tearDownModule():
+ support.reap_children()
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_multiprocessing_spawn.py b/Lib/test/test_multiprocessing_spawn.py
new file mode 100644
index 0000000..334ae9e
--- /dev/null
+++ b/Lib/test/test_multiprocessing_spawn.py
@@ -0,0 +1,7 @@
+import unittest
+import test._test_multiprocessing
+
+test._test_multiprocessing.install_tests_in_module_dict(globals(), 'spawn')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py
index 1d52713..71a4ec0 100644
--- a/Lib/test/test_nntplib.py
+++ b/Lib/test/test_nntplib.py
@@ -266,7 +266,7 @@ class NetworkedNNTPTestsMixin:
return False
try:
server.help()
- except (socket.error, EOFError):
+ except (OSError, EOFError):
return False
return True
diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py
index 28ede34..ab2eeb7 100644
--- a/Lib/test/test_normalization.py
+++ b/Lib/test/test_normalization.py
@@ -43,7 +43,7 @@ class NormalizationTest(unittest.TestCase):
try:
testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
check=check_version)
- except (IOError, HTTPException):
+ except (OSError, HTTPException):
self.skipTest("Could not retrieve " + TESTDATAURL)
self.addCleanup(testdata.close)
for line in testdata:
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index e9e1d71..000fc75 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -306,6 +306,40 @@ class TestNtpath(unittest.TestCase):
# dialogs (#4804)
ntpath.sameopenfile(-1, -1)
+ def test_ismount(self):
+ self.assertTrue(ntpath.ismount("c:\\"))
+ self.assertTrue(ntpath.ismount("C:\\"))
+ self.assertTrue(ntpath.ismount("c:/"))
+ self.assertTrue(ntpath.ismount("C:/"))
+ self.assertTrue(ntpath.ismount("\\\\.\\c:\\"))
+ self.assertTrue(ntpath.ismount("\\\\.\\C:\\"))
+
+ self.assertTrue(ntpath.ismount(b"c:\\"))
+ self.assertTrue(ntpath.ismount(b"C:\\"))
+ self.assertTrue(ntpath.ismount(b"c:/"))
+ self.assertTrue(ntpath.ismount(b"C:/"))
+ self.assertTrue(ntpath.ismount(b"\\\\.\\c:\\"))
+ self.assertTrue(ntpath.ismount(b"\\\\.\\C:\\"))
+
+ with support.temp_dir() as d:
+ self.assertFalse(ntpath.ismount(d))
+
+ if sys.platform == "win32":
+ #
+ # Make sure the current folder isn't the root folder
+ # (or any other volume root). The drive-relative
+ # locations below cannot then refer to mount points
+ #
+ drive, path = ntpath.splitdrive(sys.executable)
+ with support.change_cwd(os.path.dirname(sys.executable)):
+ self.assertFalse(ntpath.ismount(drive.lower()))
+ self.assertFalse(ntpath.ismount(drive.upper()))
+
+ self.assertTrue(ntpath.ismount("\\\\localhost\\c$"))
+ self.assertTrue(ntpath.ismount("\\\\localhost\\c$\\"))
+
+ self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$"))
+ self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\"))
class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase):
pathmodule = ntpath
diff --git a/Lib/test/test_openpty.py b/Lib/test/test_openpty.py
index 6384370..4785107 100644
--- a/Lib/test/test_openpty.py
+++ b/Lib/test/test_openpty.py
@@ -4,7 +4,7 @@ import os, unittest
from test.support import run_unittest
if not hasattr(os, "openpty"):
- raise unittest.SkipTest("No openpty() available.")
+ raise unittest.SkipTest("os.openpty() not available.")
class OpenptyTest(unittest.TestCase):
diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py
index fa608b9..ab58a98 100644
--- a/Lib/test/test_operator.py
+++ b/Lib/test/test_operator.py
@@ -1,8 +1,10 @@
-import operator
import unittest
from test import support
+py_operator = support.import_fresh_module('operator', blocked=['_operator'])
+c_operator = support.import_fresh_module('operator', fresh=['_operator'])
+
class Seq1:
def __init__(self, lst):
self.lst = lst
@@ -32,8 +34,9 @@ class Seq2(object):
return other * self.lst
-class OperatorTestCase(unittest.TestCase):
+class OperatorTestCase:
def test_lt(self):
+ operator = self.module
self.assertRaises(TypeError, operator.lt)
self.assertRaises(TypeError, operator.lt, 1j, 2j)
self.assertFalse(operator.lt(1, 0))
@@ -44,6 +47,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertTrue(operator.lt(1, 2.0))
def test_le(self):
+ operator = self.module
self.assertRaises(TypeError, operator.le)
self.assertRaises(TypeError, operator.le, 1j, 2j)
self.assertFalse(operator.le(1, 0))
@@ -54,6 +58,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertTrue(operator.le(1, 2.0))
def test_eq(self):
+ operator = self.module
class C(object):
def __eq__(self, other):
raise SyntaxError
@@ -67,6 +72,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertFalse(operator.eq(1, 2.0))
def test_ne(self):
+ operator = self.module
class C(object):
def __ne__(self, other):
raise SyntaxError
@@ -80,6 +86,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertTrue(operator.ne(1, 2.0))
def test_ge(self):
+ operator = self.module
self.assertRaises(TypeError, operator.ge)
self.assertRaises(TypeError, operator.ge, 1j, 2j)
self.assertTrue(operator.ge(1, 0))
@@ -90,6 +97,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertFalse(operator.ge(1, 2.0))
def test_gt(self):
+ operator = self.module
self.assertRaises(TypeError, operator.gt)
self.assertRaises(TypeError, operator.gt, 1j, 2j)
self.assertTrue(operator.gt(1, 0))
@@ -100,22 +108,26 @@ class OperatorTestCase(unittest.TestCase):
self.assertFalse(operator.gt(1, 2.0))
def test_abs(self):
+ operator = self.module
self.assertRaises(TypeError, operator.abs)
self.assertRaises(TypeError, operator.abs, None)
self.assertEqual(operator.abs(-1), 1)
self.assertEqual(operator.abs(1), 1)
def test_add(self):
+ operator = self.module
self.assertRaises(TypeError, operator.add)
self.assertRaises(TypeError, operator.add, None, None)
self.assertTrue(operator.add(3, 4) == 7)
def test_bitwise_and(self):
+ operator = self.module
self.assertRaises(TypeError, operator.and_)
self.assertRaises(TypeError, operator.and_, None, None)
self.assertTrue(operator.and_(0xf, 0xa) == 0xa)
def test_concat(self):
+ operator = self.module
self.assertRaises(TypeError, operator.concat)
self.assertRaises(TypeError, operator.concat, None, None)
self.assertTrue(operator.concat('py', 'thon') == 'python')
@@ -125,12 +137,14 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(TypeError, operator.concat, 13, 29)
def test_countOf(self):
+ operator = self.module
self.assertRaises(TypeError, operator.countOf)
self.assertRaises(TypeError, operator.countOf, None, None)
self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1)
self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0)
def test_delitem(self):
+ operator = self.module
a = [4, 3, 2, 1]
self.assertRaises(TypeError, operator.delitem, a)
self.assertRaises(TypeError, operator.delitem, a, None)
@@ -138,33 +152,39 @@ class OperatorTestCase(unittest.TestCase):
self.assertTrue(a == [4, 2, 1])
def test_floordiv(self):
+ operator = self.module
self.assertRaises(TypeError, operator.floordiv, 5)
self.assertRaises(TypeError, operator.floordiv, None, None)
self.assertTrue(operator.floordiv(5, 2) == 2)
def test_truediv(self):
+ operator = self.module
self.assertRaises(TypeError, operator.truediv, 5)
self.assertRaises(TypeError, operator.truediv, None, None)
self.assertTrue(operator.truediv(5, 2) == 2.5)
def test_getitem(self):
+ operator = self.module
a = range(10)
self.assertRaises(TypeError, operator.getitem)
self.assertRaises(TypeError, operator.getitem, a, None)
self.assertTrue(operator.getitem(a, 2) == 2)
def test_indexOf(self):
+ operator = self.module
self.assertRaises(TypeError, operator.indexOf)
self.assertRaises(TypeError, operator.indexOf, None, None)
self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1)
self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0)
def test_invert(self):
+ operator = self.module
self.assertRaises(TypeError, operator.invert)
self.assertRaises(TypeError, operator.invert, None)
self.assertEqual(operator.inv(4), -5)
def test_lshift(self):
+ operator = self.module
self.assertRaises(TypeError, operator.lshift)
self.assertRaises(TypeError, operator.lshift, None, 42)
self.assertTrue(operator.lshift(5, 1) == 10)
@@ -172,16 +192,19 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(ValueError, operator.lshift, 2, -1)
def test_mod(self):
+ operator = self.module
self.assertRaises(TypeError, operator.mod)
self.assertRaises(TypeError, operator.mod, None, 42)
self.assertTrue(operator.mod(5, 2) == 1)
def test_mul(self):
+ operator = self.module
self.assertRaises(TypeError, operator.mul)
self.assertRaises(TypeError, operator.mul, None, None)
self.assertTrue(operator.mul(5, 2) == 10)
def test_neg(self):
+ operator = self.module
self.assertRaises(TypeError, operator.neg)
self.assertRaises(TypeError, operator.neg, None)
self.assertEqual(operator.neg(5), -5)
@@ -190,11 +213,13 @@ class OperatorTestCase(unittest.TestCase):
self.assertEqual(operator.neg(-0), 0)
def test_bitwise_or(self):
+ operator = self.module
self.assertRaises(TypeError, operator.or_)
self.assertRaises(TypeError, operator.or_, None, None)
self.assertTrue(operator.or_(0xa, 0x5) == 0xf)
def test_pos(self):
+ operator = self.module
self.assertRaises(TypeError, operator.pos)
self.assertRaises(TypeError, operator.pos, None)
self.assertEqual(operator.pos(5), 5)
@@ -203,14 +228,15 @@ class OperatorTestCase(unittest.TestCase):
self.assertEqual(operator.pos(-0), 0)
def test_pow(self):
+ operator = self.module
self.assertRaises(TypeError, operator.pow)
self.assertRaises(TypeError, operator.pow, None, None)
self.assertEqual(operator.pow(3,5), 3**5)
- self.assertEqual(operator.__pow__(3,5), 3**5)
self.assertRaises(TypeError, operator.pow, 1)
self.assertRaises(TypeError, operator.pow, 1, 2, 3)
def test_rshift(self):
+ operator = self.module
self.assertRaises(TypeError, operator.rshift)
self.assertRaises(TypeError, operator.rshift, None, 42)
self.assertTrue(operator.rshift(5, 1) == 2)
@@ -218,12 +244,14 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(ValueError, operator.rshift, 2, -1)
def test_contains(self):
+ operator = self.module
self.assertRaises(TypeError, operator.contains)
self.assertRaises(TypeError, operator.contains, None, None)
self.assertTrue(operator.contains(range(4), 2))
self.assertFalse(operator.contains(range(4), 5))
def test_setitem(self):
+ operator = self.module
a = list(range(3))
self.assertRaises(TypeError, operator.setitem, a)
self.assertRaises(TypeError, operator.setitem, a, None, None)
@@ -232,11 +260,13 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(IndexError, operator.setitem, a, 4, 2)
def test_sub(self):
+ operator = self.module
self.assertRaises(TypeError, operator.sub)
self.assertRaises(TypeError, operator.sub, None, None)
self.assertTrue(operator.sub(5, 2) == 3)
def test_truth(self):
+ operator = self.module
class C(object):
def __bool__(self):
raise SyntaxError
@@ -248,11 +278,13 @@ class OperatorTestCase(unittest.TestCase):
self.assertFalse(operator.truth([]))
def test_bitwise_xor(self):
+ operator = self.module
self.assertRaises(TypeError, operator.xor)
self.assertRaises(TypeError, operator.xor, None, None)
self.assertTrue(operator.xor(0xb, 0xc) == 0x7)
def test_is(self):
+ operator = self.module
a = b = 'xyzpdq'
c = a[:3] + b[3:]
self.assertRaises(TypeError, operator.is_)
@@ -260,6 +292,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertFalse(operator.is_(a,c))
def test_is_not(self):
+ operator = self.module
a = b = 'xyzpdq'
c = a[:3] + b[3:]
self.assertRaises(TypeError, operator.is_not)
@@ -267,6 +300,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertTrue(operator.is_not(a,c))
def test_attrgetter(self):
+ operator = self.module
class A:
pass
a = A()
@@ -316,6 +350,7 @@ class OperatorTestCase(unittest.TestCase):
self.assertEqual(f(a), ('arthur', 'thomas', 'johnson'))
def test_itemgetter(self):
+ operator = self.module
a = 'ABCDE'
f = operator.itemgetter(2)
self.assertEqual(f(a), 'C')
@@ -350,12 +385,15 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(TypeError, operator.itemgetter(2, 'x', 5), data)
def test_methodcaller(self):
+ operator = self.module
self.assertRaises(TypeError, operator.methodcaller)
class A:
def foo(self, *args, **kwds):
return args[0] + args[1]
def bar(self, f=42):
return f
+ def baz(*args, **kwds):
+ return kwds['name'], kwds['self']
a = A()
f = operator.methodcaller('foo')
self.assertRaises(IndexError, f, a)
@@ -366,8 +404,11 @@ class OperatorTestCase(unittest.TestCase):
self.assertRaises(TypeError, f, a, a)
f = operator.methodcaller('bar', f=5)
self.assertEqual(f(a), 5)
+ f = operator.methodcaller('baz', name='spam', self='eggs')
+ self.assertEqual(f(a), ('spam', 'eggs'))
def test_inplace(self):
+ operator = self.module
class C(object):
def __iadd__ (self, other): return "iadd"
def __iand__ (self, other): return "iand"
@@ -396,37 +437,48 @@ class OperatorTestCase(unittest.TestCase):
self.assertEqual(operator.itruediv (c, 5), "itruediv")
self.assertEqual(operator.ixor (c, 5), "ixor")
self.assertEqual(operator.iconcat (c, c), "iadd")
- self.assertEqual(operator.__iadd__ (c, 5), "iadd")
- self.assertEqual(operator.__iand__ (c, 5), "iand")
- self.assertEqual(operator.__ifloordiv__(c, 5), "ifloordiv")
- self.assertEqual(operator.__ilshift__ (c, 5), "ilshift")
- self.assertEqual(operator.__imod__ (c, 5), "imod")
- self.assertEqual(operator.__imul__ (c, 5), "imul")
- self.assertEqual(operator.__ior__ (c, 5), "ior")
- self.assertEqual(operator.__ipow__ (c, 5), "ipow")
- self.assertEqual(operator.__irshift__ (c, 5), "irshift")
- self.assertEqual(operator.__isub__ (c, 5), "isub")
- self.assertEqual(operator.__itruediv__ (c, 5), "itruediv")
- self.assertEqual(operator.__ixor__ (c, 5), "ixor")
- self.assertEqual(operator.__iconcat__ (c, c), "iadd")
-
-def test_main(verbose=None):
- import sys
- test_classes = (
- OperatorTestCase,
- )
-
- support.run_unittest(*test_classes)
-
- # verify reference counting
- if verbose and hasattr(sys, "gettotalrefcount"):
- import gc
- counts = [None] * 5
- for i in range(len(counts)):
- support.run_unittest(*test_classes)
- gc.collect()
- counts[i] = sys.gettotalrefcount()
- print(counts)
+
+ def test_length_hint(self):
+ operator = self.module
+ class X(object):
+ def __init__(self, value):
+ self.value = value
+
+ def __length_hint__(self):
+ if type(self.value) is type:
+ raise self.value
+ else:
+ return self.value
+
+ self.assertEqual(operator.length_hint([], 2), 0)
+ self.assertEqual(operator.length_hint(iter([1, 2, 3])), 3)
+
+ self.assertEqual(operator.length_hint(X(2)), 2)
+ self.assertEqual(operator.length_hint(X(NotImplemented), 4), 4)
+ self.assertEqual(operator.length_hint(X(TypeError), 12), 12)
+ with self.assertRaises(TypeError):
+ operator.length_hint(X("abc"))
+ with self.assertRaises(ValueError):
+ operator.length_hint(X(-2))
+ with self.assertRaises(LookupError):
+ operator.length_hint(X(LookupError))
+
+ def test_dunder_is_original(self):
+ operator = self.module
+
+ names = [name for name in dir(operator) if not name.startswith('_')]
+ for name in names:
+ orig = getattr(operator, name)
+ dunder = getattr(operator, '__' + name.strip('_') + '__', None)
+ if dunder:
+ self.assertIs(dunder, orig)
+
+class PyOperatorTestCase(OperatorTestCase, unittest.TestCase):
+ module = py_operator
+
+@unittest.skipUnless(c_operator, 'requires _operator')
+class COperatorTestCase(OperatorTestCase, unittest.TestCase):
+ module = c_operator
if __name__ == "__main__":
- test_main(verbose=True)
+ unittest.main()
diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py
index eaee504..3c8c612 100644
--- a/Lib/test/test_optparse.py
+++ b/Lib/test/test_optparse.py
@@ -730,7 +730,7 @@ class TestStandard(BaseTest):
def test_short_and_long_option_split(self):
self.assertParseOK(["-a", "xyz", "--foo", "bar"],
{'a': 'xyz', 'boo': None, 'foo': ["bar"]},
- []),
+ [])
def test_short_option_split_long_option_append(self):
self.assertParseOK(["--foo=bar", "-b", "123", "--foo", "baz"],
@@ -740,15 +740,15 @@ class TestStandard(BaseTest):
def test_short_option_split_one_positional_arg(self):
self.assertParseOK(["-a", "foo", "bar"],
{'a': "foo", 'boo': None, 'foo': None},
- ["bar"]),
+ ["bar"])
def test_short_option_consumes_separator(self):
self.assertParseOK(["-a", "--", "foo", "bar"],
{'a': "--", 'boo': None, 'foo': None},
- ["foo", "bar"]),
+ ["foo", "bar"])
self.assertParseOK(["-a", "--", "--foo", "bar"],
{'a': "--", 'boo': None, 'foo': ["bar"]},
- []),
+ [])
def test_short_option_joined_and_separator(self):
self.assertParseOK(["-ab", "--", "--foo", "bar"],
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index d70a0ae..5024093 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -24,6 +24,9 @@ import itertools
import stat
import locale
import codecs
+import decimal
+import fractions
+import pickle
try:
import threading
except ImportError:
@@ -32,6 +35,10 @@ try:
import resource
except ImportError:
resource = None
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
from test.script_helper import assert_python_ok
@@ -256,6 +263,13 @@ class StatAttributeTests(unittest.TestCase):
warnings.simplefilter("ignore", DeprecationWarning)
self.check_stat_attributes(fname)
+ def test_stat_result_pickle(self):
+ result = os.stat(self.fname)
+ p = pickle.dumps(result)
+ self.assertIn(b'\x03cos\nstat_result\n', p)
+ unpickled = pickle.loads(p)
+ self.assertEqual(result, unpickled)
+
@unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()')
def test_statvfs_attributes(self):
try:
@@ -263,7 +277,7 @@ class StatAttributeTests(unittest.TestCase):
except OSError as e:
# On AtheOS, glibc always returns ENOSYS
if e.errno == errno.ENOSYS:
- self.skipTest('glibc always returns ENOSYS on AtheOS')
+ self.skipTest('os.statvfs() failed with ENOSYS')
# Make sure direct access works
self.assertEqual(result.f_bfree, result[3])
@@ -300,6 +314,21 @@ class StatAttributeTests(unittest.TestCase):
except TypeError:
pass
+ @unittest.skipUnless(hasattr(os, 'statvfs'),
+ "need os.statvfs()")
+ def test_statvfs_result_pickle(self):
+ try:
+ result = os.statvfs(self.fname)
+ except OSError as e:
+ # On AtheOS, glibc always returns ENOSYS
+ if e.errno == errno.ENOSYS:
+ self.skipTest('os.statvfs() failed with ENOSYS')
+
+ p = pickle.dumps(result)
+ self.assertIn(b'\x03cos\nstatvfs_result\n', p)
+ unpickled = pickle.loads(p)
+ self.assertEqual(result, unpickled)
+
def test_utime_dir(self):
delta = 1000000
st = os.stat(support.TESTFN)
@@ -478,9 +507,9 @@ class StatAttributeTests(unittest.TestCase):
# Verify that an open file can be stat'ed
try:
os.stat(r"c:\pagefile.sys")
- except WindowsError as e:
- if e.errno == 2: # file does not exist; cannot run test
- self.skipTest(r'c:\pagefile.sys does not exist')
+ except FileNotFoundError:
+ self.skipTest(r'c:\pagefile.sys does not exist')
+ except OSError as e:
self.fail("Could not stat pagefile.sys")
@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
@@ -876,6 +905,17 @@ class MakedirTests(unittest.TestCase):
os.makedirs(path, mode=mode, exist_ok=True)
os.umask(old_mask)
+ @unittest.skipUnless(hasattr(os, 'chown'), 'test needs os.chown')
+ def test_chown_uid_gid_arguments_must_be_index(self):
+ stat = os.stat(support.TESTFN)
+ uid = stat.st_uid
+ gid = stat.st_gid
+ for value in (-1.0, -1j, decimal.Decimal(-1), fractions.Fraction(-2, 2)):
+ self.assertRaises(TypeError, os.chown, support.TESTFN, value, gid)
+ self.assertRaises(TypeError, os.chown, support.TESTFN, uid, value)
+ self.assertIsNone(os.chown(support.TESTFN, uid, gid))
+ self.assertIsNone(os.chown(support.TESTFN, -1, -1))
+
def test_exist_ok_s_isgid_directory(self):
path = os.path.join(support.TESTFN, 'dir1')
S_ISGID = stat.S_ISGID
@@ -1133,27 +1173,27 @@ class ExecTests(unittest.TestCase):
@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
class Win32ErrorTests(unittest.TestCase):
def test_rename(self):
- self.assertRaises(WindowsError, os.rename, support.TESTFN, support.TESTFN+".bak")
+ self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak")
def test_remove(self):
- self.assertRaises(WindowsError, os.remove, support.TESTFN)
+ self.assertRaises(OSError, os.remove, support.TESTFN)
def test_chdir(self):
- self.assertRaises(WindowsError, os.chdir, support.TESTFN)
+ self.assertRaises(OSError, os.chdir, support.TESTFN)
def test_mkdir(self):
f = open(support.TESTFN, "w")
try:
- self.assertRaises(WindowsError, os.mkdir, support.TESTFN)
+ self.assertRaises(OSError, os.mkdir, support.TESTFN)
finally:
f.close()
os.unlink(support.TESTFN)
def test_utime(self):
- self.assertRaises(WindowsError, os.utime, support.TESTFN, None)
+ self.assertRaises(OSError, os.utime, support.TESTFN, None)
def test_chmod(self):
- self.assertRaises(WindowsError, os.chmod, support.TESTFN, 0)
+ self.assertRaises(OSError, os.chmod, support.TESTFN, 0)
class TestInvalidFD(unittest.TestCase):
singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat",
@@ -1287,41 +1327,57 @@ class PosixUidGidTests(unittest.TestCase):
@unittest.skipUnless(hasattr(os, 'setuid'), 'test needs os.setuid()')
def test_setuid(self):
if os.getuid() != 0:
- self.assertRaises(os.error, os.setuid, 0)
+ self.assertRaises(OSError, os.setuid, 0)
self.assertRaises(OverflowError, os.setuid, 1<<32)
@unittest.skipUnless(hasattr(os, 'setgid'), 'test needs os.setgid()')
def test_setgid(self):
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
- self.assertRaises(os.error, os.setgid, 0)
+ self.assertRaises(OSError, os.setgid, 0)
self.assertRaises(OverflowError, os.setgid, 1<<32)
@unittest.skipUnless(hasattr(os, 'seteuid'), 'test needs os.seteuid()')
def test_seteuid(self):
if os.getuid() != 0:
- self.assertRaises(os.error, os.seteuid, 0)
+ self.assertRaises(OSError, os.seteuid, 0)
self.assertRaises(OverflowError, os.seteuid, 1<<32)
@unittest.skipUnless(hasattr(os, 'setegid'), 'test needs os.setegid()')
def test_setegid(self):
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
- self.assertRaises(os.error, os.setegid, 0)
+ self.assertRaises(OSError, os.setegid, 0)
self.assertRaises(OverflowError, os.setegid, 1<<32)
@unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()')
def test_setreuid(self):
if os.getuid() != 0:
- self.assertRaises(os.error, os.setreuid, 0, 0)
+ self.assertRaises(OSError, os.setreuid, 0, 0)
self.assertRaises(OverflowError, os.setreuid, 1<<32, 0)
self.assertRaises(OverflowError, os.setreuid, 0, 1<<32)
+ @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()')
+ def test_setreuid_neg1(self):
+ # Needs to accept -1. We run this in a subprocess to avoid
+ # altering the test runner's process state (issue8045).
+ subprocess.check_call([
+ sys.executable, '-c',
+ 'import os,sys;os.setreuid(-1,-1);sys.exit(0)'])
+
@unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()')
def test_setregid(self):
if os.getuid() != 0 and not HAVE_WHEEL_GROUP:
- self.assertRaises(os.error, os.setregid, 0, 0)
+ self.assertRaises(OSError, os.setregid, 0, 0)
self.assertRaises(OverflowError, os.setregid, 1<<32, 0)
self.assertRaises(OverflowError, os.setregid, 0, 1<<32)
+ @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()')
+ def test_setregid_neg1(self):
+ # Needs to accept -1. We run this in a subprocess to avoid
+ # altering the test runner's process state (issue8045).
+ subprocess.check_call([
+ sys.executable, '-c',
+ 'import os,sys;os.setregid(-1,-1);sys.exit(0)'])
+
@unittest.skipIf(sys.platform == "win32", "Posix specific tests")
class Pep383Tests(unittest.TestCase):
def setUp(self):
@@ -1511,6 +1567,52 @@ class Win32KillTests(unittest.TestCase):
@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+class Win32ListdirTests(unittest.TestCase):
+ """Test listdir on Windows."""
+
+ def setUp(self):
+ self.created_paths = []
+ for i in range(2):
+ dir_name = 'SUB%d' % i
+ dir_path = os.path.join(support.TESTFN, dir_name)
+ file_name = 'FILE%d' % i
+ file_path = os.path.join(support.TESTFN, file_name)
+ os.makedirs(dir_path)
+ with open(file_path, 'w') as f:
+ f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
+ self.created_paths.extend([dir_name, file_name])
+ self.created_paths.sort()
+
+ def tearDown(self):
+ shutil.rmtree(support.TESTFN)
+
+ def test_listdir_no_extended_path(self):
+ """Test when the path is not an "extended" path."""
+ # unicode
+ self.assertEqual(
+ sorted(os.listdir(support.TESTFN)),
+ self.created_paths)
+ # bytes
+ self.assertEqual(
+ sorted(os.listdir(os.fsencode(support.TESTFN))),
+ [os.fsencode(path) for path in self.created_paths])
+
+ def test_listdir_extended_path(self):
+ """Test when the path starts with '\\\\?\\'."""
+ # See: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+ # unicode
+ path = '\\\\?\\' + os.path.abspath(support.TESTFN)
+ self.assertEqual(
+ sorted(os.listdir(path)),
+ self.created_paths)
+ # bytes
+ path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN))
+ self.assertEqual(
+ sorted(os.listdir(path)),
+ [os.fsencode(path) for path in self.created_paths])
+
+
+@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
@support.skip_unless_symlink
class Win32SymlinkTests(unittest.TestCase):
filelink = 'filelinktest'
@@ -2175,6 +2277,197 @@ class TermsizeTests(unittest.TestCase):
self.assertEqual(expected, actual)
+class OSErrorTests(unittest.TestCase):
+ def setUp(self):
+ class Str(str):
+ pass
+
+ self.bytes_filenames = []
+ self.unicode_filenames = []
+ if support.TESTFN_UNENCODABLE is not None:
+ decoded = support.TESTFN_UNENCODABLE
+ else:
+ decoded = support.TESTFN
+ self.unicode_filenames.append(decoded)
+ self.unicode_filenames.append(Str(decoded))
+ if support.TESTFN_UNDECODABLE is not None:
+ encoded = support.TESTFN_UNDECODABLE
+ else:
+ encoded = os.fsencode(support.TESTFN)
+ self.bytes_filenames.append(encoded)
+ self.bytes_filenames.append(memoryview(encoded))
+
+ self.filenames = self.bytes_filenames + self.unicode_filenames
+
+ def test_oserror_filename(self):
+ funcs = [
+ (self.filenames, os.chdir,),
+ (self.filenames, os.chmod, 0o777),
+ (self.filenames, os.lstat,),
+ (self.filenames, os.open, os.O_RDONLY),
+ (self.filenames, os.rmdir,),
+ (self.filenames, os.stat,),
+ (self.filenames, os.unlink,),
+ ]
+ if sys.platform == "win32":
+ funcs.extend((
+ (self.bytes_filenames, os.rename, b"dst"),
+ (self.bytes_filenames, os.replace, b"dst"),
+ (self.unicode_filenames, os.rename, "dst"),
+ (self.unicode_filenames, os.replace, "dst"),
+ # Issue #16414: Don't test undecodable names with listdir()
+ # because of a Windows bug.
+ #
+ # With the ANSI code page 932, os.listdir(b'\xe7') return an
+ # empty list (instead of failing), whereas os.listdir(b'\xff')
+ # raises a FileNotFoundError. It looks like a Windows bug:
+ # b'\xe7' directory does not exist, FindFirstFileA(b'\xe7')
+ # fails with ERROR_FILE_NOT_FOUND (2), instead of
+ # ERROR_PATH_NOT_FOUND (3).
+ (self.unicode_filenames, os.listdir,),
+ ))
+ else:
+ funcs.extend((
+ (self.filenames, os.listdir,),
+ (self.filenames, os.rename, "dst"),
+ (self.filenames, os.replace, "dst"),
+ ))
+ if hasattr(os, "chown"):
+ funcs.append((self.filenames, os.chown, 0, 0))
+ if hasattr(os, "lchown"):
+ funcs.append((self.filenames, os.lchown, 0, 0))
+ if hasattr(os, "truncate"):
+ funcs.append((self.filenames, os.truncate, 0))
+ if hasattr(os, "chflags"):
+ funcs.append((self.filenames, os.chflags, 0))
+ if hasattr(os, "lchflags"):
+ funcs.append((self.filenames, os.lchflags, 0))
+ if hasattr(os, "chroot"):
+ funcs.append((self.filenames, os.chroot,))
+ if hasattr(os, "link"):
+ if sys.platform == "win32":
+ funcs.append((self.bytes_filenames, os.link, b"dst"))
+ funcs.append((self.unicode_filenames, os.link, "dst"))
+ else:
+ funcs.append((self.filenames, os.link, "dst"))
+ if hasattr(os, "listxattr"):
+ funcs.extend((
+ (self.filenames, os.listxattr,),
+ (self.filenames, os.getxattr, "user.test"),
+ (self.filenames, os.setxattr, "user.test", b'user'),
+ (self.filenames, os.removexattr, "user.test"),
+ ))
+ if hasattr(os, "lchmod"):
+ funcs.append((self.filenames, os.lchmod, 0o777))
+ if hasattr(os, "readlink"):
+ if sys.platform == "win32":
+ funcs.append((self.unicode_filenames, os.readlink,))
+ else:
+ funcs.append((self.filenames, os.readlink,))
+
+ for filenames, func, *func_args in funcs:
+ for name in filenames:
+ try:
+ func(name, *func_args)
+ except OSError as err:
+ self.assertIs(err.filename, name)
+ else:
+ self.fail("No exception thrown by {}".format(func))
+
+class CPUCountTests(unittest.TestCase):
+ def test_cpu_count(self):
+ cpus = os.cpu_count()
+ if cpus is not None:
+ self.assertIsInstance(cpus, int)
+ self.assertGreater(cpus, 0)
+ else:
+ self.skipTest("Could not determine the number of CPUs")
+
+
+class FDInheritanceTests(unittest.TestCase):
+ def test_get_set_inheritable(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+ self.assertEqual(os.get_inheritable(fd), False)
+
+ os.set_inheritable(fd, True)
+ self.assertEqual(os.get_inheritable(fd), True)
+
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_get_inheritable_cloexec(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+ self.assertEqual(os.get_inheritable(fd), False)
+
+ # clear FD_CLOEXEC flag
+ flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+ flags &= ~fcntl.FD_CLOEXEC
+ fcntl.fcntl(fd, fcntl.F_SETFD, flags)
+
+ self.assertEqual(os.get_inheritable(fd), True)
+
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_set_inheritable_cloexec(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ fcntl.FD_CLOEXEC)
+
+ os.set_inheritable(fd, True)
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ 0)
+
+ def test_open(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+ self.assertEqual(os.get_inheritable(fd), False)
+
+ @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()")
+ def test_pipe(self):
+ rfd, wfd = os.pipe()
+ self.addCleanup(os.close, rfd)
+ self.addCleanup(os.close, wfd)
+ self.assertEqual(os.get_inheritable(rfd), False)
+ self.assertEqual(os.get_inheritable(wfd), False)
+
+ def test_dup(self):
+ fd1 = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd1)
+
+ fd2 = os.dup(fd1)
+ self.addCleanup(os.close, fd2)
+ self.assertEqual(os.get_inheritable(fd2), False)
+
+ @unittest.skipUnless(hasattr(os, 'dup2'), "need os.dup2()")
+ def test_dup2(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+
+ # inheritable by default
+ fd2 = os.open(__file__, os.O_RDONLY)
+ try:
+ os.dup2(fd, fd2)
+ self.assertEqual(os.get_inheritable(fd2), True)
+ finally:
+ os.close(fd2)
+
+ # force non-inheritable
+ fd3 = os.open(__file__, os.O_RDONLY)
+ try:
+ os.dup2(fd, fd3, inheritable=False)
+ self.assertEqual(os.get_inheritable(fd3), False)
+ finally:
+ os.close(fd3)
+
+ @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()")
+ def test_openpty(self):
+ master_fd, slave_fd = os.openpty()
+ self.addCleanup(os.close, master_fd)
+ self.addCleanup(os.close, slave_fd)
+ self.assertEqual(os.get_inheritable(master_fd), False)
+ self.assertEqual(os.get_inheritable(slave_fd), False)
+
+
@support.reap_threads
def test_main():
support.run_unittest(
@@ -2192,6 +2485,7 @@ def test_main():
PosixUidGidTests,
Pep383Tests,
Win32KillTests,
+ Win32ListdirTests,
Win32SymlinkTests,
NonLocalSymlinkTests,
FSEncodingTests,
@@ -2204,7 +2498,10 @@ def test_main():
ExtendedAttributeTests,
Win32DeprecatedBytesAPI,
TermsizeTests,
+ OSErrorTests,
RemoveDirsTests,
+ CPUCountTests,
+ FDInheritanceTests,
)
if __name__ == "__main__":
diff --git a/Lib/test/test_ossaudiodev.py b/Lib/test/test_ossaudiodev.py
index 3908a05..c9e2a24 100644
--- a/Lib/test/test_ossaudiodev.py
+++ b/Lib/test/test_ossaudiodev.py
@@ -44,7 +44,7 @@ class OSSAudioDevTests(unittest.TestCase):
def play_sound_file(self, data, rate, ssize, nchannels):
try:
dsp = ossaudiodev.open('w')
- except IOError as msg:
+ except OSError as msg:
if msg.args[0] in (errno.EACCES, errno.ENOENT,
errno.ENODEV, errno.EBUSY):
raise unittest.SkipTest(msg)
@@ -190,7 +190,7 @@ class OSSAudioDevTests(unittest.TestCase):
def test_main():
try:
dsp = ossaudiodev.open('w')
- except (ossaudiodev.error, IOError) as msg:
+ except (ossaudiodev.error, OSError) as msg:
if msg.args[0] in (errno.EACCES, errno.ENOENT,
errno.ENODEV, errno.EBUSY):
raise unittest.SkipTest(msg)
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
new file mode 100644
index 0000000..2268f94
--- /dev/null
+++ b/Lib/test/test_pathlib.py
@@ -0,0 +1,1840 @@
+import collections
+import io
+import os
+import errno
+import pathlib
+import pickle
+import shutil
+import socket
+import stat
+import sys
+import tempfile
+import unittest
+from contextlib import contextmanager
+
+from test import support
+TESTFN = support.TESTFN
+
+try:
+ import grp, pwd
+except ImportError:
+ grp = pwd = None
+
+
+class _BaseFlavourTest(object):
+
+ def _check_parse_parts(self, arg, expected):
+ f = self.flavour.parse_parts
+ sep = self.flavour.sep
+ altsep = self.flavour.altsep
+ actual = f([x.replace('/', sep) for x in arg])
+ self.assertEqual(actual, expected)
+ if altsep:
+ actual = f([x.replace('/', altsep) for x in arg])
+ self.assertEqual(actual, expected)
+
+ def test_parse_parts_common(self):
+ check = self._check_parse_parts
+ sep = self.flavour.sep
+ # Unanchored parts
+ check([], ('', '', []))
+ check(['a'], ('', '', ['a']))
+ check(['a/'], ('', '', ['a']))
+ check(['a', 'b'], ('', '', ['a', 'b']))
+ # Expansion
+ check(['a/b'], ('', '', ['a', 'b']))
+ check(['a/b/'], ('', '', ['a', 'b']))
+ check(['a', 'b/c', 'd'], ('', '', ['a', 'b', 'c', 'd']))
+ # Collapsing and stripping excess slashes
+ check(['a', 'b//c', 'd'], ('', '', ['a', 'b', 'c', 'd']))
+ check(['a', 'b/c/', 'd'], ('', '', ['a', 'b', 'c', 'd']))
+ # Eliminating standalone dots
+ check(['.'], ('', '', []))
+ check(['.', '.', 'b'], ('', '', ['b']))
+ check(['a', '.', 'b'], ('', '', ['a', 'b']))
+ check(['a', '.', '.'], ('', '', ['a']))
+ # The first part is anchored
+ check(['/a/b'], ('', sep, [sep, 'a', 'b']))
+ check(['/a', 'b'], ('', sep, [sep, 'a', 'b']))
+ check(['/a/', 'b'], ('', sep, [sep, 'a', 'b']))
+ # Ignoring parts before an anchored part
+ check(['a', '/b', 'c'], ('', sep, [sep, 'b', 'c']))
+ check(['a', '/b', '/c'], ('', sep, [sep, 'c']))
+
+
+class PosixFlavourTest(_BaseFlavourTest, unittest.TestCase):
+ flavour = pathlib._posix_flavour
+
+ def test_parse_parts(self):
+ check = self._check_parse_parts
+ # Collapsing of excess leading slashes, except for the double-slash
+ # special case.
+ check(['//a', 'b'], ('', '//', ['//', 'a', 'b']))
+ check(['///a', 'b'], ('', '/', ['/', 'a', 'b']))
+ check(['////a', 'b'], ('', '/', ['/', 'a', 'b']))
+ # Paths which look like NT paths aren't treated specially
+ check(['c:a'], ('', '', ['c:a']))
+ check(['c:\\a'], ('', '', ['c:\\a']))
+ check(['\\a'], ('', '', ['\\a']))
+
+ def test_splitroot(self):
+ f = self.flavour.splitroot
+ self.assertEqual(f(''), ('', '', ''))
+ self.assertEqual(f('a'), ('', '', 'a'))
+ self.assertEqual(f('a/b'), ('', '', 'a/b'))
+ self.assertEqual(f('a/b/'), ('', '', 'a/b/'))
+ self.assertEqual(f('/a'), ('', '/', 'a'))
+ self.assertEqual(f('/a/b'), ('', '/', 'a/b'))
+ self.assertEqual(f('/a/b/'), ('', '/', 'a/b/'))
+ # The root is collapsed when there are redundant slashes
+ # except when there are exactly two leading slashes, which
+ # is a special case in POSIX.
+ self.assertEqual(f('//a'), ('', '//', 'a'))
+ self.assertEqual(f('///a'), ('', '/', 'a'))
+ self.assertEqual(f('///a/b'), ('', '/', 'a/b'))
+ # Paths which look like NT paths aren't treated specially
+ self.assertEqual(f('c:/a/b'), ('', '', 'c:/a/b'))
+ self.assertEqual(f('\\/a/b'), ('', '', '\\/a/b'))
+ self.assertEqual(f('\\a\\b'), ('', '', '\\a\\b'))
+
+
+class NTFlavourTest(_BaseFlavourTest, unittest.TestCase):
+ flavour = pathlib._windows_flavour
+
+ def test_parse_parts(self):
+ check = self._check_parse_parts
+ # First part is anchored
+ check(['c:'], ('c:', '', ['c:']))
+ check(['c:\\'], ('c:', '\\', ['c:\\']))
+ check(['\\'], ('', '\\', ['\\']))
+ check(['c:a'], ('c:', '', ['c:', 'a']))
+ check(['c:\\a'], ('c:', '\\', ['c:\\', 'a']))
+ check(['\\a'], ('', '\\', ['\\', 'a']))
+ # UNC paths
+ check(['\\\\a\\b'], ('\\\\a\\b', '\\', ['\\\\a\\b\\']))
+ check(['\\\\a\\b\\'], ('\\\\a\\b', '\\', ['\\\\a\\b\\']))
+ check(['\\\\a\\b\\c'], ('\\\\a\\b', '\\', ['\\\\a\\b\\', 'c']))
+ # Second part is anchored, so that the first part is ignored
+ check(['a', 'Z:b', 'c'], ('Z:', '', ['Z:', 'b', 'c']))
+ check(['a', 'Z:\\b', 'c'], ('Z:', '\\', ['Z:\\', 'b', 'c']))
+ check(['a', '\\b', 'c'], ('', '\\', ['\\', 'b', 'c']))
+ # UNC paths
+ check(['a', '\\\\b\\c', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd']))
+ # Collapsing and stripping excess slashes
+ check(['a', 'Z:\\\\b\\\\c\\', 'd\\'], ('Z:', '\\', ['Z:\\', 'b', 'c', 'd']))
+ # UNC paths
+ check(['a', '\\\\b\\c\\\\', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd']))
+ # Extended paths
+ check(['\\\\?\\c:\\'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\']))
+ check(['\\\\?\\c:\\a'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\', 'a']))
+ # Extended UNC paths (format is "\\?\UNC\server\share")
+ check(['\\\\?\\UNC\\b\\c'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\']))
+ check(['\\\\?\\UNC\\b\\c\\d'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\', 'd']))
+
+ def test_splitroot(self):
+ f = self.flavour.splitroot
+ self.assertEqual(f(''), ('', '', ''))
+ self.assertEqual(f('a'), ('', '', 'a'))
+ self.assertEqual(f('a\\b'), ('', '', 'a\\b'))
+ self.assertEqual(f('\\a'), ('', '\\', 'a'))
+ self.assertEqual(f('\\a\\b'), ('', '\\', 'a\\b'))
+ self.assertEqual(f('c:a\\b'), ('c:', '', 'a\\b'))
+ self.assertEqual(f('c:\\a\\b'), ('c:', '\\', 'a\\b'))
+ # Redundant slashes in the root are collapsed
+ self.assertEqual(f('\\\\a'), ('', '\\', 'a'))
+ self.assertEqual(f('\\\\\\a/b'), ('', '\\', 'a/b'))
+ self.assertEqual(f('c:\\\\a'), ('c:', '\\', 'a'))
+ self.assertEqual(f('c:\\\\\\a/b'), ('c:', '\\', 'a/b'))
+ # Valid UNC paths
+ self.assertEqual(f('\\\\a\\b'), ('\\\\a\\b', '\\', ''))
+ self.assertEqual(f('\\\\a\\b\\'), ('\\\\a\\b', '\\', ''))
+ self.assertEqual(f('\\\\a\\b\\c\\d'), ('\\\\a\\b', '\\', 'c\\d'))
+ # These are non-UNC paths (according to ntpath.py and test_ntpath)
+ # However, command.com says such paths are invalid, so it's
+ # difficult to know what the right semantics are
+ self.assertEqual(f('\\\\\\a\\b'), ('', '\\', 'a\\b'))
+ self.assertEqual(f('\\\\a'), ('', '\\', 'a'))
+
+
+#
+# Tests for the pure classes
+#
+
+class _BasePurePathTest(object):
+
+ # keys are canonical paths, values are list of tuples of arguments
+ # supposed to produce equal paths
+ equivalences = {
+ 'a/b': [
+ ('a', 'b'), ('a/', 'b'), ('a', 'b/'), ('a/', 'b/'),
+ ('a/b/',), ('a//b',), ('a//b//',),
+ # empty components get removed
+ ('', 'a', 'b'), ('a', '', 'b'), ('a', 'b', ''),
+ ],
+ '/b/c/d': [
+ ('a', '/b/c', 'd'), ('a', '///b//c', 'd/'),
+ ('/a', '/b/c', 'd'),
+ # empty components get removed
+ ('/', 'b', '', 'c/d'), ('/', '', 'b/c/d'), ('', '/b/c/d'),
+ ],
+ }
+
+ def setUp(self):
+ p = self.cls('a')
+ self.flavour = p._flavour
+ self.sep = self.flavour.sep
+ self.altsep = self.flavour.altsep
+
+ def test_constructor_common(self):
+ P = self.cls
+ p = P('a')
+ self.assertIsInstance(p, P)
+ P('a', 'b', 'c')
+ P('/a', 'b', 'c')
+ P('a/b/c')
+ P('/a/b/c')
+ self.assertEqual(P(P('a')), P('a'))
+ self.assertEqual(P(P('a'), 'b'), P('a/b'))
+ self.assertEqual(P(P('a'), P('b')), P('a/b'))
+
+ def test_join_common(self):
+ P = self.cls
+ p = P('a/b')
+ pp = p.joinpath('c')
+ self.assertEqual(pp, P('a/b/c'))
+ self.assertIs(type(pp), type(p))
+ pp = p.joinpath('c', 'd')
+ self.assertEqual(pp, P('a/b/c/d'))
+ pp = p.joinpath(P('c'))
+ self.assertEqual(pp, P('a/b/c'))
+ pp = p.joinpath('/c')
+ self.assertEqual(pp, P('/c'))
+
+ def test_div_common(self):
+ # Basically the same as joinpath()
+ P = self.cls
+ p = P('a/b')
+ pp = p / 'c'
+ self.assertEqual(pp, P('a/b/c'))
+ self.assertIs(type(pp), type(p))
+ pp = p / 'c/d'
+ self.assertEqual(pp, P('a/b/c/d'))
+ pp = p / 'c' / 'd'
+ self.assertEqual(pp, P('a/b/c/d'))
+ pp = 'c' / p / 'd'
+ self.assertEqual(pp, P('c/a/b/d'))
+ pp = p / P('c')
+ self.assertEqual(pp, P('a/b/c'))
+ pp = p/ '/c'
+ self.assertEqual(pp, P('/c'))
+
+ def _check_str(self, expected, args):
+ p = self.cls(*args)
+ self.assertEqual(str(p), expected.replace('/', self.sep))
+
+ def test_str_common(self):
+ # Canonicalized paths roundtrip
+ for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'):
+ self._check_str(pathstr, (pathstr,))
+ # Special case for the empty path
+ self._check_str('.', ('',))
+ # Other tests for str() are in test_equivalences()
+
+ def test_as_posix_common(self):
+ P = self.cls
+ for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'):
+ self.assertEqual(P(pathstr).as_posix(), pathstr)
+ # Other tests for as_posix() are in test_equivalences()
+
+ def test_as_bytes_common(self):
+ sep = os.fsencode(self.sep)
+ P = self.cls
+ self.assertEqual(bytes(P('a/b')), b'a' + sep + b'b')
+
+ def test_as_uri_common(self):
+ P = self.cls
+ with self.assertRaises(ValueError):
+ P('a').as_uri()
+ with self.assertRaises(ValueError):
+ P().as_uri()
+
+ def test_repr_common(self):
+ for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'):
+ p = self.cls(pathstr)
+ clsname = p.__class__.__name__
+ r = repr(p)
+ # The repr() is in the form ClassName("forward-slashes path")
+ self.assertTrue(r.startswith(clsname + '('), r)
+ self.assertTrue(r.endswith(')'), r)
+ inner = r[len(clsname) + 1 : -1]
+ self.assertEqual(eval(inner), p.as_posix())
+ # The repr() roundtrips
+ q = eval(r, pathlib.__dict__)
+ self.assertIs(q.__class__, p.__class__)
+ self.assertEqual(q, p)
+ self.assertEqual(repr(q), r)
+
+ def test_eq_common(self):
+ P = self.cls
+ self.assertEqual(P('a/b'), P('a/b'))
+ self.assertEqual(P('a/b'), P('a', 'b'))
+ self.assertNotEqual(P('a/b'), P('a'))
+ self.assertNotEqual(P('a/b'), P('/a/b'))
+ self.assertNotEqual(P('a/b'), P())
+ self.assertNotEqual(P('/a/b'), P('/'))
+ self.assertNotEqual(P(), P('/'))
+ self.assertNotEqual(P(), "")
+ self.assertNotEqual(P(), {})
+ self.assertNotEqual(P(), int)
+
+ def test_match_common(self):
+ P = self.cls
+ self.assertRaises(ValueError, P('a').match, '')
+ self.assertRaises(ValueError, P('a').match, '.')
+ # Simple relative pattern
+ self.assertTrue(P('b.py').match('b.py'))
+ self.assertTrue(P('a/b.py').match('b.py'))
+ self.assertTrue(P('/a/b.py').match('b.py'))
+ self.assertFalse(P('a.py').match('b.py'))
+ self.assertFalse(P('b/py').match('b.py'))
+ self.assertFalse(P('/a.py').match('b.py'))
+ self.assertFalse(P('b.py/c').match('b.py'))
+ # Wilcard relative pattern
+ self.assertTrue(P('b.py').match('*.py'))
+ self.assertTrue(P('a/b.py').match('*.py'))
+ self.assertTrue(P('/a/b.py').match('*.py'))
+ self.assertFalse(P('b.pyc').match('*.py'))
+ self.assertFalse(P('b./py').match('*.py'))
+ self.assertFalse(P('b.py/c').match('*.py'))
+ # Multi-part relative pattern
+ self.assertTrue(P('ab/c.py').match('a*/*.py'))
+ self.assertTrue(P('/d/ab/c.py').match('a*/*.py'))
+ self.assertFalse(P('a.py').match('a*/*.py'))
+ self.assertFalse(P('/dab/c.py').match('a*/*.py'))
+ self.assertFalse(P('ab/c.py/d').match('a*/*.py'))
+ # Absolute pattern
+ self.assertTrue(P('/b.py').match('/*.py'))
+ self.assertFalse(P('b.py').match('/*.py'))
+ self.assertFalse(P('a/b.py').match('/*.py'))
+ self.assertFalse(P('/a/b.py').match('/*.py'))
+ # Multi-part absolute pattern
+ self.assertTrue(P('/a/b.py').match('/a/*.py'))
+ self.assertFalse(P('/ab.py').match('/a/*.py'))
+ self.assertFalse(P('/a/b/c.py').match('/a/*.py'))
+
+ def test_ordering_common(self):
+ # Ordering is tuple-alike
+ def assertLess(a, b):
+ self.assertLess(a, b)
+ self.assertGreater(b, a)
+ P = self.cls
+ a = P('a')
+ b = P('a/b')
+ c = P('abc')
+ d = P('b')
+ assertLess(a, b)
+ assertLess(a, c)
+ assertLess(a, d)
+ assertLess(b, c)
+ assertLess(c, d)
+ P = self.cls
+ a = P('/a')
+ b = P('/a/b')
+ c = P('/abc')
+ d = P('/b')
+ assertLess(a, b)
+ assertLess(a, c)
+ assertLess(a, d)
+ assertLess(b, c)
+ assertLess(c, d)
+ with self.assertRaises(TypeError):
+ P() < {}
+
+ def test_parts_common(self):
+ # `parts` returns a tuple
+ sep = self.sep
+ P = self.cls
+ p = P('a/b')
+ parts = p.parts
+ self.assertEqual(parts, ('a', 'b'))
+ # The object gets reused
+ self.assertIs(parts, p.parts)
+ # When the path is absolute, the anchor is a separate part
+ p = P('/a/b')
+ parts = p.parts
+ self.assertEqual(parts, (sep, 'a', 'b'))
+
+ def test_equivalences(self):
+ for k, tuples in self.equivalences.items():
+ canon = k.replace('/', self.sep)
+ posix = k.replace(self.sep, '/')
+ if canon != posix:
+ tuples = tuples + [
+ tuple(part.replace('/', self.sep) for part in t)
+ for t in tuples
+ ]
+ tuples.append((posix, ))
+ pcanon = self.cls(canon)
+ for t in tuples:
+ p = self.cls(*t)
+ self.assertEqual(p, pcanon, "failed with args {}".format(t))
+ self.assertEqual(hash(p), hash(pcanon))
+ self.assertEqual(str(p), canon)
+ self.assertEqual(p.as_posix(), posix)
+
+ def test_parent_common(self):
+ # Relative
+ P = self.cls
+ p = P('a/b/c')
+ self.assertEqual(p.parent, P('a/b'))
+ self.assertEqual(p.parent.parent, P('a'))
+ self.assertEqual(p.parent.parent.parent, P())
+ self.assertEqual(p.parent.parent.parent.parent, P())
+ # Anchored
+ p = P('/a/b/c')
+ self.assertEqual(p.parent, P('/a/b'))
+ self.assertEqual(p.parent.parent, P('/a'))
+ self.assertEqual(p.parent.parent.parent, P('/'))
+ self.assertEqual(p.parent.parent.parent.parent, P('/'))
+
+ def test_parents_common(self):
+ # Relative
+ P = self.cls
+ p = P('a/b/c')
+ par = p.parents
+ self.assertEqual(len(par), 3)
+ self.assertEqual(par[0], P('a/b'))
+ self.assertEqual(par[1], P('a'))
+ self.assertEqual(par[2], P('.'))
+ self.assertEqual(list(par), [P('a/b'), P('a'), P('.')])
+ with self.assertRaises(IndexError):
+ par[-1]
+ with self.assertRaises(IndexError):
+ par[3]
+ with self.assertRaises(TypeError):
+ par[0] = p
+ # Anchored
+ p = P('/a/b/c')
+ par = p.parents
+ self.assertEqual(len(par), 3)
+ self.assertEqual(par[0], P('/a/b'))
+ self.assertEqual(par[1], P('/a'))
+ self.assertEqual(par[2], P('/'))
+ self.assertEqual(list(par), [P('/a/b'), P('/a'), P('/')])
+ with self.assertRaises(IndexError):
+ par[3]
+
+ def test_drive_common(self):
+ P = self.cls
+ self.assertEqual(P('a/b').drive, '')
+ self.assertEqual(P('/a/b').drive, '')
+ self.assertEqual(P('').drive, '')
+
+ def test_root_common(self):
+ P = self.cls
+ sep = self.sep
+ self.assertEqual(P('').root, '')
+ self.assertEqual(P('a/b').root, '')
+ self.assertEqual(P('/').root, sep)
+ self.assertEqual(P('/a/b').root, sep)
+
+ def test_anchor_common(self):
+ P = self.cls
+ sep = self.sep
+ self.assertEqual(P('').anchor, '')
+ self.assertEqual(P('a/b').anchor, '')
+ self.assertEqual(P('/').anchor, sep)
+ self.assertEqual(P('/a/b').anchor, sep)
+
+ def test_name_common(self):
+ P = self.cls
+ self.assertEqual(P('').name, '')
+ self.assertEqual(P('.').name, '')
+ self.assertEqual(P('/').name, '')
+ self.assertEqual(P('a/b').name, 'b')
+ self.assertEqual(P('/a/b').name, 'b')
+ self.assertEqual(P('/a/b/.').name, 'b')
+ self.assertEqual(P('a/b.py').name, 'b.py')
+ self.assertEqual(P('/a/b.py').name, 'b.py')
+
+ def test_suffix_common(self):
+ P = self.cls
+ self.assertEqual(P('').suffix, '')
+ self.assertEqual(P('.').suffix, '')
+ self.assertEqual(P('..').suffix, '')
+ self.assertEqual(P('/').suffix, '')
+ self.assertEqual(P('a/b').suffix, '')
+ self.assertEqual(P('/a/b').suffix, '')
+ self.assertEqual(P('/a/b/.').suffix, '')
+ self.assertEqual(P('a/b.py').suffix, '.py')
+ self.assertEqual(P('/a/b.py').suffix, '.py')
+ self.assertEqual(P('a/.hgrc').suffix, '')
+ self.assertEqual(P('/a/.hgrc').suffix, '')
+ self.assertEqual(P('a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('/a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('/a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '')
+ self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '')
+
+ def test_suffixes_common(self):
+ P = self.cls
+ self.assertEqual(P('').suffixes, [])
+ self.assertEqual(P('.').suffixes, [])
+ self.assertEqual(P('/').suffixes, [])
+ self.assertEqual(P('a/b').suffixes, [])
+ self.assertEqual(P('/a/b').suffixes, [])
+ self.assertEqual(P('/a/b/.').suffixes, [])
+ self.assertEqual(P('a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('/a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('a/.hgrc').suffixes, [])
+ self.assertEqual(P('/a/.hgrc').suffixes, [])
+ self.assertEqual(P('a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, [])
+ self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, [])
+
+ def test_stem_common(self):
+ P = self.cls
+ self.assertEqual(P('').stem, '')
+ self.assertEqual(P('.').stem, '')
+ self.assertEqual(P('..').stem, '..')
+ self.assertEqual(P('/').stem, '')
+ self.assertEqual(P('a/b').stem, 'b')
+ self.assertEqual(P('a/b.py').stem, 'b')
+ self.assertEqual(P('a/.hgrc').stem, '.hgrc')
+ self.assertEqual(P('a/.hg.rc').stem, '.hg')
+ self.assertEqual(P('a/b.tar.gz').stem, 'b.tar')
+ self.assertEqual(P('a/Some name. Ending with a dot.').stem,
+ 'Some name. Ending with a dot.')
+
+ def test_with_name_common(self):
+ P = self.cls
+ self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml'))
+ self.assertEqual(P('/a/b').with_name('d.xml'), P('/a/d.xml'))
+ self.assertEqual(P('a/b.py').with_name('d.xml'), P('a/d.xml'))
+ self.assertEqual(P('/a/b.py').with_name('d.xml'), P('/a/d.xml'))
+ self.assertEqual(P('a/Dot ending.').with_name('d.xml'), P('a/d.xml'))
+ self.assertEqual(P('/a/Dot ending.').with_name('d.xml'), P('/a/d.xml'))
+ self.assertRaises(ValueError, P('').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('.').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('/').with_name, 'd.xml')
+
+ def test_with_suffix_common(self):
+ P = self.cls
+ self.assertEqual(P('a/b').with_suffix('.gz'), P('a/b.gz'))
+ self.assertEqual(P('/a/b').with_suffix('.gz'), P('/a/b.gz'))
+ self.assertEqual(P('a/b.py').with_suffix('.gz'), P('a/b.gz'))
+ self.assertEqual(P('/a/b.py').with_suffix('.gz'), P('/a/b.gz'))
+ # Path doesn't have a "filename" component
+ self.assertRaises(ValueError, P('').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('.').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('/').with_suffix, '.gz')
+ # Invalid suffix
+ self.assertRaises(ValueError, P('a/b').with_suffix, 'gz')
+ self.assertRaises(ValueError, P('a/b').with_suffix, '/')
+ self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz')
+ self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d')
+ self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d')
+
+ def test_relative_to_common(self):
+ P = self.cls
+ p = P('a/b')
+ self.assertRaises(TypeError, p.relative_to)
+ self.assertRaises(TypeError, p.relative_to, b'a')
+ self.assertEqual(p.relative_to(P()), P('a/b'))
+ self.assertEqual(p.relative_to(''), P('a/b'))
+ self.assertEqual(p.relative_to(P('a')), P('b'))
+ self.assertEqual(p.relative_to('a'), P('b'))
+ self.assertEqual(p.relative_to('a/'), P('b'))
+ self.assertEqual(p.relative_to(P('a/b')), P())
+ self.assertEqual(p.relative_to('a/b'), P())
+ # With several args
+ self.assertEqual(p.relative_to('a', 'b'), P())
+ # Unrelated paths
+ self.assertRaises(ValueError, p.relative_to, P('c'))
+ self.assertRaises(ValueError, p.relative_to, P('a/b/c'))
+ self.assertRaises(ValueError, p.relative_to, P('a/c'))
+ self.assertRaises(ValueError, p.relative_to, P('/a'))
+ p = P('/a/b')
+ self.assertEqual(p.relative_to(P('/')), P('a/b'))
+ self.assertEqual(p.relative_to('/'), P('a/b'))
+ self.assertEqual(p.relative_to(P('/a')), P('b'))
+ self.assertEqual(p.relative_to('/a'), P('b'))
+ self.assertEqual(p.relative_to('/a/'), P('b'))
+ self.assertEqual(p.relative_to(P('/a/b')), P())
+ self.assertEqual(p.relative_to('/a/b'), P())
+ # Unrelated paths
+ self.assertRaises(ValueError, p.relative_to, P('/c'))
+ self.assertRaises(ValueError, p.relative_to, P('/a/b/c'))
+ self.assertRaises(ValueError, p.relative_to, P('/a/c'))
+ self.assertRaises(ValueError, p.relative_to, P())
+ self.assertRaises(ValueError, p.relative_to, '')
+ self.assertRaises(ValueError, p.relative_to, P('a'))
+
+ def test_pickling_common(self):
+ P = self.cls
+ p = P('/a/b')
+ for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
+ dumped = pickle.dumps(p, proto)
+ pp = pickle.loads(dumped)
+ self.assertIs(pp.__class__, p.__class__)
+ self.assertEqual(pp, p)
+ self.assertEqual(hash(pp), hash(p))
+ self.assertEqual(str(pp), str(p))
+
+
+class PurePosixPathTest(_BasePurePathTest, unittest.TestCase):
+ cls = pathlib.PurePosixPath
+
+ def test_root(self):
+ P = self.cls
+ self.assertEqual(P('/a/b').root, '/')
+ self.assertEqual(P('///a/b').root, '/')
+ # POSIX special case for two leading slashes
+ self.assertEqual(P('//a/b').root, '//')
+
+ def test_eq(self):
+ P = self.cls
+ self.assertNotEqual(P('a/b'), P('A/b'))
+ self.assertEqual(P('/a'), P('///a'))
+ self.assertNotEqual(P('/a'), P('//a'))
+
+ def test_as_uri(self):
+ P = self.cls
+ self.assertEqual(P('/').as_uri(), 'file:///')
+ self.assertEqual(P('/a/b.c').as_uri(), 'file:///a/b.c')
+ self.assertEqual(P('/a/b%#c').as_uri(), 'file:///a/b%25%23c')
+
+ def test_as_uri_non_ascii(self):
+ from urllib.parse import quote_from_bytes
+ P = self.cls
+ try:
+ os.fsencode('\xe9')
+ except UnicodeEncodeError:
+ self.skipTest("\\xe9 cannot be encoded to the filesystem encoding")
+ self.assertEqual(P('/a/b\xe9').as_uri(),
+ 'file:///a/b' + quote_from_bytes(os.fsencode('\xe9')))
+
+ def test_match(self):
+ P = self.cls
+ self.assertFalse(P('A.py').match('a.PY'))
+
+ def test_is_absolute(self):
+ P = self.cls
+ self.assertFalse(P().is_absolute())
+ self.assertFalse(P('a').is_absolute())
+ self.assertFalse(P('a/b/').is_absolute())
+ self.assertTrue(P('/').is_absolute())
+ self.assertTrue(P('/a').is_absolute())
+ self.assertTrue(P('/a/b/').is_absolute())
+ self.assertTrue(P('//a').is_absolute())
+ self.assertTrue(P('//a/b').is_absolute())
+
+ def test_is_reserved(self):
+ P = self.cls
+ self.assertIs(False, P('').is_reserved())
+ self.assertIs(False, P('/').is_reserved())
+ self.assertIs(False, P('/foo/bar').is_reserved())
+ self.assertIs(False, P('/dev/con/PRN/NUL').is_reserved())
+
+ def test_join(self):
+ P = self.cls
+ p = P('//a')
+ pp = p.joinpath('b')
+ self.assertEqual(pp, P('//a/b'))
+ pp = P('/a').joinpath('//c')
+ self.assertEqual(pp, P('//c'))
+ pp = P('//a').joinpath('/c')
+ self.assertEqual(pp, P('/c'))
+
+ def test_div(self):
+ # Basically the same as joinpath()
+ P = self.cls
+ p = P('//a')
+ pp = p / 'b'
+ self.assertEqual(pp, P('//a/b'))
+ pp = P('/a') / '//c'
+ self.assertEqual(pp, P('//c'))
+ pp = P('//a') / '/c'
+ self.assertEqual(pp, P('/c'))
+
+
+class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase):
+ cls = pathlib.PureWindowsPath
+
+ equivalences = _BasePurePathTest.equivalences.copy()
+ equivalences.update({
+ 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ],
+ 'c:/a': [
+ ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'),
+ ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'),
+ ],
+ '//a/b/': [ ('//a/b',) ],
+ '//a/b/c': [
+ ('//a/b', 'c'), ('//a/b/', 'c'),
+ ],
+ })
+
+ def test_str(self):
+ p = self.cls('a/b/c')
+ self.assertEqual(str(p), 'a\\b\\c')
+ p = self.cls('c:/a/b/c')
+ self.assertEqual(str(p), 'c:\\a\\b\\c')
+ p = self.cls('//a/b')
+ self.assertEqual(str(p), '\\\\a\\b\\')
+ p = self.cls('//a/b/c')
+ self.assertEqual(str(p), '\\\\a\\b\\c')
+ p = self.cls('//a/b/c/d')
+ self.assertEqual(str(p), '\\\\a\\b\\c\\d')
+
+ def test_eq(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b'), P('c:a/b'))
+ self.assertEqual(P('c:a/b'), P('c:', 'a', 'b'))
+ self.assertNotEqual(P('c:a/b'), P('d:a/b'))
+ self.assertNotEqual(P('c:a/b'), P('c:/a/b'))
+ self.assertNotEqual(P('/a/b'), P('c:/a/b'))
+ # Case-insensitivity
+ self.assertEqual(P('a/B'), P('A/b'))
+ self.assertEqual(P('C:a/B'), P('c:A/b'))
+ self.assertEqual(P('//Some/SHARE/a/B'), P('//somE/share/A/b'))
+
+ def test_as_uri(self):
+ from urllib.parse import quote_from_bytes
+ P = self.cls
+ with self.assertRaises(ValueError):
+ P('/a/b').as_uri()
+ with self.assertRaises(ValueError):
+ P('c:a/b').as_uri()
+ self.assertEqual(P('c:/').as_uri(), 'file:///c:/')
+ self.assertEqual(P('c:/a/b.c').as_uri(), 'file:///c:/a/b.c')
+ self.assertEqual(P('c:/a/b%#c').as_uri(), 'file:///c:/a/b%25%23c')
+ self.assertEqual(P('c:/a/b\xe9').as_uri(), 'file:///c:/a/b%C3%A9')
+ self.assertEqual(P('//some/share/').as_uri(), 'file://some/share/')
+ self.assertEqual(P('//some/share/a/b.c').as_uri(),
+ 'file://some/share/a/b.c')
+ self.assertEqual(P('//some/share/a/b%#c\xe9').as_uri(),
+ 'file://some/share/a/b%25%23c%C3%A9')
+
+ def test_match_common(self):
+ P = self.cls
+ # Absolute patterns
+ self.assertTrue(P('c:/b.py').match('/*.py'))
+ self.assertTrue(P('c:/b.py').match('c:*.py'))
+ self.assertTrue(P('c:/b.py').match('c:/*.py'))
+ self.assertFalse(P('d:/b.py').match('c:/*.py')) # wrong drive
+ self.assertFalse(P('b.py').match('/*.py'))
+ self.assertFalse(P('b.py').match('c:*.py'))
+ self.assertFalse(P('b.py').match('c:/*.py'))
+ self.assertFalse(P('c:b.py').match('/*.py'))
+ self.assertFalse(P('c:b.py').match('c:/*.py'))
+ self.assertFalse(P('/b.py').match('c:*.py'))
+ self.assertFalse(P('/b.py').match('c:/*.py'))
+ # UNC patterns
+ self.assertTrue(P('//some/share/a.py').match('/*.py'))
+ self.assertTrue(P('//some/share/a.py').match('//some/share/*.py'))
+ self.assertFalse(P('//other/share/a.py').match('//some/share/*.py'))
+ self.assertFalse(P('//some/share/a/b.py').match('//some/share/*.py'))
+ # Case-insensitivity
+ self.assertTrue(P('B.py').match('b.PY'))
+ self.assertTrue(P('c:/a/B.Py').match('C:/A/*.pY'))
+ self.assertTrue(P('//Some/Share/B.Py').match('//somE/sharE/*.pY'))
+
+ def test_ordering_common(self):
+ # Case-insensitivity
+ def assertOrderedEqual(a, b):
+ self.assertLessEqual(a, b)
+ self.assertGreaterEqual(b, a)
+ P = self.cls
+ p = P('c:A/b')
+ q = P('C:a/B')
+ assertOrderedEqual(p, q)
+ self.assertFalse(p < q)
+ self.assertFalse(p > q)
+ p = P('//some/Share/A/b')
+ q = P('//Some/SHARE/a/B')
+ assertOrderedEqual(p, q)
+ self.assertFalse(p < q)
+ self.assertFalse(p > q)
+
+ def test_parts(self):
+ P = self.cls
+ p = P('c:a/b')
+ parts = p.parts
+ self.assertEqual(parts, ('c:', 'a', 'b'))
+ p = P('c:/a/b')
+ parts = p.parts
+ self.assertEqual(parts, ('c:\\', 'a', 'b'))
+ p = P('//a/b/c/d')
+ parts = p.parts
+ self.assertEqual(parts, ('\\\\a\\b\\', 'c', 'd'))
+
+ def test_parent(self):
+ # Anchored
+ P = self.cls
+ p = P('z:a/b/c')
+ self.assertEqual(p.parent, P('z:a/b'))
+ self.assertEqual(p.parent.parent, P('z:a'))
+ self.assertEqual(p.parent.parent.parent, P('z:'))
+ self.assertEqual(p.parent.parent.parent.parent, P('z:'))
+ p = P('z:/a/b/c')
+ self.assertEqual(p.parent, P('z:/a/b'))
+ self.assertEqual(p.parent.parent, P('z:/a'))
+ self.assertEqual(p.parent.parent.parent, P('z:/'))
+ self.assertEqual(p.parent.parent.parent.parent, P('z:/'))
+ p = P('//a/b/c/d')
+ self.assertEqual(p.parent, P('//a/b/c'))
+ self.assertEqual(p.parent.parent, P('//a/b'))
+ self.assertEqual(p.parent.parent.parent, P('//a/b'))
+
+ def test_parents(self):
+ # Anchored
+ P = self.cls
+ p = P('z:a/b/')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('z:a'))
+ self.assertEqual(par[1], P('z:'))
+ self.assertEqual(list(par), [P('z:a'), P('z:')])
+ with self.assertRaises(IndexError):
+ par[2]
+ p = P('z:/a/b/')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('z:/a'))
+ self.assertEqual(par[1], P('z:/'))
+ self.assertEqual(list(par), [P('z:/a'), P('z:/')])
+ with self.assertRaises(IndexError):
+ par[2]
+ p = P('//a/b/c/d')
+ par = p.parents
+ self.assertEqual(len(par), 2)
+ self.assertEqual(par[0], P('//a/b/c'))
+ self.assertEqual(par[1], P('//a/b'))
+ self.assertEqual(list(par), [P('//a/b/c'), P('//a/b')])
+ with self.assertRaises(IndexError):
+ par[2]
+
+ def test_drive(self):
+ P = self.cls
+ self.assertEqual(P('c:').drive, 'c:')
+ self.assertEqual(P('c:a/b').drive, 'c:')
+ self.assertEqual(P('c:/').drive, 'c:')
+ self.assertEqual(P('c:/a/b/').drive, 'c:')
+ self.assertEqual(P('//a/b').drive, '\\\\a\\b')
+ self.assertEqual(P('//a/b/').drive, '\\\\a\\b')
+ self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b')
+
+ def test_root(self):
+ P = self.cls
+ self.assertEqual(P('c:').root, '')
+ self.assertEqual(P('c:a/b').root, '')
+ self.assertEqual(P('c:/').root, '\\')
+ self.assertEqual(P('c:/a/b/').root, '\\')
+ self.assertEqual(P('//a/b').root, '\\')
+ self.assertEqual(P('//a/b/').root, '\\')
+ self.assertEqual(P('//a/b/c/d').root, '\\')
+
+ def test_anchor(self):
+ P = self.cls
+ self.assertEqual(P('c:').anchor, 'c:')
+ self.assertEqual(P('c:a/b').anchor, 'c:')
+ self.assertEqual(P('c:/').anchor, 'c:\\')
+ self.assertEqual(P('c:/a/b/').anchor, 'c:\\')
+ self.assertEqual(P('//a/b').anchor, '\\\\a\\b\\')
+ self.assertEqual(P('//a/b/').anchor, '\\\\a\\b\\')
+ self.assertEqual(P('//a/b/c/d').anchor, '\\\\a\\b\\')
+
+ def test_name(self):
+ P = self.cls
+ self.assertEqual(P('c:').name, '')
+ self.assertEqual(P('c:/').name, '')
+ self.assertEqual(P('c:a/b').name, 'b')
+ self.assertEqual(P('c:/a/b').name, 'b')
+ self.assertEqual(P('c:a/b.py').name, 'b.py')
+ self.assertEqual(P('c:/a/b.py').name, 'b.py')
+ self.assertEqual(P('//My.py/Share.php').name, '')
+ self.assertEqual(P('//My.py/Share.php/a/b').name, 'b')
+
+ def test_suffix(self):
+ P = self.cls
+ self.assertEqual(P('c:').suffix, '')
+ self.assertEqual(P('c:/').suffix, '')
+ self.assertEqual(P('c:a/b').suffix, '')
+ self.assertEqual(P('c:/a/b').suffix, '')
+ self.assertEqual(P('c:a/b.py').suffix, '.py')
+ self.assertEqual(P('c:/a/b.py').suffix, '.py')
+ self.assertEqual(P('c:a/.hgrc').suffix, '')
+ self.assertEqual(P('c:/a/.hgrc').suffix, '')
+ self.assertEqual(P('c:a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc')
+ self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz')
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '')
+ self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '')
+ self.assertEqual(P('//My.py/Share.php').suffix, '')
+ self.assertEqual(P('//My.py/Share.php/a/b').suffix, '')
+
+ def test_suffixes(self):
+ P = self.cls
+ self.assertEqual(P('c:').suffixes, [])
+ self.assertEqual(P('c:/').suffixes, [])
+ self.assertEqual(P('c:a/b').suffixes, [])
+ self.assertEqual(P('c:/a/b').suffixes, [])
+ self.assertEqual(P('c:a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('c:/a/b.py').suffixes, ['.py'])
+ self.assertEqual(P('c:a/.hgrc').suffixes, [])
+ self.assertEqual(P('c:/a/.hgrc').suffixes, [])
+ self.assertEqual(P('c:a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('c:/a/.hg.rc').suffixes, ['.rc'])
+ self.assertEqual(P('c:a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz'])
+ self.assertEqual(P('//My.py/Share.php').suffixes, [])
+ self.assertEqual(P('//My.py/Share.php/a/b').suffixes, [])
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, [])
+ self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, [])
+
+ def test_stem(self):
+ P = self.cls
+ self.assertEqual(P('c:').stem, '')
+ self.assertEqual(P('c:.').stem, '')
+ self.assertEqual(P('c:..').stem, '..')
+ self.assertEqual(P('c:/').stem, '')
+ self.assertEqual(P('c:a/b').stem, 'b')
+ self.assertEqual(P('c:a/b.py').stem, 'b')
+ self.assertEqual(P('c:a/.hgrc').stem, '.hgrc')
+ self.assertEqual(P('c:a/.hg.rc').stem, '.hg')
+ self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar')
+ self.assertEqual(P('c:a/Some name. Ending with a dot.').stem,
+ 'Some name. Ending with a dot.')
+
+ def test_with_name(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b').with_name('d.xml'), P('c:a/d.xml'))
+ self.assertEqual(P('c:/a/b').with_name('d.xml'), P('c:/a/d.xml'))
+ self.assertEqual(P('c:a/Dot ending.').with_name('d.xml'), P('c:a/d.xml'))
+ self.assertEqual(P('c:/a/Dot ending.').with_name('d.xml'), P('c:/a/d.xml'))
+ self.assertRaises(ValueError, P('c:').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('c:/').with_name, 'd.xml')
+ self.assertRaises(ValueError, P('//My/Share').with_name, 'd.xml')
+
+ def test_with_suffix(self):
+ P = self.cls
+ self.assertEqual(P('c:a/b').with_suffix('.gz'), P('c:a/b.gz'))
+ self.assertEqual(P('c:/a/b').with_suffix('.gz'), P('c:/a/b.gz'))
+ self.assertEqual(P('c:a/b.py').with_suffix('.gz'), P('c:a/b.gz'))
+ self.assertEqual(P('c:/a/b.py').with_suffix('.gz'), P('c:/a/b.gz'))
+ # Path doesn't have a "filename" component
+ self.assertRaises(ValueError, P('').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('.').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('/').with_suffix, '.gz')
+ self.assertRaises(ValueError, P('//My/Share').with_suffix, '.gz')
+ # Invalid suffix
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '/')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '/.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '\\.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c:.gz')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c/d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, 'c\\d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c/d')
+ self.assertRaises(ValueError, P('c:a/b').with_suffix, '.c\\d')
+
+ def test_relative_to(self):
+ P = self.cls
+ p = P('C:Foo/Bar')
+ self.assertEqual(p.relative_to(P('c:')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:foO')), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO'), P('Bar'))
+ self.assertEqual(p.relative_to('c:foO/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:foO/baR')), P())
+ self.assertEqual(p.relative_to('c:foO/baR'), P())
+ # Unrelated paths
+ self.assertRaises(ValueError, p.relative_to, P())
+ self.assertRaises(ValueError, p.relative_to, '')
+ self.assertRaises(ValueError, p.relative_to, P('d:'))
+ self.assertRaises(ValueError, p.relative_to, P('/'))
+ self.assertRaises(ValueError, p.relative_to, P('Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo/Bar/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo/Baz'))
+ p = P('C:/Foo/Bar')
+ self.assertEqual(p.relative_to(P('c:')), P('/Foo/Bar'))
+ self.assertEqual(p.relative_to('c:'), P('/Foo/Bar'))
+ self.assertEqual(str(p.relative_to(P('c:'))), '\\Foo\\Bar')
+ self.assertEqual(str(p.relative_to('c:')), '\\Foo\\Bar')
+ self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('c:/'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO')), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO'), P('Bar'))
+ self.assertEqual(p.relative_to('c:/foO/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('c:/foO/baR')), P())
+ self.assertEqual(p.relative_to('c:/foO/baR'), P())
+ # Unrelated paths
+ self.assertRaises(ValueError, p.relative_to, P('C:/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz'))
+ self.assertRaises(ValueError, p.relative_to, P('C:Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('d:'))
+ self.assertRaises(ValueError, p.relative_to, P('d:/'))
+ self.assertRaises(ValueError, p.relative_to, P('/'))
+ self.assertRaises(ValueError, p.relative_to, P('/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//C/Foo'))
+ # UNC paths
+ p = P('//Server/Share/Foo/Bar')
+ self.assertEqual(p.relative_to(P('//sErver/sHare')), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/'), P('Foo/Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo')), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo'), P('Bar'))
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/'), P('Bar'))
+ self.assertEqual(p.relative_to(P('//sErver/sHare/Foo/Bar')), P())
+ self.assertEqual(p.relative_to('//sErver/sHare/Foo/Bar'), P())
+ # Unrelated paths
+ self.assertRaises(ValueError, p.relative_to, P('/Server/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('c:/Server/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//z/Share/Foo'))
+ self.assertRaises(ValueError, p.relative_to, P('//Server/z/Foo'))
+
+ def test_is_absolute(self):
+ P = self.cls
+ # Under NT, only paths with both a drive and a root are absolute
+ self.assertFalse(P().is_absolute())
+ self.assertFalse(P('a').is_absolute())
+ self.assertFalse(P('a/b/').is_absolute())
+ self.assertFalse(P('/').is_absolute())
+ self.assertFalse(P('/a').is_absolute())
+ self.assertFalse(P('/a/b/').is_absolute())
+ self.assertFalse(P('c:').is_absolute())
+ self.assertFalse(P('c:a').is_absolute())
+ self.assertFalse(P('c:a/b/').is_absolute())
+ self.assertTrue(P('c:/').is_absolute())
+ self.assertTrue(P('c:/a').is_absolute())
+ self.assertTrue(P('c:/a/b/').is_absolute())
+ # UNC paths are absolute by definition
+ self.assertTrue(P('//a/b').is_absolute())
+ self.assertTrue(P('//a/b/').is_absolute())
+ self.assertTrue(P('//a/b/c').is_absolute())
+ self.assertTrue(P('//a/b/c/d').is_absolute())
+
+ def test_join(self):
+ P = self.cls
+ p = P('C:/a/b')
+ pp = p.joinpath('x/y')
+ self.assertEqual(pp, P('C:/a/b/x/y'))
+ pp = p.joinpath('/x/y')
+ self.assertEqual(pp, P('C:/x/y'))
+ # Joining with a different drive => the first path is ignored, even
+ # if the second path is relative.
+ pp = p.joinpath('D:x/y')
+ self.assertEqual(pp, P('D:x/y'))
+ pp = p.joinpath('D:/x/y')
+ self.assertEqual(pp, P('D:/x/y'))
+ pp = p.joinpath('//host/share/x/y')
+ self.assertEqual(pp, P('//host/share/x/y'))
+ # Joining with the same drive => the first path is appended to if
+ # the second path is relative.
+ pp = p.joinpath('c:x/y')
+ self.assertEqual(pp, P('C:/a/b/x/y'))
+ pp = p.joinpath('c:/x/y')
+ self.assertEqual(pp, P('C:/x/y'))
+
+ def test_div(self):
+ # Basically the same as joinpath()
+ P = self.cls
+ p = P('C:/a/b')
+ self.assertEqual(p / 'x/y', P('C:/a/b/x/y'))
+ self.assertEqual(p / 'x' / 'y', P('C:/a/b/x/y'))
+ self.assertEqual(p / '/x/y', P('C:/x/y'))
+ self.assertEqual(p / '/x' / 'y', P('C:/x/y'))
+ # Joining with a different drive => the first path is ignored, even
+ # if the second path is relative.
+ self.assertEqual(p / 'D:x/y', P('D:x/y'))
+ self.assertEqual(p / 'D:' / 'x/y', P('D:x/y'))
+ self.assertEqual(p / 'D:/x/y', P('D:/x/y'))
+ self.assertEqual(p / 'D:' / '/x/y', P('D:/x/y'))
+ self.assertEqual(p / '//host/share/x/y', P('//host/share/x/y'))
+ # Joining with the same drive => the first path is appended to if
+ # the second path is relative.
+ self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y'))
+ self.assertEqual(p / 'c:/x/y', P('C:/x/y'))
+
+ def test_is_reserved(self):
+ P = self.cls
+ self.assertIs(False, P('').is_reserved())
+ self.assertIs(False, P('/').is_reserved())
+ self.assertIs(False, P('/foo/bar').is_reserved())
+ self.assertIs(True, P('con').is_reserved())
+ self.assertIs(True, P('NUL').is_reserved())
+ self.assertIs(True, P('NUL.txt').is_reserved())
+ self.assertIs(True, P('com1').is_reserved())
+ self.assertIs(True, P('com9.bar').is_reserved())
+ self.assertIs(False, P('bar.com9').is_reserved())
+ self.assertIs(True, P('lpt1').is_reserved())
+ self.assertIs(True, P('lpt9.bar').is_reserved())
+ self.assertIs(False, P('bar.lpt9').is_reserved())
+ # Only the last component matters
+ self.assertIs(False, P('c:/NUL/con/baz').is_reserved())
+ # UNC paths are never reserved
+ self.assertIs(False, P('//my/share/nul/con/aux').is_reserved())
+
+
+class PurePathTest(_BasePurePathTest, unittest.TestCase):
+ cls = pathlib.PurePath
+
+ def test_concrete_class(self):
+ p = self.cls('a')
+ self.assertIs(type(p),
+ pathlib.PureWindowsPath if os.name == 'nt' else pathlib.PurePosixPath)
+
+ def test_different_flavours_unequal(self):
+ p = pathlib.PurePosixPath('a')
+ q = pathlib.PureWindowsPath('a')
+ self.assertNotEqual(p, q)
+
+ def test_different_flavours_unordered(self):
+ p = pathlib.PurePosixPath('a')
+ q = pathlib.PureWindowsPath('a')
+ with self.assertRaises(TypeError):
+ p < q
+ with self.assertRaises(TypeError):
+ p <= q
+ with self.assertRaises(TypeError):
+ p > q
+ with self.assertRaises(TypeError):
+ p >= q
+
+
+#
+# Tests for the concrete classes
+#
+
+# Make sure any symbolic links in the base test path are resolved
+BASE = os.path.realpath(TESTFN)
+join = lambda *x: os.path.join(BASE, *x)
+rel_join = lambda *x: os.path.join(TESTFN, *x)
+
+def symlink_skip_reason():
+ if not pathlib.supports_symlinks:
+ return "no system support for symlinks"
+ try:
+ os.symlink(__file__, BASE)
+ except OSError as e:
+ return str(e)
+ else:
+ support.unlink(BASE)
+ return None
+
+symlink_skip_reason = symlink_skip_reason()
+
+only_nt = unittest.skipIf(os.name != 'nt',
+ 'test requires a Windows-compatible system')
+only_posix = unittest.skipIf(os.name == 'nt',
+ 'test requires a POSIX-compatible system')
+with_symlinks = unittest.skipIf(symlink_skip_reason, symlink_skip_reason)
+
+
+@only_posix
+class PosixPathAsPureTest(PurePosixPathTest):
+ cls = pathlib.PosixPath
+
+@only_nt
+class WindowsPathAsPureTest(PureWindowsPathTest):
+ cls = pathlib.WindowsPath
+
+
+class _BasePathTest(object):
+ """Tests for the FS-accessing functionalities of the Path classes."""
+
+ # (BASE)
+ # |
+ # |-- dirA/
+ # |-- linkC -> "../dirB"
+ # |-- dirB/
+ # | |-- fileB
+ # |-- linkD -> "../dirB"
+ # |-- dirC/
+ # | |-- fileC
+ # | |-- fileD
+ # |-- fileA
+ # |-- linkA -> "fileA"
+ # |-- linkB -> "dirB"
+ #
+
+ def setUp(self):
+ os.mkdir(BASE)
+ self.addCleanup(shutil.rmtree, BASE)
+ os.mkdir(join('dirA'))
+ os.mkdir(join('dirB'))
+ os.mkdir(join('dirC'))
+ os.mkdir(join('dirC', 'dirD'))
+ with open(join('fileA'), 'wb') as f:
+ f.write(b"this is file A\n")
+ with open(join('dirB', 'fileB'), 'wb') as f:
+ f.write(b"this is file B\n")
+ with open(join('dirC', 'fileC'), 'wb') as f:
+ f.write(b"this is file C\n")
+ with open(join('dirC', 'dirD', 'fileD'), 'wb') as f:
+ f.write(b"this is file D\n")
+ if not symlink_skip_reason:
+ # Relative symlinks
+ os.symlink('fileA', join('linkA'))
+ os.symlink('non-existing', join('brokenLink'))
+ self.dirlink('dirB', join('linkB'))
+ self.dirlink(os.path.join('..', 'dirB'), join('dirA', 'linkC'))
+ # This one goes upwards but doesn't create a loop
+ self.dirlink(os.path.join('..', 'dirB'), join('dirB', 'linkD'))
+
+ if os.name == 'nt':
+ # Workaround for http://bugs.python.org/issue13772
+ def dirlink(self, src, dest):
+ os.symlink(src, dest, target_is_directory=True)
+ else:
+ def dirlink(self, src, dest):
+ os.symlink(src, dest)
+
+ def assertSame(self, path_a, path_b):
+ self.assertTrue(os.path.samefile(str(path_a), str(path_b)),
+ "%r and %r don't point to the same file" %
+ (path_a, path_b))
+
+ def assertFileNotFound(self, func, *args, **kwargs):
+ with self.assertRaises(FileNotFoundError) as cm:
+ func(*args, **kwargs)
+ self.assertEqual(cm.exception.errno, errno.ENOENT)
+
+ def _test_cwd(self, p):
+ q = self.cls(os.getcwd())
+ self.assertEqual(p, q)
+ self.assertEqual(str(p), str(q))
+ self.assertIs(type(p), type(q))
+ self.assertTrue(p.is_absolute())
+
+ def test_cwd(self):
+ p = self.cls.cwd()
+ self._test_cwd(p)
+
+ def test_empty_path(self):
+ # The empty path points to '.'
+ p = self.cls('')
+ self.assertEqual(p.stat(), os.stat('.'))
+
+ def test_exists(self):
+ P = self.cls
+ p = P(BASE)
+ self.assertIs(True, p.exists())
+ self.assertIs(True, (p / 'dirA').exists())
+ self.assertIs(True, (p / 'fileA').exists())
+ if not symlink_skip_reason:
+ self.assertIs(True, (p / 'linkA').exists())
+ self.assertIs(True, (p / 'linkB').exists())
+ self.assertIs(False, (p / 'foo').exists())
+ self.assertIs(False, P('/xyzzy').exists())
+
+ def test_open_common(self):
+ p = self.cls(BASE)
+ with (p / 'fileA').open('r') as f:
+ self.assertIsInstance(f, io.TextIOBase)
+ self.assertEqual(f.read(), "this is file A\n")
+ with (p / 'fileA').open('rb') as f:
+ self.assertIsInstance(f, io.BufferedIOBase)
+ self.assertEqual(f.read().strip(), b"this is file A")
+ with (p / 'fileA').open('rb', buffering=0) as f:
+ self.assertIsInstance(f, io.RawIOBase)
+ self.assertEqual(f.read().strip(), b"this is file A")
+
+ def test_iterdir(self):
+ P = self.cls
+ p = P(BASE)
+ it = p.iterdir()
+ paths = set(it)
+ expected = ['dirA', 'dirB', 'dirC', 'fileA']
+ if not symlink_skip_reason:
+ expected += ['linkA', 'linkB', 'brokenLink']
+ self.assertEqual(paths, { P(BASE, q) for q in expected })
+
+ @with_symlinks
+ def test_iterdir_symlink(self):
+ # __iter__ on a symlink to a directory
+ P = self.cls
+ p = P(BASE, 'linkB')
+ paths = set(p.iterdir())
+ expected = { P(BASE, 'linkB', q) for q in ['fileB', 'linkD'] }
+ self.assertEqual(paths, expected)
+
+ def test_iterdir_nodir(self):
+ # __iter__ on something that is not a directory
+ p = self.cls(BASE, 'fileA')
+ with self.assertRaises(OSError) as cm:
+ next(p.iterdir())
+ # ENOENT or EINVAL under Windows, ENOTDIR otherwise
+ # (see issue #12802)
+ self.assertIn(cm.exception.errno, (errno.ENOTDIR,
+ errno.ENOENT, errno.EINVAL))
+
+ def test_glob_common(self):
+ def _check(glob, expected):
+ self.assertEqual(set(glob), { P(BASE, q) for q in expected })
+ P = self.cls
+ p = P(BASE)
+ it = p.glob("fileA")
+ self.assertIsInstance(it, collections.Iterator)
+ _check(it, ["fileA"])
+ _check(p.glob("fileB"), [])
+ _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"])
+ if symlink_skip_reason:
+ _check(p.glob("*A"), ['dirA', 'fileA'])
+ else:
+ _check(p.glob("*A"), ['dirA', 'fileA', 'linkA'])
+ if symlink_skip_reason:
+ _check(p.glob("*B/*"), ['dirB/fileB'])
+ else:
+ _check(p.glob("*B/*"), ['dirB/fileB', 'dirB/linkD',
+ 'linkB/fileB', 'linkB/linkD'])
+ if symlink_skip_reason:
+ _check(p.glob("*/fileB"), ['dirB/fileB'])
+ else:
+ _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB'])
+
+ def test_rglob_common(self):
+ def _check(glob, expected):
+ self.assertEqual(set(glob), { P(BASE, q) for q in expected })
+ P = self.cls
+ p = P(BASE)
+ it = p.rglob("fileA")
+ self.assertIsInstance(it, collections.Iterator)
+ # XXX cannot test because of symlink loops in the test setup
+ #_check(it, ["fileA"])
+ #_check(p.rglob("fileB"), ["dirB/fileB"])
+ #_check(p.rglob("*/fileA"), [""])
+ #_check(p.rglob("*/fileB"), ["dirB/fileB"])
+ #_check(p.rglob("file*"), ["fileA", "dirB/fileB"])
+ # No symlink loops here
+ p = P(BASE, "dirC")
+ _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"])
+ _check(p.rglob("*/*"), ["dirC/dirD/fileD"])
+
+ def test_glob_dotdot(self):
+ # ".." is not special in globs
+ P = self.cls
+ p = P(BASE)
+ self.assertEqual(set(p.glob("..")), { P(BASE, "..") })
+ self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") })
+ self.assertEqual(set(p.glob("../xyzzy")), set())
+
+ def _check_resolve_relative(self, p, expected):
+ q = p.resolve()
+ self.assertEqual(q, expected)
+
+ def _check_resolve_absolute(self, p, expected):
+ q = p.resolve()
+ self.assertEqual(q, expected)
+
+ @with_symlinks
+ def test_resolve_common(self):
+ P = self.cls
+ p = P(BASE, 'foo')
+ with self.assertRaises(OSError) as cm:
+ p.resolve()
+ self.assertEqual(cm.exception.errno, errno.ENOENT)
+ # These are all relative symlinks
+ p = P(BASE, 'dirB', 'fileB')
+ self._check_resolve_relative(p, p)
+ p = P(BASE, 'linkA')
+ self._check_resolve_relative(p, P(BASE, 'fileA'))
+ p = P(BASE, 'dirA', 'linkC', 'fileB')
+ self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB'))
+ p = P(BASE, 'dirB', 'linkD', 'fileB')
+ self._check_resolve_relative(p, P(BASE, 'dirB', 'fileB'))
+ # Now create absolute symlinks
+ d = tempfile.mkdtemp(suffix='-dirD')
+ self.addCleanup(shutil.rmtree, d)
+ os.symlink(os.path.join(d), join('dirA', 'linkX'))
+ os.symlink(join('dirB'), os.path.join(d, 'linkY'))
+ p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB')
+ self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB'))
+
+ @with_symlinks
+ def test_resolve_dot(self):
+ # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks
+ p = self.cls(BASE)
+ self.dirlink('.', join('0'))
+ self.dirlink(os.path.join('0', '0'), join('1'))
+ self.dirlink(os.path.join('1', '1'), join('2'))
+ q = p / '2'
+ self.assertEqual(q.resolve(), p)
+
+ def test_with(self):
+ p = self.cls(BASE)
+ it = p.iterdir()
+ it2 = p.iterdir()
+ next(it2)
+ with p:
+ pass
+ # I/O operation on closed path
+ self.assertRaises(ValueError, next, it)
+ self.assertRaises(ValueError, next, it2)
+ self.assertRaises(ValueError, p.open)
+ self.assertRaises(ValueError, p.resolve)
+ self.assertRaises(ValueError, p.absolute)
+ self.assertRaises(ValueError, p.__enter__)
+
+ def test_chmod(self):
+ p = self.cls(BASE) / 'fileA'
+ mode = p.stat().st_mode
+ # Clear writable bit
+ new_mode = mode & ~0o222
+ p.chmod(new_mode)
+ self.assertEqual(p.stat().st_mode, new_mode)
+ # Set writable bit
+ new_mode = mode | 0o222
+ p.chmod(new_mode)
+ self.assertEqual(p.stat().st_mode, new_mode)
+
+ # XXX also need a test for lchmod
+
+ def test_stat(self):
+ p = self.cls(BASE) / 'fileA'
+ st = p.stat()
+ self.assertEqual(p.stat(), st)
+ # Change file mode by flipping write bit
+ p.chmod(st.st_mode ^ 0o222)
+ self.addCleanup(p.chmod, st.st_mode)
+ self.assertNotEqual(p.stat(), st)
+
+ @with_symlinks
+ def test_lstat(self):
+ p = self.cls(BASE)/ 'linkA'
+ st = p.stat()
+ self.assertNotEqual(st, p.lstat())
+
+ def test_lstat_nosymlink(self):
+ p = self.cls(BASE) / 'fileA'
+ st = p.stat()
+ self.assertEqual(st, p.lstat())
+
+ @unittest.skipUnless(pwd, "the pwd module is needed for this test")
+ def test_owner(self):
+ p = self.cls(BASE) / 'fileA'
+ uid = p.stat().st_uid
+ try:
+ name = pwd.getpwuid(uid).pw_name
+ except KeyError:
+ self.skipTest(
+ "user %d doesn't have an entry in the system database" % uid)
+ self.assertEqual(name, p.owner())
+
+ @unittest.skipUnless(grp, "the grp module is needed for this test")
+ def test_group(self):
+ p = self.cls(BASE) / 'fileA'
+ gid = p.stat().st_gid
+ try:
+ name = grp.getgrgid(gid).gr_name
+ except KeyError:
+ self.skipTest(
+ "group %d doesn't have an entry in the system database" % gid)
+ self.assertEqual(name, p.group())
+
+ def test_unlink(self):
+ p = self.cls(BASE) / 'fileA'
+ p.unlink()
+ self.assertFileNotFound(p.stat)
+ self.assertFileNotFound(p.unlink)
+
+ def test_rmdir(self):
+ p = self.cls(BASE) / 'dirA'
+ for q in p.iterdir():
+ q.unlink()
+ p.rmdir()
+ self.assertFileNotFound(p.stat)
+ self.assertFileNotFound(p.unlink)
+
+ def test_rename(self):
+ P = self.cls(BASE)
+ p = P / 'fileA'
+ size = p.stat().st_size
+ # Renaming to another path
+ q = P / 'dirA' / 'fileAA'
+ p.rename(q)
+ self.assertEqual(q.stat().st_size, size)
+ self.assertFileNotFound(p.stat)
+ # Renaming to a str of a relative path
+ r = rel_join('fileAAA')
+ q.rename(r)
+ self.assertEqual(os.stat(r).st_size, size)
+ self.assertFileNotFound(q.stat)
+
+ def test_replace(self):
+ P = self.cls(BASE)
+ p = P / 'fileA'
+ size = p.stat().st_size
+ # Replacing a non-existing path
+ q = P / 'dirA' / 'fileAA'
+ p.replace(q)
+ self.assertEqual(q.stat().st_size, size)
+ self.assertFileNotFound(p.stat)
+ # Replacing another (existing) path
+ r = rel_join('dirB', 'fileB')
+ q.replace(r)
+ self.assertEqual(os.stat(r).st_size, size)
+ self.assertFileNotFound(q.stat)
+
+ def test_touch_common(self):
+ P = self.cls(BASE)
+ p = P / 'newfileA'
+ self.assertFalse(p.exists())
+ p.touch()
+ self.assertTrue(p.exists())
+ st = p.stat()
+ old_mtime = st.st_mtime
+ old_mtime_ns = st.st_mtime_ns
+ # Rewind the mtime sufficiently far in the past to work around
+ # filesystem-specific timestamp granularity.
+ os.utime(str(p), (old_mtime - 10, old_mtime - 10))
+ # The file mtime should be refreshed by calling touch() again
+ p.touch()
+ st = p.stat()
+ self.assertGreaterEqual(st.st_mtime_ns, old_mtime_ns)
+ self.assertGreaterEqual(st.st_mtime, old_mtime)
+ # Now with exist_ok=False
+ p = P / 'newfileB'
+ self.assertFalse(p.exists())
+ p.touch(mode=0o700, exist_ok=False)
+ self.assertTrue(p.exists())
+ self.assertRaises(OSError, p.touch, exist_ok=False)
+
+ def test_touch_nochange(self):
+ P = self.cls(BASE)
+ p = P / 'fileA'
+ p.touch()
+ with p.open('rb') as f:
+ self.assertEqual(f.read().strip(), b"this is file A")
+
+ def test_mkdir(self):
+ P = self.cls(BASE)
+ p = P / 'newdirA'
+ self.assertFalse(p.exists())
+ p.mkdir()
+ self.assertTrue(p.exists())
+ self.assertTrue(p.is_dir())
+ with self.assertRaises(OSError) as cm:
+ p.mkdir()
+ self.assertEqual(cm.exception.errno, errno.EEXIST)
+
+ def test_mkdir_parents(self):
+ # Creating a chain of directories
+ p = self.cls(BASE, 'newdirB', 'newdirC')
+ self.assertFalse(p.exists())
+ with self.assertRaises(OSError) as cm:
+ p.mkdir()
+ self.assertEqual(cm.exception.errno, errno.ENOENT)
+ p.mkdir(parents=True)
+ self.assertTrue(p.exists())
+ self.assertTrue(p.is_dir())
+ with self.assertRaises(OSError) as cm:
+ p.mkdir(parents=True)
+ self.assertEqual(cm.exception.errno, errno.EEXIST)
+ # test `mode` arg
+ mode = stat.S_IMODE(p.stat().st_mode) # default mode
+ p = self.cls(BASE, 'newdirD', 'newdirE')
+ p.mkdir(0o555, parents=True)
+ self.assertTrue(p.exists())
+ self.assertTrue(p.is_dir())
+ if os.name != 'nt':
+ # the directory's permissions follow the mode argument
+ self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o7555 & mode)
+ # the parent's permissions follow the default process settings
+ self.assertEqual(stat.S_IMODE(p.parent.stat().st_mode), mode)
+
+ @with_symlinks
+ def test_symlink_to(self):
+ P = self.cls(BASE)
+ target = P / 'fileA'
+ # Symlinking a path target
+ link = P / 'dirA' / 'linkAA'
+ link.symlink_to(target)
+ self.assertEqual(link.stat(), target.stat())
+ self.assertNotEqual(link.lstat(), target.stat())
+ # Symlinking a str target
+ link = P / 'dirA' / 'linkAAA'
+ link.symlink_to(str(target))
+ self.assertEqual(link.stat(), target.stat())
+ self.assertNotEqual(link.lstat(), target.stat())
+ self.assertFalse(link.is_dir())
+ # Symlinking to a directory
+ target = P / 'dirB'
+ link = P / 'dirA' / 'linkAAAA'
+ link.symlink_to(target, target_is_directory=True)
+ self.assertEqual(link.stat(), target.stat())
+ self.assertNotEqual(link.lstat(), target.stat())
+ self.assertTrue(link.is_dir())
+ self.assertTrue(list(link.iterdir()))
+
+ def test_is_dir(self):
+ P = self.cls(BASE)
+ self.assertTrue((P / 'dirA').is_dir())
+ self.assertFalse((P / 'fileA').is_dir())
+ self.assertFalse((P / 'non-existing').is_dir())
+ if not symlink_skip_reason:
+ self.assertFalse((P / 'linkA').is_dir())
+ self.assertTrue((P / 'linkB').is_dir())
+ self.assertFalse((P/ 'brokenLink').is_dir())
+
+ def test_is_file(self):
+ P = self.cls(BASE)
+ self.assertTrue((P / 'fileA').is_file())
+ self.assertFalse((P / 'dirA').is_file())
+ self.assertFalse((P / 'non-existing').is_file())
+ if not symlink_skip_reason:
+ self.assertTrue((P / 'linkA').is_file())
+ self.assertFalse((P / 'linkB').is_file())
+ self.assertFalse((P/ 'brokenLink').is_file())
+
+ def test_is_symlink(self):
+ P = self.cls(BASE)
+ self.assertFalse((P / 'fileA').is_symlink())
+ self.assertFalse((P / 'dirA').is_symlink())
+ self.assertFalse((P / 'non-existing').is_symlink())
+ if not symlink_skip_reason:
+ self.assertTrue((P / 'linkA').is_symlink())
+ self.assertTrue((P / 'linkB').is_symlink())
+ self.assertTrue((P/ 'brokenLink').is_symlink())
+
+ def test_is_fifo_false(self):
+ P = self.cls(BASE)
+ self.assertFalse((P / 'fileA').is_fifo())
+ self.assertFalse((P / 'dirA').is_fifo())
+ self.assertFalse((P / 'non-existing').is_fifo())
+
+ @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required")
+ def test_is_fifo_true(self):
+ P = self.cls(BASE, 'myfifo')
+ os.mkfifo(str(P))
+ self.assertTrue(P.is_fifo())
+ self.assertFalse(P.is_socket())
+ self.assertFalse(P.is_file())
+
+ def test_is_socket_false(self):
+ P = self.cls(BASE)
+ self.assertFalse((P / 'fileA').is_socket())
+ self.assertFalse((P / 'dirA').is_socket())
+ self.assertFalse((P / 'non-existing').is_socket())
+
+ @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required")
+ def test_is_socket_true(self):
+ P = self.cls(BASE, 'mysock')
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ self.addCleanup(sock.close)
+ try:
+ sock.bind(str(P))
+ except OSError as e:
+ if "AF_UNIX path too long" in str(e):
+ self.skipTest("cannot bind Unix socket: " + str(e))
+ self.assertTrue(P.is_socket())
+ self.assertFalse(P.is_fifo())
+ self.assertFalse(P.is_file())
+
+ def test_is_block_device_false(self):
+ P = self.cls(BASE)
+ self.assertFalse((P / 'fileA').is_block_device())
+ self.assertFalse((P / 'dirA').is_block_device())
+ self.assertFalse((P / 'non-existing').is_block_device())
+
+ def test_is_char_device_false(self):
+ P = self.cls(BASE)
+ self.assertFalse((P / 'fileA').is_char_device())
+ self.assertFalse((P / 'dirA').is_char_device())
+ self.assertFalse((P / 'non-existing').is_char_device())
+
+ def test_is_char_device_true(self):
+ # Under Unix, /dev/null should generally be a char device
+ P = self.cls('/dev/null')
+ if not P.exists():
+ self.skipTest("/dev/null required")
+ self.assertTrue(P.is_char_device())
+ self.assertFalse(P.is_block_device())
+ self.assertFalse(P.is_file())
+
+ def test_pickling_common(self):
+ p = self.cls(BASE, 'fileA')
+ for proto in range(0, pickle.HIGHEST_PROTOCOL + 1):
+ dumped = pickle.dumps(p, proto)
+ pp = pickle.loads(dumped)
+ self.assertEqual(pp.stat(), p.stat())
+
+ def test_parts_interning(self):
+ P = self.cls
+ p = P('/usr/bin/foo')
+ q = P('/usr/local/bin')
+ # 'usr'
+ self.assertIs(p.parts[1], q.parts[1])
+ # 'bin'
+ self.assertIs(p.parts[2], q.parts[3])
+
+ def _check_complex_symlinks(self, link0_target):
+ # Test solving a non-looping chain of symlinks (issue #19887)
+ P = self.cls(BASE)
+ self.dirlink(os.path.join('link0', 'link0'), join('link1'))
+ self.dirlink(os.path.join('link1', 'link1'), join('link2'))
+ self.dirlink(os.path.join('link2', 'link2'), join('link3'))
+ self.dirlink(link0_target, join('link0'))
+
+ # Resolve absolute paths
+ p = (P / 'link0').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = (P / 'link1').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = (P / 'link2').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = (P / 'link3').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+
+ # Resolve relative paths
+ old_path = os.getcwd()
+ os.chdir(BASE)
+ try:
+ p = self.cls('link0').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = self.cls('link1').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = self.cls('link2').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ p = self.cls('link3').resolve()
+ self.assertEqual(p, P)
+ self.assertEqual(str(p), BASE)
+ finally:
+ os.chdir(old_path)
+
+ @with_symlinks
+ def test_complex_symlinks_absolute(self):
+ self._check_complex_symlinks(BASE)
+
+ @with_symlinks
+ def test_complex_symlinks_relative(self):
+ self._check_complex_symlinks('.')
+
+ @with_symlinks
+ def test_complex_symlinks_relative_dot_dot(self):
+ self._check_complex_symlinks(os.path.join('dirA', '..'))
+
+
+class PathTest(_BasePathTest, unittest.TestCase):
+ cls = pathlib.Path
+
+ def test_concrete_class(self):
+ p = self.cls('a')
+ self.assertIs(type(p),
+ pathlib.WindowsPath if os.name == 'nt' else pathlib.PosixPath)
+
+ def test_unsupported_flavour(self):
+ if os.name == 'nt':
+ self.assertRaises(NotImplementedError, pathlib.PosixPath)
+ else:
+ self.assertRaises(NotImplementedError, pathlib.WindowsPath)
+
+
+@only_posix
+class PosixPathTest(_BasePathTest, unittest.TestCase):
+ cls = pathlib.PosixPath
+
+ def _check_symlink_loop(self, *args):
+ path = self.cls(*args)
+ with self.assertRaises(RuntimeError):
+ print(path.resolve())
+
+ def test_open_mode(self):
+ old_mask = os.umask(0)
+ self.addCleanup(os.umask, old_mask)
+ p = self.cls(BASE)
+ with (p / 'new_file').open('wb'):
+ pass
+ st = os.stat(join('new_file'))
+ self.assertEqual(stat.S_IMODE(st.st_mode), 0o666)
+ os.umask(0o022)
+ with (p / 'other_new_file').open('wb'):
+ pass
+ st = os.stat(join('other_new_file'))
+ self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
+
+ def test_touch_mode(self):
+ old_mask = os.umask(0)
+ self.addCleanup(os.umask, old_mask)
+ p = self.cls(BASE)
+ (p / 'new_file').touch()
+ st = os.stat(join('new_file'))
+ self.assertEqual(stat.S_IMODE(st.st_mode), 0o666)
+ os.umask(0o022)
+ (p / 'other_new_file').touch()
+ st = os.stat(join('other_new_file'))
+ self.assertEqual(stat.S_IMODE(st.st_mode), 0o644)
+ (p / 'masked_new_file').touch(mode=0o750)
+ st = os.stat(join('masked_new_file'))
+ self.assertEqual(stat.S_IMODE(st.st_mode), 0o750)
+
+ @with_symlinks
+ def test_resolve_loop(self):
+ # Loop detection for broken symlinks under POSIX
+ P = self.cls
+ # Loops with relative symlinks
+ os.symlink('linkX/inside', join('linkX'))
+ self._check_symlink_loop(BASE, 'linkX')
+ os.symlink('linkY', join('linkY'))
+ self._check_symlink_loop(BASE, 'linkY')
+ os.symlink('linkZ/../linkZ', join('linkZ'))
+ self._check_symlink_loop(BASE, 'linkZ')
+ # Loops with absolute symlinks
+ os.symlink(join('linkU/inside'), join('linkU'))
+ self._check_symlink_loop(BASE, 'linkU')
+ os.symlink(join('linkV'), join('linkV'))
+ self._check_symlink_loop(BASE, 'linkV')
+ os.symlink(join('linkW/../linkW'), join('linkW'))
+ self._check_symlink_loop(BASE, 'linkW')
+
+ def test_glob(self):
+ P = self.cls
+ p = P(BASE)
+ given = set(p.glob("FILEa"))
+ expect = set() if not support.fs_is_case_insensitive(BASE) else given
+ self.assertEqual(given, expect)
+ self.assertEqual(set(p.glob("FILEa*")), set())
+
+ def test_rglob(self):
+ P = self.cls
+ p = P(BASE, "dirC")
+ given = set(p.rglob("FILEd"))
+ expect = set() if not support.fs_is_case_insensitive(BASE) else given
+ self.assertEqual(given, expect)
+ self.assertEqual(set(p.rglob("FILEd*")), set())
+
+
+@only_nt
+class WindowsPathTest(_BasePathTest, unittest.TestCase):
+ cls = pathlib.WindowsPath
+
+ def test_glob(self):
+ P = self.cls
+ p = P(BASE)
+ self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") })
+
+ def test_rglob(self):
+ P = self.cls
+ p = P(BASE, "dirC")
+ self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") })
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 03084e4..74253b3 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -1,9 +1,9 @@
# A test suite for pdb; not very comprehensive at the moment.
import doctest
-import imp
import pdb
import sys
+import types
import unittest
import subprocess
import textwrap
@@ -205,7 +205,8 @@ def test_pdb_breakpoint_commands():
... 'enable 1',
... 'clear 1',
... 'commands 2',
- ... 'print 42',
+ ... 'p "42"',
+ ... 'print("42", 7*6)', # Issue 18764 (not about breakpoints)
... 'end',
... 'continue', # will stop at breakpoint 2 (line 4)
... 'clear', # clear all!
@@ -252,11 +253,13 @@ def test_pdb_breakpoint_commands():
(Pdb) clear 1
Deleted breakpoint 1 at <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>:3
(Pdb) commands 2
- (com) print 42
+ (com) p "42"
+ (com) print("42", 7*6)
(com) end
(Pdb) continue
1
- 42
+ '42'
+ 42 42
> <doctest test.test_pdb.test_pdb_breakpoint_commands[0]>(4)test_function()
-> print(2)
(Pdb) clear
@@ -464,7 +467,7 @@ def test_pdb_skip_modules():
# Module for testing skipping of module that makes a callback
-mod = imp.new_module('module_to_skip')
+mod = types.ModuleType('module_to_skip')
exec('def foo_pony(callback): x = 1; callback(); return None', mod.__dict__)
@@ -597,6 +600,311 @@ def test_pdb_run_with_code_object():
(Pdb) continue
"""
+def test_next_until_return_at_return_event():
+ """Test that pdb stops after a next/until/return issued at a return debug event.
+
+ >>> def test_function_2():
+ ... x = 1
+ ... x = 2
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... test_function_2()
+ ... test_function_2()
+ ... test_function_2()
+ ... end = 1
+
+ >>> with PdbTestInput(['break test_function_2',
+ ... 'continue',
+ ... 'return',
+ ... 'next',
+ ... 'continue',
+ ... 'return',
+ ... 'until',
+ ... 'continue',
+ ... 'return',
+ ... 'return',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[1]>(3)test_function()
+ -> test_function_2()
+ (Pdb) break test_function_2
+ Breakpoint 1 at <doctest test.test_pdb.test_next_until_return_at_return_event[0]>:1
+ (Pdb) continue
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(2)test_function_2()
+ -> x = 1
+ (Pdb) return
+ --Return--
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(3)test_function_2()->None
+ -> x = 2
+ (Pdb) next
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[1]>(4)test_function()
+ -> test_function_2()
+ (Pdb) continue
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(2)test_function_2()
+ -> x = 1
+ (Pdb) return
+ --Return--
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(3)test_function_2()->None
+ -> x = 2
+ (Pdb) until
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[1]>(5)test_function()
+ -> test_function_2()
+ (Pdb) continue
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(2)test_function_2()
+ -> x = 1
+ (Pdb) return
+ --Return--
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[0]>(3)test_function_2()->None
+ -> x = 2
+ (Pdb) return
+ > <doctest test.test_pdb.test_next_until_return_at_return_event[1]>(6)test_function()
+ -> end = 1
+ (Pdb) continue
+ """
+
+def test_pdb_next_command_for_generator():
+ """Testing skip unwindng stack on yield for generators for "next" command
+
+ >>> def test_gen():
+ ... yield 0
+ ... return 1
+ ... yield 2
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... it = test_gen()
+ ... try:
+ ... assert next(it) == 0
+ ... next(it)
+ ... except StopIteration as ex:
+ ... assert ex.value == 1
+ ... print("finished")
+
+ >>> with PdbTestInput(['step',
+ ... 'step',
+ ... 'step',
+ ... 'next',
+ ... 'next',
+ ... 'step',
+ ... 'step',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[1]>(3)test_function()
+ -> it = test_gen()
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[1]>(4)test_function()
+ -> try:
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[1]>(5)test_function()
+ -> assert next(it) == 0
+ (Pdb) step
+ --Call--
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[0]>(1)test_gen()
+ -> def test_gen():
+ (Pdb) next
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[0]>(2)test_gen()
+ -> yield 0
+ (Pdb) next
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[0]>(3)test_gen()
+ -> return 1
+ (Pdb) step
+ --Return--
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[0]>(3)test_gen()->1
+ -> return 1
+ (Pdb) step
+ StopIteration: 1
+ > <doctest test.test_pdb.test_pdb_next_command_for_generator[1]>(6)test_function()
+ -> next(it)
+ (Pdb) continue
+ finished
+ """
+
+def test_pdb_return_command_for_generator():
+ """Testing no unwindng stack on yield for generators
+ for "return" command
+
+ >>> def test_gen():
+ ... yield 0
+ ... return 1
+ ... yield 2
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... it = test_gen()
+ ... try:
+ ... assert next(it) == 0
+ ... next(it)
+ ... except StopIteration as ex:
+ ... assert ex.value == 1
+ ... print("finished")
+
+ >>> with PdbTestInput(['step',
+ ... 'step',
+ ... 'step',
+ ... 'return',
+ ... 'step',
+ ... 'step',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(3)test_function()
+ -> it = test_gen()
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(4)test_function()
+ -> try:
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(5)test_function()
+ -> assert next(it) == 0
+ (Pdb) step
+ --Call--
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[0]>(1)test_gen()
+ -> def test_gen():
+ (Pdb) return
+ StopIteration: 1
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(6)test_function()
+ -> next(it)
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(7)test_function()
+ -> except StopIteration as ex:
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_return_command_for_generator[1]>(8)test_function()
+ -> assert ex.value == 1
+ (Pdb) continue
+ finished
+ """
+
+def test_pdb_until_command_for_generator():
+ """Testing no unwindng stack on yield for generators
+ for "until" command if target breakpoing is not reached
+
+ >>> def test_gen():
+ ... yield 0
+ ... yield 1
+ ... yield 2
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... for i in test_gen():
+ ... print(i)
+ ... print("finished")
+
+ >>> with PdbTestInput(['step',
+ ... 'until 4',
+ ... 'step',
+ ... 'step',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_pdb_until_command_for_generator[1]>(3)test_function()
+ -> for i in test_gen():
+ (Pdb) step
+ --Call--
+ > <doctest test.test_pdb.test_pdb_until_command_for_generator[0]>(1)test_gen()
+ -> def test_gen():
+ (Pdb) until 4
+ 0
+ 1
+ > <doctest test.test_pdb.test_pdb_until_command_for_generator[0]>(4)test_gen()
+ -> yield 2
+ (Pdb) step
+ --Return--
+ > <doctest test.test_pdb.test_pdb_until_command_for_generator[0]>(4)test_gen()->2
+ -> yield 2
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_until_command_for_generator[1]>(4)test_function()
+ -> print(i)
+ (Pdb) continue
+ 2
+ finished
+ """
+
+def test_pdb_next_command_in_generator_for_loop():
+ """The next command on returning from a generator controled by a for loop.
+
+ >>> def test_gen():
+ ... yield 0
+ ... return 1
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... for i in test_gen():
+ ... print('value', i)
+ ... x = 123
+
+ >>> with PdbTestInput(['break test_gen',
+ ... 'continue',
+ ... 'next',
+ ... 'next',
+ ... 'next',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[1]>(3)test_function()
+ -> for i in test_gen():
+ (Pdb) break test_gen
+ Breakpoint 6 at <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[0]>:1
+ (Pdb) continue
+ > <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[0]>(2)test_gen()
+ -> yield 0
+ (Pdb) next
+ value 0
+ > <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[0]>(3)test_gen()
+ -> return 1
+ (Pdb) next
+ Internal StopIteration: 1
+ > <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[1]>(3)test_function()
+ -> for i in test_gen():
+ (Pdb) next
+ > <doctest test.test_pdb.test_pdb_next_command_in_generator_for_loop[1]>(5)test_function()
+ -> x = 123
+ (Pdb) continue
+ """
+
+def test_pdb_next_command_subiterator():
+ """The next command in a generator with a subiterator.
+
+ >>> def test_subgenerator():
+ ... yield 0
+ ... return 1
+
+ >>> def test_gen():
+ ... x = yield from test_subgenerator()
+ ... return x
+
+ >>> def test_function():
+ ... import pdb; pdb.Pdb(nosigint=True).set_trace()
+ ... for i in test_gen():
+ ... print('value', i)
+ ... x = 123
+
+ >>> with PdbTestInput(['step',
+ ... 'step',
+ ... 'next',
+ ... 'next',
+ ... 'next',
+ ... 'continue']):
+ ... test_function()
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[2]>(3)test_function()
+ -> for i in test_gen():
+ (Pdb) step
+ --Call--
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[1]>(1)test_gen()
+ -> def test_gen():
+ (Pdb) step
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[1]>(2)test_gen()
+ -> x = yield from test_subgenerator()
+ (Pdb) next
+ value 0
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[1]>(3)test_gen()
+ -> return x
+ (Pdb) next
+ Internal StopIteration: 1
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[2]>(3)test_function()
+ -> for i in test_gen():
+ (Pdb) next
+ > <doctest test.test_pdb.test_pdb_next_command_subiterator[2]>(5)test_function()
+ -> x = 123
+ (Pdb) continue
+ """
+
class PdbTestCase(unittest.TestCase):
@@ -617,6 +925,36 @@ class PdbTestCase(unittest.TestCase):
stderr = stderr and bytes.decode(stderr)
return stdout, stderr
+ def _assert_find_function(self, file_content, func_name, expected):
+ file_content = textwrap.dedent(file_content)
+
+ with open(support.TESTFN, 'w') as f:
+ f.write(file_content)
+
+ expected = None if not expected else (
+ expected[0], support.TESTFN, expected[1])
+ self.assertEqual(
+ expected, pdb.find_function(func_name, support.TESTFN))
+
+ def test_find_function_empty_file(self):
+ self._assert_find_function('', 'foo', None)
+
+ def test_find_function_found(self):
+ self._assert_find_function(
+ """\
+ def foo():
+ pass
+
+ def bar():
+ pass
+
+ def quux():
+ pass
+ """,
+ 'bar',
+ ('bar', 4),
+ )
+
def test_issue7964(self):
# open the file as binary so we can force \r\n newline
with open(support.TESTFN, 'wb') as f:
diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index 1cacdea..5025792 100644
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -5,44 +5,28 @@ from io import StringIO
import unittest
from math import copysign
-def disassemble(func):
- f = StringIO()
- tmp = sys.stdout
- sys.stdout = f
- try:
- dis.dis(func)
- finally:
- sys.stdout = tmp
- result = f.getvalue()
- f.close()
- return result
+from test.bytecode_helper import BytecodeTestCase
-def dis_single(line):
- return disassemble(compile(line, '', 'single'))
-
-
-class TestTranforms(unittest.TestCase):
+class TestTranforms(BytecodeTestCase):
def test_unot(self):
# UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE'
def unot(x):
if not x == 2:
del x
- asm = disassemble(unot)
- for elem in ('UNARY_NOT', 'POP_JUMP_IF_FALSE'):
- self.assertNotIn(elem, asm)
- for elem in ('POP_JUMP_IF_TRUE',):
- self.assertIn(elem, asm)
+ self.assertNotInBytecode(unot, 'UNARY_NOT')
+ self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE')
+ self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE')
def test_elim_inversion_of_is_or_in(self):
- for line, elem in (
- ('not a is b', '(is not)',),
- ('not a in b', '(not in)',),
- ('not a is not b', '(is)',),
- ('not a not in b', '(in)',),
+ for line, cmp_op in (
+ ('not a is b', 'is not',),
+ ('not a in b', 'not in',),
+ ('not a is not b', 'is',),
+ ('not a not in b', 'in',),
):
- asm = dis_single(line)
- self.assertIn(elem, asm)
+ code = compile(line, '', 'single')
+ self.assertInBytecode(code, 'COMPARE_OP', cmp_op)
def test_global_as_constant(self):
# LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False
@@ -56,17 +40,14 @@ class TestTranforms(unittest.TestCase):
def h(x):
False
return x
- for func, name in ((f, 'None'), (g, 'True'), (h, 'False')):
- asm = disassemble(func)
- for elem in ('LOAD_GLOBAL',):
- self.assertNotIn(elem, asm)
- for elem in ('LOAD_CONST', '('+name+')'):
- self.assertIn(elem, asm)
+ for func, elem in ((f, None), (g, True), (h, False)):
+ self.assertNotInBytecode(func, 'LOAD_GLOBAL')
+ self.assertInBytecode(func, 'LOAD_CONST', elem)
def f():
'Adding a docstring made this test fail in Py2.5.0'
return None
- self.assertIn('LOAD_CONST', disassemble(f))
- self.assertNotIn('LOAD_GLOBAL', disassemble(f))
+ self.assertNotInBytecode(f, 'LOAD_GLOBAL')
+ self.assertInBytecode(f, 'LOAD_CONST', None)
def test_while_one(self):
# Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx
@@ -74,11 +55,10 @@ class TestTranforms(unittest.TestCase):
while 1:
pass
return list
- asm = disassemble(f)
for elem in ('LOAD_CONST', 'POP_JUMP_IF_FALSE'):
- self.assertNotIn(elem, asm)
+ self.assertNotInBytecode(f, elem)
for elem in ('JUMP_ABSOLUTE',):
- self.assertIn(elem, asm)
+ self.assertInBytecode(f, elem)
def test_pack_unpack(self):
for line, elem in (
@@ -86,28 +66,30 @@ class TestTranforms(unittest.TestCase):
('a, b = a, b', 'ROT_TWO',),
('a, b, c = a, b, c', 'ROT_THREE',),
):
- asm = dis_single(line)
- self.assertIn(elem, asm)
- self.assertNotIn('BUILD_TUPLE', asm)
- self.assertNotIn('UNPACK_TUPLE', asm)
+ code = compile(line,'','single')
+ self.assertInBytecode(code, elem)
+ self.assertNotInBytecode(code, 'BUILD_TUPLE')
+ self.assertNotInBytecode(code, 'UNPACK_TUPLE')
def test_folding_of_tuples_of_constants(self):
for line, elem in (
- ('a = 1,2,3', '((1, 2, 3))'),
- ('("a","b","c")', "(('a', 'b', 'c'))"),
- ('a,b,c = 1,2,3', '((1, 2, 3))'),
- ('(None, 1, None)', '((None, 1, None))'),
- ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
+ ('a = 1,2,3', (1, 2, 3)),
+ ('("a","b","c")', ('a', 'b', 'c')),
+ ('a,b,c = 1,2,3', (1, 2, 3)),
+ ('(None, 1, None)', (None, 1, None)),
+ ('((1, 2), 3, 4)', ((1, 2), 3, 4)),
):
- asm = dis_single(line)
- self.assertIn(elem, asm)
- self.assertNotIn('BUILD_TUPLE', asm)
+ code = compile(line,'','single')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
+ self.assertNotInBytecode(code, 'BUILD_TUPLE')
# Long tuples should be folded too.
- asm = dis_single(repr(tuple(range(10000))))
+ code = compile(repr(tuple(range(10000))),'','single')
+ self.assertNotInBytecode(code, 'BUILD_TUPLE')
# One LOAD_CONST for the tuple, one for the None return value
- self.assertEqual(asm.count('LOAD_CONST'), 2)
- self.assertNotIn('BUILD_TUPLE', asm)
+ load_consts = [instr for instr in dis.get_instructions(code)
+ if instr.opname == 'LOAD_CONST']
+ self.assertEqual(len(load_consts), 2)
# Bug 1053819: Tuple of constants misidentified when presented with:
# . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . .
@@ -129,14 +111,14 @@ class TestTranforms(unittest.TestCase):
def test_folding_of_lists_of_constants(self):
for line, elem in (
# in/not in constants with BUILD_LIST should be folded to a tuple:
- ('a in [1,2,3]', '(1, 2, 3)'),
- ('a not in ["a","b","c"]', "(('a', 'b', 'c'))"),
- ('a in [None, 1, None]', '((None, 1, None))'),
- ('a not in [(1, 2), 3, 4]', '(((1, 2), 3, 4))'),
+ ('a in [1,2,3]', (1, 2, 3)),
+ ('a not in ["a","b","c"]', ('a', 'b', 'c')),
+ ('a in [None, 1, None]', (None, 1, None)),
+ ('a not in [(1, 2), 3, 4]', ((1, 2), 3, 4)),
):
- asm = dis_single(line)
- self.assertIn(elem, asm)
- self.assertNotIn('BUILD_LIST', asm)
+ code = compile(line, '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
+ self.assertNotInBytecode(code, 'BUILD_LIST')
def test_folding_of_sets_of_constants(self):
for line, elem in (
@@ -147,18 +129,9 @@ class TestTranforms(unittest.TestCase):
('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})),
('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})),
):
- asm = dis_single(line)
- self.assertNotIn('BUILD_SET', asm)
-
- # Verify that the frozenset 'elem' is in the disassembly
- # The ordering of the elements in repr( frozenset ) isn't
- # guaranteed, so we jump through some hoops to ensure that we have
- # the frozenset we expect:
- self.assertIn('frozenset', asm)
- # Extract the frozenset literal from the disassembly:
- m = re.match(r'.*(frozenset\({.*}\)).*', asm, re.DOTALL)
- self.assertTrue(m)
- self.assertEqual(eval(m.group(1)), elem)
+ code = compile(line, '', 'single')
+ self.assertNotInBytecode(code, 'BUILD_SET')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
# Ensure that the resulting code actually works:
def f(a):
@@ -176,98 +149,103 @@ class TestTranforms(unittest.TestCase):
def test_folding_of_binops_on_constants(self):
for line, elem in (
- ('a = 2+3+4', '(9)'), # chained fold
- ('"@"*4', "('@@@@')"), # check string ops
- ('a="abc" + "def"', "('abcdef')"), # check string ops
- ('a = 3**4', '(81)'), # binary power
- ('a = 3*4', '(12)'), # binary multiply
- ('a = 13//4', '(3)'), # binary floor divide
- ('a = 14%4', '(2)'), # binary modulo
- ('a = 2+3', '(5)'), # binary add
- ('a = 13-4', '(9)'), # binary subtract
- ('a = (12,13)[1]', '(13)'), # binary subscr
- ('a = 13 << 2', '(52)'), # binary lshift
- ('a = 13 >> 2', '(3)'), # binary rshift
- ('a = 13 & 7', '(5)'), # binary and
- ('a = 13 ^ 7', '(10)'), # binary xor
- ('a = 13 | 7', '(15)'), # binary or
+ ('a = 2+3+4', 9), # chained fold
+ ('"@"*4', '@@@@'), # check string ops
+ ('a="abc" + "def"', 'abcdef'), # check string ops
+ ('a = 3**4', 81), # binary power
+ ('a = 3*4', 12), # binary multiply
+ ('a = 13//4', 3), # binary floor divide
+ ('a = 14%4', 2), # binary modulo
+ ('a = 2+3', 5), # binary add
+ ('a = 13-4', 9), # binary subtract
+ ('a = (12,13)[1]', 13), # binary subscr
+ ('a = 13 << 2', 52), # binary lshift
+ ('a = 13 >> 2', 3), # binary rshift
+ ('a = 13 & 7', 5), # binary and
+ ('a = 13 ^ 7', 10), # binary xor
+ ('a = 13 | 7', 15), # binary or
):
- asm = dis_single(line)
- self.assertIn(elem, asm, asm)
- self.assertNotIn('BINARY_', asm)
+ code = compile(line, '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
+ for instr in dis.get_instructions(code):
+ self.assertFalse(instr.opname.startswith('BINARY_'))
# Verify that unfoldables are skipped
- asm = dis_single('a=2+"b"')
- self.assertIn('(2)', asm)
- self.assertIn("('b')", asm)
+ code = compile('a=2+"b"', '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', 2)
+ self.assertInBytecode(code, 'LOAD_CONST', 'b')
# Verify that large sequences do not result from folding
- asm = dis_single('a="x"*1000')
- self.assertIn('(1000)', asm)
+ code = compile('a="x"*1000', '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', 1000)
def test_binary_subscr_on_unicode(self):
# valid code get optimized
- asm = dis_single('"foo"[0]')
- self.assertIn("('f')", asm)
- self.assertNotIn('BINARY_SUBSCR', asm)
- asm = dis_single('"\u0061\uffff"[1]')
- self.assertIn("('\\uffff')", asm)
- self.assertNotIn('BINARY_SUBSCR', asm)
- asm = dis_single('"\U00012345abcdef"[3]')
- self.assertIn("('c')", asm)
- self.assertNotIn('BINARY_SUBSCR', asm)
+ code = compile('"foo"[0]', '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', 'f')
+ self.assertNotInBytecode(code, 'BINARY_SUBSCR')
+ code = compile('"\u0061\uffff"[1]', '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', '\uffff')
+ self.assertNotInBytecode(code,'BINARY_SUBSCR')
+
+ # With PEP 393, non-BMP char get optimized
+ code = compile('"\U00012345"[0]', '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', '\U00012345')
+ self.assertNotInBytecode(code, 'BINARY_SUBSCR')
# invalid code doesn't get optimized
# out of range
- asm = dis_single('"fuu"[10]')
- self.assertIn('BINARY_SUBSCR', asm)
+ code = compile('"fuu"[10]', '', 'single')
+ self.assertInBytecode(code, 'BINARY_SUBSCR')
def test_folding_of_unaryops_on_constants(self):
for line, elem in (
- ('-0.5', '(-0.5)'), # unary negative
- ('-0.0', '(-0.0)'), # -0.0
- ('-(1.0-1.0)','(-0.0)'), # -0.0 after folding
- ('-0', '(0)'), # -0
- ('~-2', '(1)'), # unary invert
- ('+1', '(1)'), # unary positive
+ ('-0.5', -0.5), # unary negative
+ ('-0.0', -0.0), # -0.0
+ ('-(1.0-1.0)', -0.0), # -0.0 after folding
+ ('-0', 0), # -0
+ ('~-2', 1), # unary invert
+ ('+1', 1), # unary positive
):
- asm = dis_single(line)
- self.assertIn(elem, asm, asm)
- self.assertNotIn('UNARY_', asm)
+ code = compile(line, '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
+ for instr in dis.get_instructions(code):
+ self.assertFalse(instr.opname.startswith('UNARY_'))
# Check that -0.0 works after marshaling
def negzero():
return -(1.0-1.0)
- self.assertNotIn('UNARY_', disassemble(negzero))
- self.assertTrue(copysign(1.0, negzero()) < 0)
+ for instr in dis.get_instructions(code):
+ self.assertFalse(instr.opname.startswith('UNARY_'))
# Verify that unfoldables are skipped
- for line, elem in (
- ('-"abc"', "('abc')"), # unary negative
- ('~"abc"', "('abc')"), # unary invert
+ for line, elem, opname in (
+ ('-"abc"', 'abc', 'UNARY_NEGATIVE'),
+ ('~"abc"', 'abc', 'UNARY_INVERT'),
):
- asm = dis_single(line)
- self.assertIn(elem, asm, asm)
- self.assertIn('UNARY_', asm)
+ code = compile(line, '', 'single')
+ self.assertInBytecode(code, 'LOAD_CONST', elem)
+ self.assertInBytecode(code, opname)
def test_elim_extra_return(self):
# RETURN LOAD_CONST None RETURN --> RETURN
def f(x):
return x
- asm = disassemble(f)
- self.assertNotIn('LOAD_CONST', asm)
- self.assertNotIn('(None)', asm)
- self.assertEqual(asm.split().count('RETURN_VALUE'), 1)
+ self.assertNotInBytecode(f, 'LOAD_CONST', None)
+ returns = [instr for instr in dis.get_instructions(f)
+ if instr.opname == 'RETURN_VALUE']
+ self.assertEqual(len(returns), 1)
def test_elim_jump_to_return(self):
# JUMP_FORWARD to RETURN --> RETURN
def f(cond, true_value, false_value):
return true_value if cond else false_value
- asm = disassemble(f)
- self.assertNotIn('JUMP_FORWARD', asm)
- self.assertNotIn('JUMP_ABSOLUTE', asm)
- self.assertEqual(asm.split().count('RETURN_VALUE'), 2)
+ self.assertNotInBytecode(f, 'JUMP_FORWARD')
+ self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
+ returns = [instr for instr in dis.get_instructions(f)
+ if instr.opname == 'RETURN_VALUE']
+ self.assertEqual(len(returns), 2)
def test_elim_jump_after_return1(self):
# Eliminate dead code: jumps immediately after returns can't be reached
@@ -280,48 +258,53 @@ class TestTranforms(unittest.TestCase):
if cond1: return 4
return 5
return 6
- asm = disassemble(f)
- self.assertNotIn('JUMP_FORWARD', asm)
- self.assertNotIn('JUMP_ABSOLUTE', asm)
- self.assertEqual(asm.split().count('RETURN_VALUE'), 6)
+ self.assertNotInBytecode(f, 'JUMP_FORWARD')
+ self.assertNotInBytecode(f, 'JUMP_ABSOLUTE')
+ returns = [instr for instr in dis.get_instructions(f)
+ if instr.opname == 'RETURN_VALUE']
+ self.assertEqual(len(returns), 6)
def test_elim_jump_after_return2(self):
# Eliminate dead code: jumps immediately after returns can't be reached
def f(cond1, cond2):
while 1:
if cond1: return 4
- asm = disassemble(f)
- self.assertNotIn('JUMP_FORWARD', asm)
+ self.assertNotInBytecode(f, 'JUMP_FORWARD')
# There should be one jump for the while loop.
- self.assertEqual(asm.split().count('JUMP_ABSOLUTE'), 1)
- self.assertEqual(asm.split().count('RETURN_VALUE'), 2)
+ returns = [instr for instr in dis.get_instructions(f)
+ if instr.opname == 'JUMP_ABSOLUTE']
+ self.assertEqual(len(returns), 1)
+ returns = [instr for instr in dis.get_instructions(f)
+ if instr.opname == 'RETURN_VALUE']
+ self.assertEqual(len(returns), 2)
def test_make_function_doesnt_bail(self):
def f():
def g()->1+1:
pass
return g
- asm = disassemble(f)
- self.assertNotIn('BINARY_ADD', asm)
+ self.assertNotInBytecode(f, 'BINARY_ADD')
def test_constant_folding(self):
# Issue #11244: aggressive constant folding.
exprs = [
- "3 * -5",
- "-3 * 5",
- "2 * (3 * 4)",
- "(2 * 3) * 4",
- "(-1, 2, 3)",
- "(1, -2, 3)",
- "(1, 2, -3)",
- "(1, 2, -3) * 6",
- "lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}",
+ '3 * -5',
+ '-3 * 5',
+ '2 * (3 * 4)',
+ '(2 * 3) * 4',
+ '(-1, 2, 3)',
+ '(1, -2, 3)',
+ '(1, 2, -3)',
+ '(1, 2, -3) * 6',
+ 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}',
]
for e in exprs:
- asm = dis_single(e)
- self.assertNotIn('UNARY_', asm, e)
- self.assertNotIn('BINARY_', asm, e)
- self.assertNotIn('BUILD_', asm, e)
+ code = compile(e, '', 'single')
+ for instr in dis.get_instructions(code):
+ self.assertFalse(instr.opname.startswith('UNARY_'))
+ self.assertFalse(instr.opname.startswith('BINARY_'))
+ self.assertFalse(instr.opname.startswith('BUILD_'))
+
class TestBuglets(unittest.TestCase):
@@ -343,7 +326,7 @@ def test_main(verbose=None):
support.run_unittest(*test_classes)
# verify reference counting
- if verbose and hasattr(sys, "gettotalrefcount"):
+ if verbose and hasattr(sys, 'gettotalrefcount'):
import gc
counts = [None] * 5
for i in range(len(counts)):
diff --git a/Lib/test/test_pep247.py b/Lib/test/test_pep247.py
index 7f10472..b85a26a 100644
--- a/Lib/test/test_pep247.py
+++ b/Lib/test/test_pep247.py
@@ -15,12 +15,14 @@ class Pep247Test(unittest.TestCase):
self.assertTrue(module.digest_size is None or module.digest_size > 0)
self.check_object(module.new, module.digest_size, key)
- def check_object(self, cls, digest_size, key):
+ def check_object(self, cls, digest_size, key, digestmod=None):
if key is not None:
- obj1 = cls(key)
- obj2 = cls(key, b'string')
- h1 = cls(key, b'string').digest()
- obj3 = cls(key)
+ if digestmod is None:
+ digestmod = md5
+ obj1 = cls(key, digestmod=digestmod)
+ obj2 = cls(key, b'string', digestmod=digestmod)
+ h1 = cls(key, b'string', digestmod=digestmod).digest()
+ obj3 = cls(key, digestmod=digestmod)
obj3.update(b'string')
h2 = obj3.digest()
else:
diff --git a/Lib/test/test_pep277.py b/Lib/test/test_pep277.py
index 4b16cbb..9bae6dc 100644
--- a/Lib/test/test_pep277.py
+++ b/Lib/test/test_pep277.py
@@ -99,10 +99,6 @@ class UnicodeFileTests(unittest.TestCase):
with self.assertRaises(expected_exception) as c:
fn(filename)
exc_filename = c.exception.filename
- # listdir may append a wildcard to the filename
- if fn is os.listdir and sys.platform == 'win32':
- exc_filename, _, wildcard = exc_filename.rpartition(os.sep)
- self.assertEqual(wildcard, '*.*')
if check_filename:
self.assertEqual(exc_filename, filename, "Function '%s(%a) failed "
"with bad filename in the exception: %a" %
diff --git a/Lib/test/test_pep3151.py b/Lib/test/test_pep3151.py
index 2792c10..7d4a5d8 100644
--- a/Lib/test/test_pep3151.py
+++ b/Lib/test/test_pep3151.py
@@ -157,8 +157,6 @@ class AttributesTest(unittest.TestCase):
e.characters_written = 5
self.assertEqual(e.characters_written, 5)
- # XXX VMSError not tested
-
class ExplicitSubclassingTest(unittest.TestCase):
diff --git a/Lib/test/test_pep352.py b/Lib/test/test_pep352.py
index 558cdb5..7c98c46 100644
--- a/Lib/test/test_pep352.py
+++ b/Lib/test/test_pep352.py
@@ -1,7 +1,6 @@
import unittest
import builtins
import warnings
-from test.support import run_unittest
import os
from platform import system as platform_system
@@ -180,8 +179,6 @@ class UsageTests(unittest.TestCase):
# Catching a string is bad.
self.catch_fails("spam")
-def test_main():
- run_unittest(ExceptionClassTests, UsageTests)
if __name__ == '__main__':
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index fbe96ac..0b2fe1e 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -83,13 +83,17 @@ class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests):
class PyDispatchTableTests(AbstractDispatchTableTests):
+
pickler_class = pickle._Pickler
+
def get_dispatch_table(self):
return pickle.dispatch_table.copy()
class PyChainDispatchTableTests(AbstractDispatchTableTests):
+
pickler_class = pickle._Pickler
+
def get_dispatch_table(self):
return collections.ChainMap({}, pickle.dispatch_table)
diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py
index 355efe7..9883000 100644
--- a/Lib/test/test_pkg.py
+++ b/Lib/test/test_pkg.py
@@ -199,14 +199,14 @@ class TestPkg(unittest.TestCase):
import t5
self.assertEqual(fixdir(dir(t5)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', '__path__', 'foo',
- 'string', 't5'])
+ '__name__', '__package__', '__path__', '__spec__',
+ 'foo', 'string', 't5'])
self.assertEqual(fixdir(dir(t5.foo)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', 'string'])
+ '__name__', '__package__', '__spec__', 'string'])
self.assertEqual(fixdir(dir(t5.string)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', 'spam'])
+ '__name__', '__package__', '__spec__', 'spam'])
def test_6(self):
hier = [
@@ -222,14 +222,15 @@ class TestPkg(unittest.TestCase):
import t6
self.assertEqual(fixdir(dir(t6)),
['__all__', '__cached__', '__doc__', '__file__',
- '__loader__', '__name__', '__package__', '__path__'])
+ '__loader__', '__name__', '__package__', '__path__',
+ '__spec__'])
s = """
import t6
from t6 import *
self.assertEqual(fixdir(dir(t6)),
['__all__', '__cached__', '__doc__', '__file__',
'__loader__', '__name__', '__package__',
- '__path__', 'eggs', 'ham', 'spam'])
+ '__path__', '__spec__', 'eggs', 'ham', 'spam'])
self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
"""
self.run_code(s)
@@ -256,18 +257,19 @@ class TestPkg(unittest.TestCase):
import t7 as tas
self.assertEqual(fixdir(dir(tas)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', '__path__'])
+ '__name__', '__package__', '__path__', '__spec__'])
self.assertFalse(t7)
from t7 import sub as subpar
self.assertEqual(fixdir(dir(subpar)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', '__path__'])
+ '__name__', '__package__', '__path__', '__spec__'])
self.assertFalse(t7)
self.assertFalse(sub)
from t7.sub import subsub as subsubsub
self.assertEqual(fixdir(dir(subsubsub)),
['__cached__', '__doc__', '__file__', '__loader__',
- '__name__', '__package__', '__path__', 'spam'])
+ '__name__', '__package__', '__path__', '__spec__',
+ 'spam'])
self.assertFalse(t7)
self.assertFalse(sub)
self.assertFalse(subsub)
diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py
index a8426b5..370b2aa 100644
--- a/Lib/test/test_pkgimport.py
+++ b/Lib/test/test_pkgimport.py
@@ -6,7 +6,7 @@ import random
import tempfile
import unittest
-from imp import cache_from_source
+from importlib.util import cache_from_source
from test.support import run_unittest, create_empty_file
class TestImport(unittest.TestCase):
diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py
index fd06614..c4410a9 100644
--- a/Lib/test/test_pkgutil.py
+++ b/Lib/test/test_pkgutil.py
@@ -1,12 +1,13 @@
from test.support import run_unittest, unload, check_warnings
import unittest
import sys
-import imp
import importlib
+from importlib.util import spec_from_file_location
import pkgutil
import os
import os.path
import tempfile
+import types
import shutil
import zipfile
@@ -103,23 +104,20 @@ class PkgutilTests(unittest.TestCase):
class PkgutilPEP302Tests(unittest.TestCase):
class MyTestLoader(object):
- def load_module(self, fullname):
- # Create an empty module
- mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
- mod.__file__ = "<%s>" % self.__class__.__name__
- mod.__loader__ = self
- # Make it a package
- mod.__path__ = []
+ def exec_module(self, mod):
# Count how many times the module is reloaded
- mod.__dict__['loads'] = mod.__dict__.get('loads',0) + 1
- return mod
+ mod.__dict__['loads'] = mod.__dict__.get('loads', 0) + 1
def get_data(self, path):
return "Hello, world!"
class MyTestImporter(object):
- def find_module(self, fullname, path=None):
- return PkgutilPEP302Tests.MyTestLoader()
+ def find_spec(self, fullname, path=None, target=None):
+ loader = PkgutilPEP302Tests.MyTestLoader()
+ return spec_from_file_location(fullname,
+ '<%s>' % loader.__class__.__name__,
+ loader=loader,
+ submodule_search_locations=[])
def setUp(self):
sys.meta_path.insert(0, self.MyTestImporter())
@@ -200,6 +198,8 @@ class ExtendPathTests(unittest.TestCase):
dirname = self.create_init(pkgname)
pathitem = os.path.join(dirname, pkgname)
fullname = '{}.{}'.format(pkgname, modname)
+ sys.modules.pop(fullname, None)
+ sys.modules.pop(pkgname, None)
try:
self.create_submodule(dirname, pkgname, modname, 0)
@@ -208,11 +208,19 @@ class ExtendPathTests(unittest.TestCase):
importers = list(iter_importers(fullname))
expected_importer = get_importer(pathitem)
for finder in importers:
+ spec = pkgutil._get_spec(finder, fullname)
+ loader = spec.loader
+ try:
+ loader = loader.loader
+ except AttributeError:
+ # For now we still allow raw loaders from
+ # find_module().
+ pass
self.assertIsInstance(finder, importlib.machinery.FileFinder)
self.assertEqual(finder, expected_importer)
- self.assertIsInstance(finder.find_module(fullname),
+ self.assertIsInstance(loader,
importlib.machinery.SourceFileLoader)
- self.assertIsNone(finder.find_module(pkgname))
+ self.assertIsNone(pkgutil._get_spec(finder, pkgname))
with self.assertRaises(ImportError):
list(iter_importers('invalid.module'))
@@ -222,8 +230,11 @@ class ExtendPathTests(unittest.TestCase):
finally:
shutil.rmtree(dirname)
del sys.path[0]
- del sys.modules['spam']
- del sys.modules['spam.eggs']
+ try:
+ del sys.modules['spam']
+ del sys.modules['spam.eggs']
+ except KeyError:
+ pass
def test_mixed_namespace(self):
@@ -323,6 +334,25 @@ class ImportlibMigrationTests(unittest.TestCase):
self.assertIsNotNone(pkgutil.get_loader("test.support"))
self.assertEqual(len(w.warnings), 0)
+ def test_get_loader_handles_missing_loader_attribute(self):
+ global __loader__
+ this_loader = __loader__
+ del __loader__
+ try:
+ with check_warnings() as w:
+ self.assertIsNotNone(pkgutil.get_loader(__name__))
+ self.assertEqual(len(w.warnings), 0)
+ finally:
+ __loader__ = this_loader
+
+
+ def test_find_loader_avoids_emulation(self):
+ with check_warnings() as w:
+ self.assertIsNotNone(pkgutil.find_loader("sys"))
+ self.assertIsNotNone(pkgutil.find_loader("os"))
+ self.assertIsNotNone(pkgutil.find_loader("test.support"))
+ self.assertEqual(len(w.warnings), 0)
+
def test_get_importer_avoids_emulation(self):
# We use an illegal path so *none* of the path hooks should fire
with check_warnings() as w:
diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py
index 9e86b3d..bb7cc15 100644
--- a/Lib/test/test_plistlib.py
+++ b/Lib/test/test_plistlib.py
@@ -1,94 +1,95 @@
-# Copyright (C) 2003 Python Software Foundation
+# Copyright (C) 2003-2013 Python Software Foundation
import unittest
import plistlib
import os
import datetime
+import codecs
+import binascii
+import collections
+import struct
from test import support
+from io import BytesIO
+ALL_FORMATS=(plistlib.FMT_XML, plistlib.FMT_BINARY)
-# This test data was generated through Cocoa's NSDictionary class
-TESTDATA = b"""<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" \
-"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
- <key>aDate</key>
- <date>2004-10-26T10:33:33Z</date>
- <key>aDict</key>
- <dict>
- <key>aFalseValue</key>
- <false/>
- <key>aTrueValue</key>
- <true/>
- <key>aUnicodeValue</key>
- <string>M\xc3\xa4ssig, Ma\xc3\x9f</string>
- <key>anotherString</key>
- <string>&lt;hello &amp; 'hi' there!&gt;</string>
- <key>deeperDict</key>
- <dict>
- <key>a</key>
- <integer>17</integer>
- <key>b</key>
- <real>32.5</real>
- <key>c</key>
- <array>
- <integer>1</integer>
- <integer>2</integer>
- <string>text</string>
- </array>
- </dict>
- </dict>
- <key>aFloat</key>
- <real>0.5</real>
- <key>aList</key>
- <array>
- <string>A</string>
- <string>B</string>
- <integer>12</integer>
- <real>32.5</real>
- <array>
- <integer>1</integer>
- <integer>2</integer>
- <integer>3</integer>
- </array>
- </array>
- <key>aString</key>
- <string>Doodah</string>
- <key>anEmptyDict</key>
- <dict/>
- <key>anEmptyList</key>
- <array/>
- <key>anInt</key>
- <integer>728</integer>
- <key>nestedData</key>
- <array>
- <data>
- PGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5r
- PgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5
- IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBi
- aW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3Rz
- IG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQID
- PGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAw==
- </data>
- </array>
- <key>someData</key>
- <data>
- PGJpbmFyeSBndW5rPg==
- </data>
- <key>someMoreData</key>
- <data>
- PGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8
- bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxs
- b3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxv
- dHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90
- cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAw==
- </data>
- <key>\xc3\x85benraa</key>
- <string>That was a unicode key.</string>
-</dict>
-</plist>
-""".replace(b" " * 8, b"\t") # Apple as well as plistlib.py output hard tabs
+# The testdata is generated using Mac/Tools/plistlib_generate_testdata.py
+# (which using PyObjC to control the Cocoa classes for generating plists)
+TESTDATA={
+ plistlib.FMT_XML: binascii.a2b_base64(b'''
+ PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCFET0NU
+ WVBFIHBsaXN0IFBVQkxJQyAiLS8vQXBwbGUvL0RURCBQTElTVCAxLjAvL0VO
+ IiAiaHR0cDovL3d3dy5hcHBsZS5jb20vRFREcy9Qcm9wZXJ0eUxpc3QtMS4w
+ LmR0ZCI+CjxwbGlzdCB2ZXJzaW9uPSIxLjAiPgo8ZGljdD4KCTxrZXk+YUJp
+ Z0ludDwva2V5PgoJPGludGVnZXI+OTIyMzM3MjAzNjg1NDc3NTc2NDwvaW50
+ ZWdlcj4KCTxrZXk+YUJpZ0ludDI8L2tleT4KCTxpbnRlZ2VyPjkyMjMzNzIw
+ MzY4NTQ3NzU4NTI8L2ludGVnZXI+Cgk8a2V5PmFEYXRlPC9rZXk+Cgk8ZGF0
+ ZT4yMDA0LTEwLTI2VDEwOjMzOjMzWjwvZGF0ZT4KCTxrZXk+YURpY3Q8L2tl
+ eT4KCTxkaWN0PgoJCTxrZXk+YUZhbHNlVmFsdWU8L2tleT4KCQk8ZmFsc2Uv
+ PgoJCTxrZXk+YVRydWVWYWx1ZTwva2V5PgoJCTx0cnVlLz4KCQk8a2V5PmFV
+ bmljb2RlVmFsdWU8L2tleT4KCQk8c3RyaW5nPk3DpHNzaWcsIE1hw588L3N0
+ cmluZz4KCQk8a2V5PmFub3RoZXJTdHJpbmc8L2tleT4KCQk8c3RyaW5nPiZs
+ dDtoZWxsbyAmYW1wOyAnaGknIHRoZXJlISZndDs8L3N0cmluZz4KCQk8a2V5
+ PmRlZXBlckRpY3Q8L2tleT4KCQk8ZGljdD4KCQkJPGtleT5hPC9rZXk+CgkJ
+ CTxpbnRlZ2VyPjE3PC9pbnRlZ2VyPgoJCQk8a2V5PmI8L2tleT4KCQkJPHJl
+ YWw+MzIuNTwvcmVhbD4KCQkJPGtleT5jPC9rZXk+CgkJCTxhcnJheT4KCQkJ
+ CTxpbnRlZ2VyPjE8L2ludGVnZXI+CgkJCQk8aW50ZWdlcj4yPC9pbnRlZ2Vy
+ PgoJCQkJPHN0cmluZz50ZXh0PC9zdHJpbmc+CgkJCTwvYXJyYXk+CgkJPC9k
+ aWN0PgoJPC9kaWN0PgoJPGtleT5hRmxvYXQ8L2tleT4KCTxyZWFsPjAuNTwv
+ cmVhbD4KCTxrZXk+YUxpc3Q8L2tleT4KCTxhcnJheT4KCQk8c3RyaW5nPkE8
+ L3N0cmluZz4KCQk8c3RyaW5nPkI8L3N0cmluZz4KCQk8aW50ZWdlcj4xMjwv
+ aW50ZWdlcj4KCQk8cmVhbD4zMi41PC9yZWFsPgoJCTxhcnJheT4KCQkJPGlu
+ dGVnZXI+MTwvaW50ZWdlcj4KCQkJPGludGVnZXI+MjwvaW50ZWdlcj4KCQkJ
+ PGludGVnZXI+MzwvaW50ZWdlcj4KCQk8L2FycmF5PgoJPC9hcnJheT4KCTxr
+ ZXk+YU5lZ2F0aXZlQmlnSW50PC9rZXk+Cgk8aW50ZWdlcj4tODAwMDAwMDAw
+ MDA8L2ludGVnZXI+Cgk8a2V5PmFOZWdhdGl2ZUludDwva2V5PgoJPGludGVn
+ ZXI+LTU8L2ludGVnZXI+Cgk8a2V5PmFTdHJpbmc8L2tleT4KCTxzdHJpbmc+
+ RG9vZGFoPC9zdHJpbmc+Cgk8a2V5PmFuRW1wdHlEaWN0PC9rZXk+Cgk8ZGlj
+ dC8+Cgk8a2V5PmFuRW1wdHlMaXN0PC9rZXk+Cgk8YXJyYXkvPgoJPGtleT5h
+ bkludDwva2V5PgoJPGludGVnZXI+NzI4PC9pbnRlZ2VyPgoJPGtleT5uZXN0
+ ZWREYXRhPC9rZXk+Cgk8YXJyYXk+CgkJPGRhdGE+CgkJUEd4dmRITWdiMlln
+ WW1sdVlYSjVJR2QxYm1zK0FBRUNBenhzYjNSeklHOW1JR0pwYm1GeWVTQm5k
+ VzVyCgkJUGdBQkFnTThiRzkwY3lCdlppQmlhVzVoY25rZ1ozVnVhejRBQVFJ
+ RFBHeHZkSE1nYjJZZ1ltbHVZWEo1CgkJSUdkMWJtcytBQUVDQXp4c2IzUnpJ
+ RzltSUdKcGJtRnllU0JuZFc1clBnQUJBZ004Ykc5MGN5QnZaaUJpCgkJYVc1
+ aGNua2daM1Z1YXo0QUFRSURQR3h2ZEhNZ2IyWWdZbWx1WVhKNUlHZDFibXMr
+ QUFFQ0F6eHNiM1J6CgkJSUc5bUlHSnBibUZ5ZVNCbmRXNXJQZ0FCQWdNOGJH
+ OTBjeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlECgkJUEd4dmRITWdiMlln
+ WW1sdVlYSjVJR2QxYm1zK0FBRUNBdz09CgkJPC9kYXRhPgoJPC9hcnJheT4K
+ CTxrZXk+c29tZURhdGE8L2tleT4KCTxkYXRhPgoJUEdKcGJtRnllU0JuZFc1
+ clBnPT0KCTwvZGF0YT4KCTxrZXk+c29tZU1vcmVEYXRhPC9rZXk+Cgk8ZGF0
+ YT4KCVBHeHZkSE1nYjJZZ1ltbHVZWEo1SUdkMWJtcytBQUVDQXp4c2IzUnpJ
+ RzltSUdKcGJtRnllU0JuZFc1clBnQUJBZ004CgliRzkwY3lCdlppQmlhVzVo
+ Y25rZ1ozVnVhejRBQVFJRFBHeHZkSE1nYjJZZ1ltbHVZWEo1SUdkMWJtcytB
+ QUVDQXp4cwoJYjNSeklHOW1JR0pwYm1GeWVTQm5kVzVyUGdBQkFnTThiRzkw
+ Y3lCdlppQmlhVzVoY25rZ1ozVnVhejRBQVFJRFBHeHYKCWRITWdiMllnWW1s
+ dVlYSjVJR2QxYm1zK0FBRUNBenhzYjNSeklHOW1JR0pwYm1GeWVTQm5kVzVy
+ UGdBQkFnTThiRzkwCgljeUJ2WmlCaWFXNWhjbmtnWjNWdWF6NEFBUUlEUEd4
+ dmRITWdiMllnWW1sdVlYSjVJR2QxYm1zK0FBRUNBdz09Cgk8L2RhdGE+Cgk8
+ a2V5PsOFYmVucmFhPC9rZXk+Cgk8c3RyaW5nPlRoYXQgd2FzIGEgdW5pY29k
+ ZSBrZXkuPC9zdHJpbmc+CjwvZGljdD4KPC9wbGlzdD4K'''),
+ plistlib.FMT_BINARY: binascii.a2b_base64(b'''
+ YnBsaXN0MDDfEBABAgMEBQYHCAkKCwwNDg8QERITFCgpLzAxMjM0NTc2OFdh
+ QmlnSW50WGFCaWdJbnQyVWFEYXRlVWFEaWN0VmFGbG9hdFVhTGlzdF8QD2FO
+ ZWdhdGl2ZUJpZ0ludFxhTmVnYXRpdmVJbnRXYVN0cmluZ1thbkVtcHR5RGlj
+ dFthbkVtcHR5TGlzdFVhbkludFpuZXN0ZWREYXRhWHNvbWVEYXRhXHNvbWVN
+ b3JlRGF0YWcAxQBiAGUAbgByAGEAYRN/////////1BQAAAAAAAAAAIAAAAAA
+ AAAsM0GcuX30AAAA1RUWFxgZGhscHR5bYUZhbHNlVmFsdWVaYVRydWVWYWx1
+ ZV1hVW5pY29kZVZhbHVlXWFub3RoZXJTdHJpbmdaZGVlcGVyRGljdAgJawBN
+ AOQAcwBzAGkAZwAsACAATQBhAN9fEBU8aGVsbG8gJiAnaGknIHRoZXJlIT7T
+ HyAhIiMkUWFRYlFjEBEjQEBAAAAAAACjJSYnEAEQAlR0ZXh0Iz/gAAAAAAAA
+ pSorLCMtUUFRQhAMoyUmLhADE////+1foOAAE//////////7VkRvb2RhaNCg
+ EQLYoTZPEPo8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmlu
+ YXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBv
+ ZiBiaW5hcnkgZ3Vuaz4AAQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxs
+ b3RzIG9mIGJpbmFyeSBndW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4A
+ AQIDPGxvdHMgb2YgYmluYXJ5IGd1bms+AAECAzxsb3RzIG9mIGJpbmFyeSBn
+ dW5rPgABAgM8bG90cyBvZiBiaW5hcnkgZ3Vuaz4AAQIDTTxiaW5hcnkgZ3Vu
+ az5fEBdUaGF0IHdhcyBhIHVuaWNvZGUga2V5LgAIACsAMwA8AEIASABPAFUA
+ ZwB0AHwAiACUAJoApQCuALsAygDTAOQA7QD4AQQBDwEdASsBNgE3ATgBTwFn
+ AW4BcAFyAXQBdgF/AYMBhQGHAYwBlQGbAZ0BnwGhAaUBpwGwAbkBwAHBAcIB
+ xQHHAsQC0gAAAAAAAAIBAAAAAAAAADkAAAAAAAAAAAAAAAAAAALs'''),
+}
class TestPlistlib(unittest.TestCase):
@@ -99,12 +100,16 @@ class TestPlistlib(unittest.TestCase):
except:
pass
- def _create(self):
+ def _create(self, fmt=None):
pl = dict(
aString="Doodah",
aList=["A", "B", 12, 32.5, [1, 2, 3]],
aFloat = 0.5,
anInt = 728,
+ aBigInt = 2 ** 63 - 44,
+ aBigInt2 = 2 ** 63 + 44,
+ aNegativeInt = -5,
+ aNegativeBigInt = -80000000000,
aDict=dict(
anotherString="<hello & 'hi' there!>",
aUnicodeValue='M\xe4ssig, Ma\xdf',
@@ -112,9 +117,9 @@ class TestPlistlib(unittest.TestCase):
aFalseValue=False,
deeperDict=dict(a=17, b=32.5, c=[1, 2, "text"]),
),
- someData = plistlib.Data(b"<binary gunk>"),
- someMoreData = plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10),
- nestedData = [plistlib.Data(b"<lots of binary gunk>\0\1\2\3" * 10)],
+ someData = b"<binary gunk>",
+ someMoreData = b"<lots of binary gunk>\0\1\2\3" * 10,
+ nestedData = [b"<lots of binary gunk>\0\1\2\3" * 10],
aDate = datetime.datetime(2004, 10, 26, 10, 33, 33),
anEmptyDict = dict(),
anEmptyList = list()
@@ -129,49 +134,215 @@ class TestPlistlib(unittest.TestCase):
def test_io(self):
pl = self._create()
- plistlib.writePlist(pl, support.TESTFN)
- pl2 = plistlib.readPlist(support.TESTFN)
+ with open(support.TESTFN, 'wb') as fp:
+ plistlib.dump(pl, fp)
+
+ with open(support.TESTFN, 'rb') as fp:
+ pl2 = plistlib.load(fp)
+
self.assertEqual(dict(pl), dict(pl2))
+ self.assertRaises(AttributeError, plistlib.dump, pl, 'filename')
+ self.assertRaises(AttributeError, plistlib.load, 'filename')
+
+ def test_invalid_type(self):
+ pl = [ object() ]
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ self.assertRaises(TypeError, plistlib.dumps, pl, fmt=fmt)
+
+ def test_int(self):
+ for pl in [0, 2**8-1, 2**8, 2**16-1, 2**16, 2**32-1, 2**32,
+ 2**63-1, 2**64-1, 1, -2**63]:
+ for fmt in ALL_FORMATS:
+ with self.subTest(pl=pl, fmt=fmt):
+ data = plistlib.dumps(pl, fmt=fmt)
+ pl2 = plistlib.loads(data)
+ self.assertIsInstance(pl2, int)
+ self.assertEqual(pl, pl2)
+ data2 = plistlib.dumps(pl2, fmt=fmt)
+ self.assertEqual(data, data2)
+
+ for fmt in ALL_FORMATS:
+ for pl in (2 ** 64 + 1, 2 ** 127-1, -2**64, -2 ** 127):
+ with self.subTest(pl=pl, fmt=fmt):
+ self.assertRaises(OverflowError, plistlib.dumps,
+ pl, fmt=fmt)
+
def test_bytes(self):
pl = self._create()
- data = plistlib.writePlistToBytes(pl)
- pl2 = plistlib.readPlistFromBytes(data)
+ data = plistlib.dumps(pl)
+ pl2 = plistlib.loads(data)
+ self.assertNotIsInstance(pl, plistlib._InternalDict)
self.assertEqual(dict(pl), dict(pl2))
- data2 = plistlib.writePlistToBytes(pl2)
+ data2 = plistlib.dumps(pl2)
self.assertEqual(data, data2)
def test_indentation_array(self):
- data = [[[[[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]]]]
- self.assertEqual(plistlib.readPlistFromBytes(plistlib.writePlistToBytes(data)), data)
+ data = [[[[[[[[{'test': b'aaaaaa'}]]]]]]]]
+ self.assertEqual(plistlib.loads(plistlib.dumps(data)), data)
def test_indentation_dict(self):
- data = {'1': {'2': {'3': {'4': {'5': {'6': {'7': {'8': {'9': plistlib.Data(b'aaaaaa')}}}}}}}}}
- self.assertEqual(plistlib.readPlistFromBytes(plistlib.writePlistToBytes(data)), data)
+ data = {'1': {'2': {'3': {'4': {'5': {'6': {'7': {'8': {'9': b'aaaaaa'}}}}}}}}}
+ self.assertEqual(plistlib.loads(plistlib.dumps(data)), data)
def test_indentation_dict_mix(self):
- data = {'1': {'2': [{'3': [[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]}]}}
- self.assertEqual(plistlib.readPlistFromBytes(plistlib.writePlistToBytes(data)), data)
+ data = {'1': {'2': [{'3': [[[[[{'test': b'aaaaaa'}]]]]]}]}}
+ self.assertEqual(plistlib.loads(plistlib.dumps(data)), data)
def test_appleformatting(self):
- pl = plistlib.readPlistFromBytes(TESTDATA)
- data = plistlib.writePlistToBytes(pl)
- self.assertEqual(data, TESTDATA,
- "generated data was not identical to Apple's output")
+ for use_builtin_types in (True, False):
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt, use_builtin_types=use_builtin_types):
+ pl = plistlib.loads(TESTDATA[fmt],
+ use_builtin_types=use_builtin_types)
+ data = plistlib.dumps(pl, fmt=fmt)
+ self.assertEqual(data, TESTDATA[fmt],
+ "generated data was not identical to Apple's output")
+
def test_appleformattingfromliteral(self):
- pl = self._create()
- pl2 = plistlib.readPlistFromBytes(TESTDATA)
- self.assertEqual(dict(pl), dict(pl2),
- "generated data was not identical to Apple's output")
+ self.maxDiff = None
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ pl = self._create(fmt=fmt)
+ pl2 = plistlib.loads(TESTDATA[fmt])
+ self.assertEqual(dict(pl), dict(pl2),
+ "generated data was not identical to Apple's output")
def test_bytesio(self):
- from io import BytesIO
- b = BytesIO()
- pl = self._create()
- plistlib.writePlist(pl, b)
- pl2 = plistlib.readPlist(BytesIO(b.getvalue()))
- self.assertEqual(dict(pl), dict(pl2))
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ b = BytesIO()
+ pl = self._create(fmt=fmt)
+ plistlib.dump(pl, b, fmt=fmt)
+ pl2 = plistlib.load(BytesIO(b.getvalue()))
+ self.assertEqual(dict(pl), dict(pl2))
+
+ def test_keysort_bytesio(self):
+ pl = collections.OrderedDict()
+ pl['b'] = 1
+ pl['a'] = 2
+ pl['c'] = 3
+
+ for fmt in ALL_FORMATS:
+ for sort_keys in (False, True):
+ with self.subTest(fmt=fmt, sort_keys=sort_keys):
+ b = BytesIO()
+
+ plistlib.dump(pl, b, fmt=fmt, sort_keys=sort_keys)
+ pl2 = plistlib.load(BytesIO(b.getvalue()),
+ dict_type=collections.OrderedDict)
+
+ self.assertEqual(dict(pl), dict(pl2))
+ if sort_keys:
+ self.assertEqual(list(pl2.keys()), ['a', 'b', 'c'])
+ else:
+ self.assertEqual(list(pl2.keys()), ['b', 'a', 'c'])
+
+ def test_keysort(self):
+ pl = collections.OrderedDict()
+ pl['b'] = 1
+ pl['a'] = 2
+ pl['c'] = 3
+
+ for fmt in ALL_FORMATS:
+ for sort_keys in (False, True):
+ with self.subTest(fmt=fmt, sort_keys=sort_keys):
+ data = plistlib.dumps(pl, fmt=fmt, sort_keys=sort_keys)
+ pl2 = plistlib.loads(data, dict_type=collections.OrderedDict)
+
+ self.assertEqual(dict(pl), dict(pl2))
+ if sort_keys:
+ self.assertEqual(list(pl2.keys()), ['a', 'b', 'c'])
+ else:
+ self.assertEqual(list(pl2.keys()), ['b', 'a', 'c'])
+
+ def test_keys_no_string(self):
+ pl = { 42: 'aNumber' }
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ self.assertRaises(TypeError, plistlib.dumps, pl, fmt=fmt)
+
+ b = BytesIO()
+ self.assertRaises(TypeError, plistlib.dump, pl, b, fmt=fmt)
+
+ def test_skipkeys(self):
+ pl = {
+ 42: 'aNumber',
+ 'snake': 'aWord',
+ }
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ data = plistlib.dumps(
+ pl, fmt=fmt, skipkeys=True, sort_keys=False)
+
+ pl2 = plistlib.loads(data)
+ self.assertEqual(pl2, {'snake': 'aWord'})
+
+ fp = BytesIO()
+ plistlib.dump(
+ pl, fp, fmt=fmt, skipkeys=True, sort_keys=False)
+ data = fp.getvalue()
+ pl2 = plistlib.loads(fp.getvalue())
+ self.assertEqual(pl2, {'snake': 'aWord'})
+
+ def test_tuple_members(self):
+ pl = {
+ 'first': (1, 2),
+ 'second': (1, 2),
+ 'third': (3, 4),
+ }
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ data = plistlib.dumps(pl, fmt=fmt)
+ pl2 = plistlib.loads(data)
+ self.assertEqual(pl2, {
+ 'first': [1, 2],
+ 'second': [1, 2],
+ 'third': [3, 4],
+ })
+ self.assertIsNot(pl2['first'], pl2['second'])
+
+ def test_list_members(self):
+ pl = {
+ 'first': [1, 2],
+ 'second': [1, 2],
+ 'third': [3, 4],
+ }
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ data = plistlib.dumps(pl, fmt=fmt)
+ pl2 = plistlib.loads(data)
+ self.assertEqual(pl2, {
+ 'first': [1, 2],
+ 'second': [1, 2],
+ 'third': [3, 4],
+ })
+ self.assertIsNot(pl2['first'], pl2['second'])
+
+ def test_dict_members(self):
+ pl = {
+ 'first': {'a': 1},
+ 'second': {'a': 1},
+ 'third': {'b': 2 },
+ }
+
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ data = plistlib.dumps(pl, fmt=fmt)
+ pl2 = plistlib.loads(data)
+ self.assertEqual(pl2, {
+ 'first': {'a': 1},
+ 'second': {'a': 1},
+ 'third': {'b': 2 },
+ })
+ self.assertIsNot(pl2['first'], pl2['second'])
def test_controlcharacters(self):
for i in range(128):
@@ -179,25 +350,27 @@ class TestPlistlib(unittest.TestCase):
testString = "string containing %s" % c
if i >= 32 or c in "\r\n\t":
# \r, \n and \t are the only legal control chars in XML
- plistlib.writePlistToBytes(testString)
+ plistlib.dumps(testString, fmt=plistlib.FMT_XML)
else:
self.assertRaises(ValueError,
- plistlib.writePlistToBytes,
+ plistlib.dumps,
testString)
def test_nondictroot(self):
- test1 = "abc"
- test2 = [1, 2, 3, "abc"]
- result1 = plistlib.readPlistFromBytes(plistlib.writePlistToBytes(test1))
- result2 = plistlib.readPlistFromBytes(plistlib.writePlistToBytes(test2))
- self.assertEqual(test1, result1)
- self.assertEqual(test2, result2)
+ for fmt in ALL_FORMATS:
+ with self.subTest(fmt=fmt):
+ test1 = "abc"
+ test2 = [1, 2, 3, "abc"]
+ result1 = plistlib.loads(plistlib.dumps(test1, fmt=fmt))
+ result2 = plistlib.loads(plistlib.dumps(test2, fmt=fmt))
+ self.assertEqual(test1, result1)
+ self.assertEqual(test2, result2)
def test_invalidarray(self):
for i in ["<key>key inside an array</key>",
"<key>key inside an array2</key><real>3</real>",
"<true/><key>key inside an array3</key>"]:
- self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ self.assertRaises(ValueError, plistlib.loads,
("<plist><array>%s</array></plist>"%i).encode())
def test_invaliddict(self):
@@ -206,22 +379,130 @@ class TestPlistlib(unittest.TestCase):
"<string>missing key</string>",
"<key>k1</key><string>v1</string><real>5.3</real>"
"<key>k1</key><key>k2</key><string>double key</string>"]:
- self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ self.assertRaises(ValueError, plistlib.loads,
("<plist><dict>%s</dict></plist>"%i).encode())
- self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ self.assertRaises(ValueError, plistlib.loads,
("<plist><array><dict>%s</dict></array></plist>"%i).encode())
def test_invalidinteger(self):
- self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ self.assertRaises(ValueError, plistlib.loads,
b"<plist><integer>not integer</integer></plist>")
def test_invalidreal(self):
- self.assertRaises(ValueError, plistlib.readPlistFromBytes,
+ self.assertRaises(ValueError, plistlib.loads,
b"<plist><integer>not real</integer></plist>")
+ def test_xml_encodings(self):
+ base = TESTDATA[plistlib.FMT_XML]
+
+ for xml_encoding, encoding, bom in [
+ (b'utf-8', 'utf-8', codecs.BOM_UTF8),
+ (b'utf-16', 'utf-16-le', codecs.BOM_UTF16_LE),
+ (b'utf-16', 'utf-16-be', codecs.BOM_UTF16_BE),
+ # Expat does not support UTF-32
+ #(b'utf-32', 'utf-32-le', codecs.BOM_UTF32_LE),
+ #(b'utf-32', 'utf-32-be', codecs.BOM_UTF32_BE),
+ ]:
+
+ pl = self._create(fmt=plistlib.FMT_XML)
+ with self.subTest(encoding=encoding):
+ data = base.replace(b'UTF-8', xml_encoding)
+ data = bom + data.decode('utf-8').encode(encoding)
+ pl2 = plistlib.loads(data)
+ self.assertEqual(dict(pl), dict(pl2))
+
+
+class TestPlistlibDeprecated(unittest.TestCase):
+ def test_io_deprecated(self):
+ pl_in = {
+ 'key': 42,
+ 'sub': {
+ 'key': 9,
+ 'alt': 'value',
+ 'data': b'buffer',
+ }
+ }
+ pl_out = plistlib._InternalDict({
+ 'key': 42,
+ 'sub': plistlib._InternalDict({
+ 'key': 9,
+ 'alt': 'value',
+ 'data': plistlib.Data(b'buffer'),
+ })
+ })
+
+ self.addCleanup(support.unlink, support.TESTFN)
+ with self.assertWarns(DeprecationWarning):
+ plistlib.writePlist(pl_in, support.TESTFN)
+
+ with self.assertWarns(DeprecationWarning):
+ pl2 = plistlib.readPlist(support.TESTFN)
+
+ self.assertEqual(pl_out, pl2)
+
+ os.unlink(support.TESTFN)
+
+ with open(support.TESTFN, 'wb') as fp:
+ with self.assertWarns(DeprecationWarning):
+ plistlib.writePlist(pl_in, fp)
+
+ with open(support.TESTFN, 'rb') as fp:
+ with self.assertWarns(DeprecationWarning):
+ pl2 = plistlib.readPlist(fp)
+
+ self.assertEqual(pl_out, pl2)
+
+ def test_bytes_deprecated(self):
+ pl = {
+ 'key': 42,
+ 'sub': {
+ 'key': 9,
+ 'alt': 'value',
+ 'data': b'buffer',
+ }
+ }
+ with self.assertWarns(DeprecationWarning):
+ data = plistlib.writePlistToBytes(pl)
+
+ with self.assertWarns(DeprecationWarning):
+ pl2 = plistlib.readPlistFromBytes(data)
+
+ self.assertIsInstance(pl2, plistlib._InternalDict)
+ self.assertEqual(pl2, plistlib._InternalDict(
+ key=42,
+ sub=plistlib._InternalDict(
+ key=9,
+ alt='value',
+ data=plistlib.Data(b'buffer'),
+ )
+ ))
+
+ with self.assertWarns(DeprecationWarning):
+ data2 = plistlib.writePlistToBytes(pl2)
+ self.assertEqual(data, data2)
+
+ def test_dataobject_deprecated(self):
+ in_data = { 'key': plistlib.Data(b'hello') }
+ out_data = { 'key': b'hello' }
+
+ buf = plistlib.dumps(in_data)
+
+ cur = plistlib.loads(buf)
+ self.assertEqual(cur, out_data)
+ self.assertNotEqual(cur, in_data)
+
+ cur = plistlib.loads(buf, use_builtin_types=False)
+ self.assertNotEqual(cur, out_data)
+ self.assertEqual(cur, in_data)
+
+ with self.assertWarns(DeprecationWarning):
+ cur = plistlib.readPlistFromBytes(buf)
+ self.assertNotEqual(cur, out_data)
+ self.assertEqual(cur, in_data)
+
def test_main():
- support.run_unittest(TestPlistlib)
+ support.run_unittest(TestPlistlib, TestPlistlibDeprecated)
if __name__ == '__main__':
diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py
index a07a719..a0a332b 100644
--- a/Lib/test/test_poll.py
+++ b/Lib/test/test_poll.py
@@ -1,6 +1,7 @@
# Test case for the os.poll() function
import os
+import subprocess
import random
import select
try:
@@ -14,7 +15,7 @@ from test.support import TESTFN, run_unittest, reap_threads, cpython_only
try:
select.poll
except AttributeError:
- raise unittest.SkipTest("select.poll not defined -- skipping test_poll")
+ raise unittest.SkipTest("select.poll not defined")
def find_ready_matching(ready, flag):
@@ -75,13 +76,11 @@ class PollTests(unittest.TestCase):
self.assertEqual(bufs, [MSG] * NUM_PIPES)
- def poll_unit_tests(self):
+ def test_poll_unit_tests(self):
# returns NVAL for invalid file descriptor
- FD = 42
- try:
- os.close(FD)
- except OSError:
- pass
+ FD, w = os.pipe()
+ os.close(FD)
+ os.close(w)
p = select.poll()
p.register(FD)
r = p.poll()
@@ -124,7 +123,9 @@ class PollTests(unittest.TestCase):
def test_poll2(self):
cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done'
- p = os.popen(cmd, 'r')
+ proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+ bufsize=0)
+ p = proc.stdout
pollster = select.poll()
pollster.register( p, select.POLLIN )
for tout in (0, 1000, 2000, 4000, 8000, 16000) + (-1,)*10:
@@ -134,7 +135,7 @@ class PollTests(unittest.TestCase):
fd, flags = fdlist[0]
if flags & select.POLLHUP:
line = p.readline()
- if line != "":
+ if line != b"":
self.fail('error: pipe seems to be closed, but still returns data')
continue
@@ -142,6 +143,7 @@ class PollTests(unittest.TestCase):
line = p.readline()
if not line:
break
+ self.assertEqual(line, b'testing...\n')
continue
else:
self.fail('Unexpected return value from select.poll: %s' % fdlist)
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 5809be6..d076fc1 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -18,6 +18,19 @@ threading = test_support.import_module('threading')
HOST = test_support.HOST
PORT = 0
+SUPPORTS_SSL = False
+if hasattr(poplib, 'POP3_SSL'):
+ import ssl
+ from ssl import HAS_SNI
+
+ SUPPORTS_SSL = True
+ CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert3.pem")
+ CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "pycacert.pem")
+else:
+ HAS_SNI = False
+
+requires_ssl = skipUnless(SUPPORTS_SSL, 'SSL not supported')
+
# the dummy data returned by server when LIST and RETR commands are issued
LIST_RESP = b'1 1\r\n2 2\r\n3 3\r\n4 4\r\n5 5\r\n.\r\n'
RETR_RESP = b"""From: postmaster@python.org\
@@ -33,11 +46,15 @@ line3\r\n\
class DummyPOP3Handler(asynchat.async_chat):
+ CAPAS = {'UIDL': [], 'IMPLEMENTATION': ['python-testlib-pop-server']}
+
def __init__(self, conn):
asynchat.async_chat.__init__(self, conn)
self.set_terminator(b"\r\n")
self.in_buffer = []
self.push('+OK dummy pop3 server ready. <timestamp>')
+ self.tls_active = False
+ self.tls_starting = False
def collect_incoming_data(self, data):
self.in_buffer.append(data)
@@ -112,6 +129,65 @@ class DummyPOP3Handler(asynchat.async_chat):
self.push('+OK closing.')
self.close_when_done()
+ def _get_capas(self):
+ _capas = dict(self.CAPAS)
+ if not self.tls_active and SUPPORTS_SSL:
+ _capas['STLS'] = []
+ return _capas
+
+ def cmd_capa(self, arg):
+ self.push('+OK Capability list follows')
+ if self._get_capas():
+ for cap, params in self._get_capas().items():
+ _ln = [cap]
+ if params:
+ _ln.extend(params)
+ self.push(' '.join(_ln))
+ self.push('.')
+
+ if SUPPORTS_SSL:
+
+ def cmd_stls(self, arg):
+ if self.tls_active is False:
+ self.push('+OK Begin TLS negotiation')
+ tls_sock = ssl.wrap_socket(self.socket, certfile=CERTFILE,
+ server_side=True,
+ do_handshake_on_connect=False,
+ suppress_ragged_eofs=False)
+ self.del_channel()
+ self.set_socket(tls_sock)
+ self.tls_active = True
+ self.tls_starting = True
+ self.in_buffer = []
+ self._do_tls_handshake()
+ else:
+ self.push('-ERR Command not permitted when TLS active')
+
+ def _do_tls_handshake(self):
+ try:
+ self.socket.do_handshake()
+ except ssl.SSLError as err:
+ if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
+ ssl.SSL_ERROR_WANT_WRITE):
+ return
+ elif err.args[0] == ssl.SSL_ERROR_EOF:
+ return self.handle_close()
+ raise
+ except OSError as err:
+ if err.args[0] == errno.ECONNABORTED:
+ return self.handle_close()
+ else:
+ self.tls_active = True
+ self.tls_starting = False
+
+ def handle_read(self):
+ if self.tls_starting:
+ self._do_tls_handshake()
+ else:
+ try:
+ asynchat.async_chat.handle_read(self)
+ except ssl.SSLEOFError:
+ self.handle_close()
class DummyPOP3Server(asyncore.dispatcher, threading.Thread):
@@ -236,19 +312,43 @@ class TestPOP3Class(TestCase):
self.client.uidl()
self.client.uidl('foo')
+ def test_capa(self):
+ capa = self.client.capa()
+ self.assertTrue('IMPLEMENTATION' in capa.keys())
+
def test_quit(self):
resp = self.client.quit()
self.assertTrue(resp)
self.assertIsNone(self.client.sock)
self.assertIsNone(self.client.file)
+ @requires_ssl
+ def test_stls_capa(self):
+ capa = self.client.capa()
+ self.assertTrue('STLS' in capa.keys())
+
+ @requires_ssl
+ def test_stls(self):
+ expected = b'+OK Begin TLS negotiation'
+ resp = self.client.stls()
+ self.assertEqual(resp, expected)
+
+ @requires_ssl
+ @skipUnless(HAS_SNI, 'No SNI support in ssl module')
+ def test_stls_context(self):
+ expected = b'+OK Begin TLS negotiation'
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
+ ctx.load_verify_locations(CAFILE)
+ ctx.verify_mode = ssl.CERT_REQUIRED
+ ctx.check_hostname = True
+ with self.assertRaises(ssl.CertificateError):
+ resp = self.client.stls(context=ctx)
+ self.client = poplib.POP3("localhost", self.server.port, timeout=3)
+ resp = self.client.stls(context=ctx)
+ self.assertEqual(resp, expected)
-SUPPORTS_SSL = False
-if hasattr(poplib, 'POP3_SSL'):
- import ssl
- SUPPORTS_SSL = True
- CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem")
+if SUPPORTS_SSL:
class DummyPOP3_SSLHandler(DummyPOP3Handler):
@@ -260,35 +360,13 @@ if hasattr(poplib, 'POP3_SSL'):
self.del_channel()
self.set_socket(ssl_socket)
# Must try handshake before calling push()
- self._ssl_accepting = True
- self._do_ssl_handshake()
+ self.tls_active = True
+ self.tls_starting = True
+ self._do_tls_handshake()
self.set_terminator(b"\r\n")
self.in_buffer = []
self.push('+OK dummy pop3 server ready. <timestamp>')
- def _do_ssl_handshake(self):
- try:
- self.socket.do_handshake()
- except ssl.SSLError as err:
- if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
- ssl.SSL_ERROR_WANT_WRITE):
- return
- elif err.args[0] == ssl.SSL_ERROR_EOF:
- return self.handle_close()
- raise
- except socket.error as err:
- if err.args[0] == errno.ECONNABORTED:
- return self.handle_close()
- else:
- self._ssl_accepting = False
-
- def handle_read(self):
- if self._ssl_accepting:
- self._do_ssl_handshake()
- else:
- DummyPOP3Handler.handle_read(self)
-
-requires_ssl = skipUnless(SUPPORTS_SSL, 'SSL not supported')
@requires_ssl
class TestPOP3_SSLClass(TestPOP3Class):
@@ -306,20 +384,60 @@ class TestPOP3_SSLClass(TestPOP3Class):
def test_context(self):
ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
- self.server.port, keyfile=CERTFILE, context=ctx)
+ self.server.port, keyfile=CERTFILE, context=ctx)
self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
- self.server.port, certfile=CERTFILE, context=ctx)
+ self.server.port, certfile=CERTFILE, context=ctx)
self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
- self.server.port, keyfile=CERTFILE,
- certfile=CERTFILE, context=ctx)
+ self.server.port, keyfile=CERTFILE,
+ certfile=CERTFILE, context=ctx)
self.client.quit()
self.client = poplib.POP3_SSL(self.server.host, self.server.port,
- context=ctx)
+ context=ctx)
self.assertIsInstance(self.client.sock, ssl.SSLSocket)
self.assertIs(self.client.sock.context, ctx)
self.assertTrue(self.client.noop().startswith(b'+OK'))
+ def test_stls(self):
+ self.assertRaises(poplib.error_proto, self.client.stls)
+
+ test_stls_context = test_stls
+
+ def test_stls_capa(self):
+ capa = self.client.capa()
+ self.assertFalse('STLS' in capa.keys())
+
+
+@requires_ssl
+class TestPOP3_TLSClass(TestPOP3Class):
+ # repeat previous tests by using poplib.POP3.stls()
+
+ def setUp(self):
+ self.server = DummyPOP3Server((HOST, PORT))
+ self.server.start()
+ self.client = poplib.POP3(self.server.host, self.server.port, timeout=3)
+ self.client.stls()
+
+ def tearDown(self):
+ if self.client.file is not None and self.client.sock is not None:
+ try:
+ self.client.quit()
+ except poplib.error_proto:
+ # happens in the test_too_long_lines case; the overlong
+ # response will be treated as response to QUIT and raise
+ # this exception
+ self.client.close()
+ self.server.stop()
+
+ def test_stls(self):
+ self.assertRaises(poplib.error_proto, self.client.stls)
+
+ test_stls_context = test_stls
+
+ def test_stls_capa(self):
+ capa = self.client.capa()
+ self.assertFalse(b'STLS' in capa.keys())
+
class TestTimeouts(TestCase):
@@ -377,7 +495,7 @@ class TestTimeouts(TestCase):
def test_main():
tests = [TestPOP3Class, TestTimeouts,
- TestPOP3_SSLClass]
+ TestPOP3_SSLClass, TestPOP3_TLSClass]
thread_info = test_support.threading_setup()
try:
test_support.run_unittest(*tests)
diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
index 5c09ebf..9a5ee91 100644
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -603,8 +603,10 @@ class PosixTester(unittest.TestCase):
r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK)
self.addCleanup(os.close, r)
self.addCleanup(os.close, w)
- self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC)
- self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC)
+ self.assertFalse(os.get_inheritable(r))
+ self.assertFalse(os.get_inheritable(w))
+ self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFL) & os.O_NONBLOCK)
+ self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFL) & os.O_NONBLOCK)
# try reading from an empty pipe: this should fail, not block
self.assertRaises(OSError, os.read, r, 1)
# try a write big enough to fill-up the pipe: this should either
@@ -652,7 +654,7 @@ class PosixTester(unittest.TestCase):
self.assertEqual(st.st_flags | stat.UF_IMMUTABLE, new_st.st_flags)
try:
fd = open(target_file, 'w+')
- except IOError as e:
+ except OSError as e:
self.assertEqual(e.errno, errno.EPERM)
finally:
posix.chflags(target_file, st.st_flags)
@@ -710,41 +712,39 @@ class PosixTester(unittest.TestCase):
@unittest.skipUnless(hasattr(posix, 'getcwd'), 'test needs posix.getcwd()')
def test_getcwd_long_pathnames(self):
- if hasattr(posix, 'getcwd'):
- dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef'
- curdir = os.getcwd()
- base_path = os.path.abspath(support.TESTFN) + '.getcwd'
+ dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef'
+ curdir = os.getcwd()
+ base_path = os.path.abspath(support.TESTFN) + '.getcwd'
- try:
- os.mkdir(base_path)
- os.chdir(base_path)
- except:
-# Just returning nothing instead of the SkipTest exception,
-# because the test results in Error in that case.
-# Is that ok?
-# raise unittest.SkipTest("cannot create directory for testing")
- return
-
- def _create_and_do_getcwd(dirname, current_path_length = 0):
- try:
- os.mkdir(dirname)
- except:
- raise unittest.SkipTest("mkdir cannot create directory sufficiently deep for getcwd test")
+ try:
+ os.mkdir(base_path)
+ os.chdir(base_path)
+ except:
+ # Just returning nothing instead of the SkipTest exception, because
+ # the test results in Error in that case. Is that ok?
+ # raise unittest.SkipTest("cannot create directory for testing")
+ return
- os.chdir(dirname)
- try:
- os.getcwd()
- if current_path_length < 1027:
- _create_and_do_getcwd(dirname, current_path_length + len(dirname) + 1)
- finally:
- os.chdir('..')
- os.rmdir(dirname)
+ def _create_and_do_getcwd(dirname, current_path_length = 0):
+ try:
+ os.mkdir(dirname)
+ except:
+ raise unittest.SkipTest("mkdir cannot create directory sufficiently deep for getcwd test")
- _create_and_do_getcwd(dirname)
+ os.chdir(dirname)
+ try:
+ os.getcwd()
+ if current_path_length < 1027:
+ _create_and_do_getcwd(dirname, current_path_length + len(dirname) + 1)
+ finally:
+ os.chdir('..')
+ os.rmdir(dirname)
- finally:
- os.chdir(curdir)
- support.rmtree(base_path)
+ _create_and_do_getcwd(dirname)
+
+ finally:
+ os.chdir(curdir)
+ support.rmtree(base_path)
@unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()")
@unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()")
@@ -1121,6 +1121,23 @@ class PosixTester(unittest.TestCase):
# http://lists.freebsd.org/pipermail/freebsd-amd64/2012-January/014332.html
raise unittest.SkipTest("OSError raised!")
+ def test_path_error2(self):
+ """
+ Test functions that call path_error2(), providing two filenames in their exceptions.
+ """
+ for name in ("rename", "replace", "link", "symlink"):
+ function = getattr(os, name, None)
+
+ if function:
+ for dst in ("noodly2", support.TESTFN):
+ try:
+ function('doesnotexistfilename', dst)
+ except OSError as e:
+ self.assertIn("'doesnotexistfilename' -> '{}'".format(dst), str(e))
+ break
+ else:
+ self.fail("No valid path_error2() test for os." + name)
+
class PosixGroupsTester(unittest.TestCase):
def setUp(self):
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index 0e7d866..412849c 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -186,63 +186,6 @@ class PosixPathTest(unittest.TestCase):
if not f.close():
f.close()
- @staticmethod
- def _create_file(filename):
- with open(filename, 'wb') as f:
- f.write(b'foo')
-
- def test_samefile(self):
- test_fn = support.TESTFN + "1"
- self._create_file(test_fn)
- self.assertTrue(posixpath.samefile(test_fn, test_fn))
- self.assertRaises(TypeError, posixpath.samefile)
-
- @unittest.skipIf(
- sys.platform.startswith('win'),
- "posixpath.samefile does not work on links in Windows")
- @unittest.skipUnless(hasattr(os, "symlink"),
- "Missing symlink implementation")
- def test_samefile_on_links(self):
- test_fn1 = support.TESTFN + "1"
- test_fn2 = support.TESTFN + "2"
- self._create_file(test_fn1)
-
- os.symlink(test_fn1, test_fn2)
- self.assertTrue(posixpath.samefile(test_fn1, test_fn2))
- os.remove(test_fn2)
-
- self._create_file(test_fn2)
- self.assertFalse(posixpath.samefile(test_fn1, test_fn2))
-
-
- def test_samestat(self):
- test_fn = support.TESTFN + "1"
- self._create_file(test_fn)
- test_fns = [test_fn]*2
- stats = map(os.stat, test_fns)
- self.assertTrue(posixpath.samestat(*stats))
-
- @unittest.skipIf(
- sys.platform.startswith('win'),
- "posixpath.samestat does not work on links in Windows")
- @unittest.skipUnless(hasattr(os, "symlink"),
- "Missing symlink implementation")
- def test_samestat_on_links(self):
- test_fn1 = support.TESTFN + "1"
- test_fn2 = support.TESTFN + "2"
- self._create_file(test_fn1)
- test_fns = (test_fn1, test_fn2)
- os.symlink(*test_fns)
- stats = map(os.stat, test_fns)
- self.assertTrue(posixpath.samestat(*stats))
- os.remove(test_fn2)
-
- self._create_file(test_fn2)
- stats = map(os.stat, test_fns)
- self.assertFalse(posixpath.samestat(*stats))
-
- self.assertRaises(TypeError, posixpath.samestat)
-
def test_ismount(self):
self.assertIs(posixpath.ismount("/"), True)
with warnings.catch_warnings():
@@ -595,11 +538,6 @@ class PosixPathTest(unittest.TestCase):
finally:
os.getcwdb = real_getcwdb
- def test_sameopenfile(self):
- fname = support.TESTFN + "1"
- with open(fname, "wb") as a, open(fname, "wb") as b:
- self.assertTrue(posixpath.sameopenfile(a.fileno(), b.fileno()))
-
class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase):
pathmodule = posixpath
diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py
index a85298e..3d364c4 100644
--- a/Lib/test/test_pprint.py
+++ b/Lib/test/test_pprint.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
import pprint
import test.support
import unittest
@@ -530,6 +532,54 @@ frozenset2({0,
self.assertEqual(pprint.pformat(dict.fromkeys(keys, 0)),
'{%r: 0, %r: 0}' % tuple(sorted(keys, key=id)))
+ def test_str_wrap(self):
+ # pprint tries to wrap strings intelligently
+ fox = 'the quick brown fox jumped over a lazy dog'
+ self.assertEqual(pprint.pformat(fox, width=20), """\
+'the quick brown '
+'fox jumped over '
+'a lazy dog'""")
+ self.assertEqual(pprint.pformat({'a': 1, 'b': fox, 'c': 2},
+ width=26), """\
+{'a': 1,
+ 'b': 'the quick brown '
+ 'fox jumped over '
+ 'a lazy dog',
+ 'c': 2}""")
+ # With some special characters
+ # - \n always triggers a new line in the pprint
+ # - \t and \n are escaped
+ # - non-ASCII is allowed
+ # - an apostrophe doesn't disrupt the pprint
+ special = "Portons dix bons \"whiskys\"\nà l'avocat goujat\t qui fumait au zoo"
+ self.assertEqual(pprint.pformat(special, width=20), """\
+'Portons dix bons '
+'"whiskys"\\n'
+"à l'avocat "
+'goujat\\t qui '
+'fumait au zoo'""")
+ # An unwrappable string is formatted as its repr
+ unwrappable = "x" * 100
+ self.assertEqual(pprint.pformat(unwrappable, width=80), repr(unwrappable))
+ self.assertEqual(pprint.pformat(''), "''")
+ # Check that the pprint is a usable repr
+ special *= 10
+ for width in range(3, 40):
+ formatted = pprint.pformat(special, width=width)
+ self.assertEqual(eval("(" + formatted + ")"), special)
+
+ def test_compact(self):
+ o = ([list(range(i * i)) for i in range(5)] +
+ [list(range(i)) for i in range(6)])
+ expected = """\
+[[], [0], [0, 1, 2, 3],
+ [0, 1, 2, 3, 4, 5, 6, 7, 8],
+ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,
+ 14, 15],
+ [], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3],
+ [0, 1, 2, 3, 4]]"""
+ self.assertEqual(pprint.pformat(o, width=48, compact=True), expected)
+
class DottedPrettyPrinter(pprint.PrettyPrinter):
diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py
index 9d6dbea..7eea349 100644
--- a/Lib/test/test_print.py
+++ b/Lib/test/test_print.py
@@ -1,62 +1,55 @@
-"""Test correct operation of the print function.
-"""
-
-# In 2.6, this gives us the behavior we want. In 3.0, it has
-# no function, but it still must parse correctly.
-from __future__ import print_function
-
import unittest
-from test import support
+from io import StringIO
-try:
- # 3.x
- from io import StringIO
-except ImportError:
- # 2.x
- from StringIO import StringIO
+from test import support
NotDefined = object()
# A dispatch table all 8 combinations of providing
-# sep, end, and file
+# sep, end, and file.
# I use this machinery so that I'm not just passing default
-# values to print, I'm either passing or not passing in the
-# arguments
+# values to print, I'm either passing or not passing in the
+# arguments.
dispatch = {
(False, False, False):
- lambda args, sep, end, file: print(*args),
+ lambda args, sep, end, file: print(*args),
(False, False, True):
- lambda args, sep, end, file: print(file=file, *args),
+ lambda args, sep, end, file: print(file=file, *args),
(False, True, False):
- lambda args, sep, end, file: print(end=end, *args),
+ lambda args, sep, end, file: print(end=end, *args),
(False, True, True):
- lambda args, sep, end, file: print(end=end, file=file, *args),
+ lambda args, sep, end, file: print(end=end, file=file, *args),
(True, False, False):
- lambda args, sep, end, file: print(sep=sep, *args),
+ lambda args, sep, end, file: print(sep=sep, *args),
(True, False, True):
- lambda args, sep, end, file: print(sep=sep, file=file, *args),
+ lambda args, sep, end, file: print(sep=sep, file=file, *args),
(True, True, False):
- lambda args, sep, end, file: print(sep=sep, end=end, *args),
+ lambda args, sep, end, file: print(sep=sep, end=end, *args),
(True, True, True):
- lambda args, sep, end, file: print(sep=sep, end=end, file=file, *args),
- }
+ lambda args, sep, end, file: print(sep=sep, end=end, file=file, *args),
+}
+
# Class used to test __str__ and print
class ClassWith__str__:
def __init__(self, x):
self.x = x
+
def __str__(self):
return self.x
+
class TestPrint(unittest.TestCase):
+ """Test correct operation of the print function."""
+
def check(self, expected, args,
- sep=NotDefined, end=NotDefined, file=NotDefined):
+ sep=NotDefined, end=NotDefined, file=NotDefined):
# Capture sys.stdout in a StringIO. Call print with args,
- # and with sep, end, and file, if they're defined. Result
- # must match expected.
+ # and with sep, end, and file, if they're defined. Result
+ # must match expected.
- # Look up the actual function to call, based on if sep, end, and file
- # are defined
+ # Look up the actual function to call, based on if sep, end,
+ # and file are defined.
fn = dispatch[(sep is not NotDefined,
end is not NotDefined,
file is not NotDefined)]
@@ -69,7 +62,7 @@ class TestPrint(unittest.TestCase):
def test_print(self):
def x(expected, args, sep=NotDefined, end=NotDefined):
# Run the test 2 ways: not using file, and using
- # file directed to a StringIO
+ # file directed to a StringIO.
self.check(expected, args, sep=sep, end=end)
@@ -101,11 +94,6 @@ class TestPrint(unittest.TestCase):
x('*\n', (ClassWith__str__('*'),))
x('abc 1\n', (ClassWith__str__('abc'), 1))
-# # 2.x unicode tests
-# x(u'1 2\n', ('1', u'2'))
-# x(u'u\1234\n', (u'u\1234',))
-# x(u' abc 1\n', (' ', ClassWith__str__(u'abc'), 1))
-
# errors
self.assertRaises(TypeError, print, '', sep=3)
self.assertRaises(TypeError, print, '', end=3)
@@ -113,12 +101,14 @@ class TestPrint(unittest.TestCase):
def test_print_flush(self):
# operation of the flush flag
- class filelike():
+ class filelike:
def __init__(self):
self.written = ''
self.flushed = 0
+
def write(self, str):
self.written += str
+
def flush(self):
self.flushed += 1
@@ -130,15 +120,13 @@ class TestPrint(unittest.TestCase):
self.assertEqual(f.flushed, 2)
# ensure exceptions from flush are passed through
- class noflush():
+ class noflush:
def write(self, str):
pass
+
def flush(self):
raise RuntimeError
self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True)
-def test_main():
- support.run_unittest(TestPrint)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py
index cd7ec58..1fc3c42 100644
--- a/Lib/test/test_profile.py
+++ b/Lib/test/test_profile.py
@@ -3,9 +3,11 @@
import sys
import pstats
import unittest
+import os
from difflib import unified_diff
from io import StringIO
-from test.support import run_unittest
+from test.support import TESTFN, run_unittest, unlink
+from contextlib import contextmanager
import profile
from test.profilee import testfunc, timer
@@ -14,9 +16,13 @@ from test.profilee import testfunc, timer
class ProfileTest(unittest.TestCase):
profilerclass = profile.Profile
+ profilermodule = profile
methodnames = ['print_stats', 'print_callers', 'print_callees']
expected_max_output = ':0(max)'
+ def tearDown(self):
+ unlink(TESTFN)
+
def get_expected_output(self):
return _ProfileOutput
@@ -74,6 +80,19 @@ class ProfileTest(unittest.TestCase):
self.assertIn(self.expected_max_output, res,
"Profiling {0!r} didn't report max:\n{1}".format(stmt, res))
+ def test_run(self):
+ with silent():
+ self.profilermodule.run("int('1')")
+ self.profilermodule.run("int('1')", filename=TESTFN)
+ self.assertTrue(os.path.exists(TESTFN))
+
+ def test_runctx(self):
+ with silent():
+ self.profilermodule.runctx("testfunc()", globals(), locals())
+ self.profilermodule.runctx("testfunc()", globals(), locals(),
+ filename=TESTFN)
+ self.assertTrue(os.path.exists(TESTFN))
+
def regenerate_expected_output(filename, cls):
filename = filename.rstrip('co')
@@ -95,6 +114,14 @@ def regenerate_expected_output(filename, cls):
method, results[i+1]))
f.write('\nif __name__ == "__main__":\n main()\n')
+@contextmanager
+def silent():
+ stdout = sys.stdout
+ try:
+ sys.stdout = StringIO()
+ yield
+ finally:
+ sys.stdout = stdout
def test_main():
run_unittest(ProfileTest)
diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py
index 29297f8..8916861 100644
--- a/Lib/test/test_pty.py
+++ b/Lib/test/test_pty.py
@@ -187,7 +187,7 @@ class PtyTest(unittest.TestCase):
##debug("Reading from master_fd now that the child has exited")
##try:
## s1 = os.read(master_fd, 1024)
- ##except os.error:
+ ##except OSError:
## pass
##else:
## raise TestFailed("Read from master_fd did not raise exception")
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index f3c1a6a..154c08a 100644
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -1,7 +1,9 @@
-import imp
+import importlib.util
import os
import py_compile
import shutil
+import stat
+import sys
import tempfile
import unittest
@@ -13,7 +15,7 @@ class PyCompileTests(unittest.TestCase):
self.directory = tempfile.mkdtemp()
self.source_path = os.path.join(self.directory, '_test.py')
self.pyc_path = self.source_path + 'c'
- self.cache_path = imp.cache_from_source(self.source_path)
+ self.cache_path = importlib.util.cache_from_source(self.source_path)
self.cwd_drive = os.path.splitdrive(os.getcwd())[0]
# In these tests we compute relative paths. When using Windows, the
# current working directory path and the 'self.source_path' might be
@@ -35,6 +37,26 @@ class PyCompileTests(unittest.TestCase):
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
+ def test_do_not_overwrite_symlinks(self):
+ # In the face of a cfile argument being a symlink, bail out.
+ # Issue #17222
+ try:
+ os.symlink(self.pyc_path + '.actual', self.pyc_path)
+ except (NotImplementedError, OSError):
+ self.skipTest('need to be able to create a symlink for a file')
+ else:
+ assert os.path.islink(self.pyc_path)
+ with self.assertRaises(FileExistsError):
+ py_compile.compile(self.source_path, self.pyc_path)
+
+ @unittest.skipIf(not os.path.exists(os.devnull) or os.path.isfile(os.devnull),
+ 'requires os.devnull and for it to be a non-regular file')
+ def test_do_not_overwrite_nonregular_files(self):
+ # In the face of a cfile argument being a non-regular file, bail out.
+ # Issue #17222
+ with self.assertRaises(FileExistsError):
+ py_compile.compile(self.source_path, os.devnull)
+
def test_cache_path(self):
py_compile.compile(self.source_path)
self.assertTrue(os.path.exists(self.cache_path))
@@ -54,8 +76,22 @@ class PyCompileTests(unittest.TestCase):
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
-def test_main():
- support.run_unittest(PyCompileTests)
+ @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
+ 'non-root user required')
+ @unittest.skipIf(os.name == 'nt',
+ 'cannot control directory permissions on Windows')
+ def test_exceptions_propagate(self):
+ # Make sure that exceptions raised thanks to issues with writing
+ # bytecode.
+ # http://bugs.python.org/issue17244
+ mode = os.stat(self.directory)
+ os.chmod(self.directory, stat.S_IREAD)
+ try:
+ with self.assertRaises(IOError):
+ py_compile.compile(self.source_path, self.pyc_path)
+ finally:
+ os.chmod(self.directory, mode.st_mode)
+
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py
index e83989e..88aff89 100644
--- a/Lib/test/test_pyclbr.py
+++ b/Lib/test/test_pyclbr.py
@@ -142,7 +142,7 @@ class PyclbrTest(TestCase):
self.checkModule('pyclbr')
self.checkModule('ast')
self.checkModule('doctest', ignore=("TestResults", "_SpoofOut",
- "DocTestCase"))
+ "DocTestCase", '_DocTestSuite'))
self.checkModule('difflib', ignore=("Match",))
def test_decorators(self):
@@ -158,7 +158,7 @@ class PyclbrTest(TestCase):
cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator
cm('cgi', ignore=('log',)) # set with = in module
cm('pickle')
- cm('aifc', ignore=('openfp',)) # set with = in module
+ cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module
cm('sre_parse', ignore=('dump',)) # from sre_constants import *
cm('pdb')
cm('pydoc')
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
index 43f4163..f26fb15 100644
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -6,11 +6,13 @@ import difflib
import inspect
import pydoc
import keyword
+import _pickle
import pkgutil
import re
import string
import test.support
import time
+import types
import unittest
import xml.etree
import textwrap
@@ -20,7 +22,7 @@ from test.script_helper import assert_python_ok
from test.support import (
TESTFN, rmtree,
reap_children, reap_threads, captured_output, captured_stdout,
- captured_stderr, unlink
+ captured_stderr, unlink, requires_docstrings
)
from test import pydoc_mod
@@ -29,10 +31,6 @@ try:
except ImportError:
threading = None
-# Just in case sys.modules["test"] has the optional attribute __loader__.
-if hasattr(pydoc_mod, "__loader__"):
- del pydoc_mod.__loader__
-
if test.support.HAVE_DOCSTRINGS:
expected_data_docstrings = (
'dictionary for instance variables (if defined)',
@@ -212,6 +210,75 @@ missing_pattern = "no Python documentation found for '%s'"
# output pattern for module with bad imports
badimport_pattern = "problem in %s - ImportError: No module named %r"
+expected_dynamicattribute_pattern = """
+Help on class DA in module %s:
+
+class DA(builtins.object)
+ | Data descriptors defined here:
+ |\x20\x20
+ | __dict__%s
+ |\x20\x20
+ | __weakref__%s
+ |\x20\x20
+ | ham
+ |\x20\x20
+ | ----------------------------------------------------------------------
+ | Data and other attributes inherited from Meta:
+ |\x20\x20
+ | ham = 'spam'
+""".strip()
+
+expected_virtualattribute_pattern1 = """
+Help on class Class in module %s:
+
+class Class(builtins.object)
+ | Data and other attributes inherited from Meta:
+ |\x20\x20
+ | LIFE = 42
+""".strip()
+
+expected_virtualattribute_pattern2 = """
+Help on class Class1 in module %s:
+
+class Class1(builtins.object)
+ | Data and other attributes inherited from Meta1:
+ |\x20\x20
+ | one = 1
+""".strip()
+
+expected_virtualattribute_pattern3 = """
+Help on class Class2 in module %s:
+
+class Class2(Class1)
+ | Method resolution order:
+ | Class2
+ | Class1
+ | builtins.object
+ |\x20\x20
+ | Data and other attributes inherited from Meta1:
+ |\x20\x20
+ | one = 1
+ |\x20\x20
+ | ----------------------------------------------------------------------
+ | Data and other attributes inherited from Meta3:
+ |\x20\x20
+ | three = 3
+ |\x20\x20
+ | ----------------------------------------------------------------------
+ | Data and other attributes inherited from Meta2:
+ |\x20\x20
+ | two = 2
+""".strip()
+
+expected_missingattribute_pattern = """
+Help on class C in module %s:
+
+class C(builtins.object)
+ | Data and other attributes defined here:
+ |\x20\x20
+ | here = 'present!'
+""".strip()
+
def run_pydoc(module_name, *args, **env):
"""
Runs pydoc on the specified module. Returns the stripped
@@ -320,6 +387,16 @@ class PydocDocTest(unittest.TestCase):
print_diffs(expected_text, result)
self.fail("outputs are not equal, see diff above")
+ def test_text_enum_member_with_value_zero(self):
+ # Test issue #20654 to ensure enum member with value 0 can be
+ # displayed. It used to throw KeyError: 'zero'.
+ import enum
+ class BinaryInteger(enum.IntEnum):
+ zero = 0
+ one = 1
+ doc = pydoc.render_doc(BinaryInteger)
+ self.assertIn('<BinaryInteger.zero: 0>', doc)
+
def test_issue8225(self):
# Test issue8225 to ensure no doc link appears for xml.etree
result, doc_loc = get_pydoc_text(xml.etree)
@@ -421,6 +498,38 @@ class PydocDocTest(unittest.TestCase):
synopsis = pydoc.synopsis(TESTFN, {})
self.assertEqual(synopsis, 'line 1: h\xe9')
+ def test_synopsis_sourceless(self):
+ expected = os.__doc__.splitlines()[0]
+ filename = os.__cached__
+ synopsis = pydoc.synopsis(filename)
+
+ self.assertEqual(synopsis, expected)
+
+ def test_splitdoc_with_description(self):
+ example_string = "I Am A Doc\n\n\nHere is my description"
+ self.assertEqual(pydoc.splitdoc(example_string),
+ ('I Am A Doc', '\nHere is my description'))
+
+ def test_is_object_or_method(self):
+ doc = pydoc.Doc()
+ # Bound Method
+ self.assertTrue(pydoc._is_some_method(doc.fail))
+ # Method Descriptor
+ self.assertTrue(pydoc._is_some_method(int.__add__))
+ # String
+ self.assertFalse(pydoc._is_some_method("I am not a method"))
+
+ def test_is_package_when_not_package(self):
+ with test.support.temp_cwd() as test_dir:
+ self.assertFalse(pydoc.ispackage(test_dir))
+
+ def test_is_package_when_is_package(self):
+ with test.support.temp_cwd() as test_dir:
+ init_path = os.path.join(test_dir, '__init__.py')
+ open(init_path, 'w').close()
+ self.assertTrue(pydoc.ispackage(test_dir))
+ os.remove(init_path)
+
def test_allmethods(self):
# issue 17476: allmethods was no longer returning unbound methods.
# This test is a bit fragile in the face of changes to object and type,
@@ -509,6 +618,55 @@ class PydocImportTest(PydocBaseTest):
self.assertEqual(out.getvalue(), '')
self.assertEqual(err.getvalue(), '')
+ @unittest.skip('causes undesireable side-effects (#20128)')
+ def test_modules(self):
+ # See Helper.listmodules().
+ num_header_lines = 2
+ num_module_lines_min = 5 # Playing it safe.
+ num_footer_lines = 3
+ expected = num_header_lines + num_module_lines_min + num_footer_lines
+
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper('modules')
+ result = output.getvalue().strip()
+ num_lines = len(result.splitlines())
+
+ self.assertGreaterEqual(num_lines, expected)
+
+ @unittest.skip('causes undesireable side-effects (#20128)')
+ def test_modules_search(self):
+ # See Helper.listmodules().
+ expected = 'pydoc - '
+
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ with captured_stdout() as help_io:
+ helper('modules pydoc')
+ result = help_io.getvalue()
+
+ self.assertIn(expected, result)
+
+ @unittest.skip('some buildbots are not cooperating (#20128)')
+ def test_modules_search_builtin(self):
+ expected = 'gc - '
+
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ with captured_stdout() as help_io:
+ helper('modules garbage')
+ result = help_io.getvalue()
+
+ self.assertTrue(result.startswith(expected))
+
+ def test_importfile(self):
+ loaded_pydoc = pydoc.importfile(pydoc.__file__)
+
+ self.assertIsNot(loaded_pydoc, pydoc)
+ self.assertEqual(loaded_pydoc.__name__, 'pydoc')
+ self.assertEqual(loaded_pydoc.__file__, pydoc.__file__)
+ self.assertEqual(loaded_pydoc.__spec__, pydoc.__spec__)
+
class TestDescriptions(unittest.TestCase):
@@ -544,6 +702,42 @@ class TestDescriptions(unittest.TestCase):
self.assertIsNone(pydoc.locate(name))
self.assertRaises(ImportError, pydoc.render_doc, name)
+ @staticmethod
+ def _get_summary_line(o):
+ text = pydoc.plain(pydoc.render_doc(o))
+ lines = text.split('\n')
+ assert len(lines) >= 2
+ return lines[2]
+
+ # these should include "self"
+ def test_unbound_python_method(self):
+ self.assertEqual(self._get_summary_line(textwrap.TextWrapper.wrap),
+ "wrap(self, text)")
+
+ @requires_docstrings
+ def test_unbound_builtin_method(self):
+ self.assertEqual(self._get_summary_line(_pickle.Pickler.dump),
+ "dump(self, obj, /)")
+
+ # these no longer include "self"
+ def test_bound_python_method(self):
+ t = textwrap.TextWrapper()
+ self.assertEqual(self._get_summary_line(t.wrap),
+ "wrap(text) method of textwrap.TextWrapper instance")
+
+ @requires_docstrings
+ def test_bound_builtin_method(self):
+ s = StringIO()
+ p = _pickle.Pickler(s)
+ self.assertEqual(self._get_summary_line(p.dump),
+ "dump(obj, /) method of _pickle.Pickler instance")
+
+ # this should *never* include self!
+ @requires_docstrings
+ def test_module_level_callable(self):
+ self.assertEqual(self._get_summary_line(os.stat),
+ "stat(path, *, dir_fd=None, follow_symlinks=True)")
+
@unittest.skipUnless(threading, 'Threading required for this test.')
class PydocServerTest(unittest.TestCase):
@@ -615,6 +809,128 @@ class TestHelper(unittest.TestCase):
self.assertEqual(sorted(pydoc.Helper.keywords),
sorted(keyword.kwlist))
+class PydocWithMetaClasses(unittest.TestCase):
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+ 'trace function introduces __locals__ unexpectedly')
+ def test_DynamicClassAttribute(self):
+ class Meta(type):
+ def __getattr__(self, name):
+ if name == 'ham':
+ return 'spam'
+ return super().__getattr__(name)
+ class DA(metaclass=Meta):
+ @types.DynamicClassAttribute
+ def ham(self):
+ return 'eggs'
+ expected_text_data_docstrings = tuple('\n | ' + s if s else ''
+ for s in expected_data_docstrings)
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(DA)
+ expected_text = expected_dynamicattribute_pattern % (
+ (__name__,) + expected_text_data_docstrings[:2])
+ result = output.getvalue().strip()
+ if result != expected_text:
+ print_diffs(expected_text, result)
+ self.fail("outputs are not equal, see diff above")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+ 'trace function introduces __locals__ unexpectedly')
+ def test_virtualClassAttributeWithOneMeta(self):
+ class Meta(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'LIFE']
+ def __getattr__(self, name):
+ if name =='LIFE':
+ return 42
+ return super().__getattr(name)
+ class Class(metaclass=Meta):
+ pass
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(Class)
+ expected_text = expected_virtualattribute_pattern1 % __name__
+ result = output.getvalue().strip()
+ if result != expected_text:
+ print_diffs(expected_text, result)
+ self.fail("outputs are not equal, see diff above")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+ 'trace function introduces __locals__ unexpectedly')
+ def test_virtualClassAttributeWithTwoMeta(self):
+ class Meta1(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'one']
+ def __getattr__(self, name):
+ if name =='one':
+ return 1
+ return super().__getattr__(name)
+ class Meta2(type):
+ def __dir__(cls):
+ return ['__class__', '__module__', '__name__', 'two']
+ def __getattr__(self, name):
+ if name =='two':
+ return 2
+ return super().__getattr__(name)
+ class Meta3(Meta1, Meta2):
+ def __dir__(cls):
+ return list(sorted(set(
+ ['__class__', '__module__', '__name__', 'three'] +
+ Meta1.__dir__(cls) + Meta2.__dir__(cls))))
+ def __getattr__(self, name):
+ if name =='three':
+ return 3
+ return super().__getattr__(name)
+ class Class1(metaclass=Meta1):
+ pass
+ class Class2(Class1, metaclass=Meta3):
+ pass
+ fail1 = fail2 = False
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(Class1)
+ expected_text1 = expected_virtualattribute_pattern2 % __name__
+ result1 = output.getvalue().strip()
+ if result1 != expected_text1:
+ print_diffs(expected_text1, result1)
+ fail1 = True
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(Class2)
+ expected_text2 = expected_virtualattribute_pattern3 % __name__
+ result2 = output.getvalue().strip()
+ if result2 != expected_text2:
+ print_diffs(expected_text2, result2)
+ fail2 = True
+ if fail1 or fail2:
+ self.fail("outputs are not equal, see diff above")
+
+ @unittest.skipIf(sys.flags.optimize >= 2,
+ "Docstrings are omitted with -O2 and above")
+ @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(),
+ 'trace function introduces __locals__ unexpectedly')
+ def test_buggy_dir(self):
+ class M(type):
+ def __dir__(cls):
+ return ['__class__', '__name__', 'missing', 'here']
+ class C(metaclass=M):
+ here = 'present!'
+ output = StringIO()
+ helper = pydoc.Helper(output=output)
+ helper(C)
+ expected_text = expected_missingattribute_pattern % __name__
+ result = output.getvalue().strip()
+ if result != expected_text:
+ print_diffs(expected_text, result)
+ self.fail("outputs are not equal, see diff above")
+
+
@reap_threads
def test_main():
try:
@@ -624,6 +940,7 @@ def test_main():
PydocServerTest,
PydocUrlHandlerTest,
TestHelper,
+ PydocWithMetaClasses,
)
finally:
reap_children()
diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
index 8ef3917..95a614b 100644
--- a/Lib/test/test_pyexpat.py
+++ b/Lib/test/test_pyexpat.py
@@ -236,6 +236,18 @@ class ParseTest(unittest.TestCase):
operations = out.out
self._verify_parse_output(operations)
+ def test_parse_again(self):
+ parser = expat.ParserCreate()
+ file = BytesIO(data)
+ parser.ParseFile(file)
+ # Issue 6676: ensure a meaningful exception is raised when attempting
+ # to parse more than one XML document per xmlparser instance,
+ # a limitation of the Expat library.
+ with self.assertRaises(expat.error) as cm:
+ parser.ParseFile(file)
+ self.assertEqual(expat.ErrorString(cm.exception.code),
+ expat.errors.XML_ERROR_FINISHED)
+
class NamespaceSeparatorTest(unittest.TestCase):
def test_legal(self):
# Tests that make sure we get errors when the namespace_separator value
diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py
index a475207..37a222d 100644
--- a/Lib/test/test_random.py
+++ b/Lib/test/test_random.py
@@ -1,8 +1,10 @@
import unittest
+import unittest.mock
import random
import time
import pickle
import warnings
+from functools import partial
from math import log, exp, pi, fsum, sin
from test import support
@@ -44,6 +46,48 @@ class TestBasicOps:
self.assertRaises(TypeError, self.gen.seed, 1, 2, 3, 4)
self.assertRaises(TypeError, type(self.gen), [])
+ @unittest.mock.patch('random._urandom') # os.urandom
+ def test_seed_when_randomness_source_not_found(self, urandom_mock):
+ # Random.seed() uses time.time() when an operating system specific
+ # randomness source is not found. To test this on machines were it
+ # exists, run the above test, test_seedargs(), again after mocking
+ # os.urandom() so that it raises the exception expected when the
+ # randomness source is not available.
+ urandom_mock.side_effect = NotImplementedError
+ self.test_seedargs()
+
+ def test_shuffle(self):
+ shuffle = self.gen.shuffle
+ lst = []
+ shuffle(lst)
+ self.assertEqual(lst, [])
+ lst = [37]
+ shuffle(lst)
+ self.assertEqual(lst, [37])
+ seqs = [list(range(n)) for n in range(10)]
+ shuffled_seqs = [list(range(n)) for n in range(10)]
+ for shuffled_seq in shuffled_seqs:
+ shuffle(shuffled_seq)
+ for (seq, shuffled_seq) in zip(seqs, shuffled_seqs):
+ self.assertEqual(len(seq), len(shuffled_seq))
+ self.assertEqual(set(seq), set(shuffled_seq))
+ # The above tests all would pass if the shuffle was a
+ # no-op. The following non-deterministic test covers that. It
+ # asserts that the shuffled sequence of 1000 distinct elements
+ # must be different from the original one. Although there is
+ # mathematically a non-zero probability that this could
+ # actually happen in a genuinely random shuffle, it is
+ # completely negligible, given that the number of possible
+ # permutations of 1000 objects is 1000! (factorial of 1000),
+ # which is considerably larger than the number of atoms in the
+ # universe...
+ lst = list(range(1000))
+ shuffled_lst = list(range(1000))
+ shuffle(shuffled_lst)
+ self.assertTrue(lst != shuffled_lst)
+ shuffle(lst)
+ self.assertTrue(lst != shuffled_lst)
+
def test_choice(self):
choice = self.gen.choice
with self.assertRaises(IndexError):
@@ -63,6 +107,8 @@ class TestBasicOps:
self.assertEqual(len(uniq), k)
self.assertTrue(uniq <= set(population))
self.assertEqual(self.gen.sample([], 0), []) # test edge case N==k==0
+ # Exception raised if size of sample exceeds that of population
+ self.assertRaises(ValueError, self.gen.sample, population, N+1)
def test_sample_distribution(self):
# For the entire allowable range of 0 <= k <= N, validate that
@@ -203,6 +249,25 @@ class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase):
self.assertEqual(set(range(start,stop)),
set([self.gen.randrange(start,stop) for i in range(100)]))
+ def test_randrange_nonunit_step(self):
+ rint = self.gen.randrange(0, 10, 2)
+ self.assertIn(rint, (0, 2, 4, 6, 8))
+ rint = self.gen.randrange(0, 2, 2)
+ self.assertEqual(rint, 0)
+
+ def test_randrange_errors(self):
+ raises = partial(self.assertRaises, ValueError, self.gen.randrange)
+ # Empty range
+ raises(3, 3)
+ raises(-721)
+ raises(0, 100, -12)
+ # Non-integer start/stop
+ raises(3.14159)
+ raises(0, 2.71828)
+ # Zero and non-integer step
+ raises(0, 42, 0)
+ raises(0, 42, 3.14159)
+
def test_genrandbits(self):
# Verify ranges
for k in range(1, 1000):
@@ -272,6 +337,16 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase):
# Last element s/b an int also
self.assertRaises(TypeError, self.gen.setstate, (2, (0,)*624+('a',), None))
+ # Little trick to make "tuple(x % (2**32) for x in internalstate)"
+ # raise ValueError. I cannot think of a simple way to achieve this, so
+ # I am opting for using a generator as the middle argument of setstate
+ # which attempts to cast a NaN to integer.
+ state_values = self.gen.getstate()[1]
+ state_values = list(state_values)
+ state_values[-1] = float('nan')
+ state = (int(x) for x in state_values)
+ self.assertRaises(TypeError, self.gen.setstate, (2, state, None))
+
def test_referenceImplementation(self):
# Compare the python implementation with results from the original
# code. Create 2000 53-bit precision random floats. Compare only
@@ -411,6 +486,38 @@ class MersenneTwister_TestBasicOps(TestBasicOps, unittest.TestCase):
self.assertEqual(k, numbits) # note the stronger assertion
self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion
+ @unittest.mock.patch('random.Random.random')
+ def test_randbelow_overriden_random(self, random_mock):
+ # Random._randbelow() can only use random() when the built-in one
+ # has been overridden but no new getrandbits() method was supplied.
+ random_mock.side_effect = random.SystemRandom().random
+ maxsize = 1<<random.BPF
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", UserWarning)
+ # Population range too large (n >= maxsize)
+ self.gen._randbelow(maxsize+1, maxsize = maxsize)
+ self.gen._randbelow(5640, maxsize = maxsize)
+
+ # This might be going too far to test a single line, but because of our
+ # noble aim of achieving 100% test coverage we need to write a case in
+ # which the following line in Random._randbelow() gets executed:
+ #
+ # rem = maxsize % n
+ # limit = (maxsize - rem) / maxsize
+ # r = random()
+ # while r >= limit:
+ # r = random() # <== *This line* <==<
+ #
+ # Therefore, to guarantee that the while loop is executed at least
+ # once, we need to mock random() so that it returns a number greater
+ # than 'limit' the first time it gets called.
+
+ n = 42
+ epsilon = 0.01
+ limit = (maxsize - (maxsize % n)) / maxsize
+ random_mock.side_effect = [limit + epsilon, limit - epsilon]
+ self.gen._randbelow(n, maxsize = maxsize)
+
def test_randrange_bug_1590891(self):
start = 1000000000000
stop = -100000000000000000000
@@ -528,6 +635,106 @@ class TestDistributions(unittest.TestCase):
random.vonmisesvariate(0, 1e15)
random.vonmisesvariate(0, 1e100)
+ def test_gammavariate_errors(self):
+ # Both alpha and beta must be > 0.0
+ self.assertRaises(ValueError, random.gammavariate, -1, 3)
+ self.assertRaises(ValueError, random.gammavariate, 0, 2)
+ self.assertRaises(ValueError, random.gammavariate, 2, 0)
+ self.assertRaises(ValueError, random.gammavariate, 1, -3)
+
+ @unittest.mock.patch('random.Random.random')
+ def test_gammavariate_full_code_coverage(self, random_mock):
+ # There are three different possibilities in the current implementation
+ # of random.gammavariate(), depending on the value of 'alpha'. What we
+ # are going to do here is to fix the values returned by random() to
+ # generate test cases that provide 100% line coverage of the method.
+
+ # #1: alpha > 1.0: we want the first random number to be outside the
+ # [1e-7, .9999999] range, so that the continue statement executes
+ # once. The values of u1 and u2 will be 0.5 and 0.3, respectively.
+ random_mock.side_effect = [1e-8, 0.5, 0.3]
+ returned_value = random.gammavariate(1.1, 2.3)
+ self.assertAlmostEqual(returned_value, 2.53)
+
+ # #2: alpha == 1: first random number less than 1e-7 to that the body
+ # of the while loop executes once. Then random.random() returns 0.45,
+ # which causes while to stop looping and the algorithm to terminate.
+ random_mock.side_effect = [1e-8, 0.45]
+ returned_value = random.gammavariate(1.0, 3.14)
+ self.assertAlmostEqual(returned_value, 2.507314166123803)
+
+ # #3: 0 < alpha < 1. This is the most complex region of code to cover,
+ # as there are multiple if-else statements. Let's take a look at the
+ # source code, and determine the values that we need accordingly:
+ #
+ # while 1:
+ # u = random()
+ # b = (_e + alpha)/_e
+ # p = b*u
+ # if p <= 1.0: # <=== (A)
+ # x = p ** (1.0/alpha)
+ # else: # <=== (B)
+ # x = -_log((b-p)/alpha)
+ # u1 = random()
+ # if p > 1.0: # <=== (C)
+ # if u1 <= x ** (alpha - 1.0): # <=== (D)
+ # break
+ # elif u1 <= _exp(-x): # <=== (E)
+ # break
+ # return x * beta
+ #
+ # First, we want (A) to be True. For that we need that:
+ # b*random() <= 1.0
+ # r1 = random() <= 1.0 / b
+ #
+ # We now get to the second if-else branch, and here, since p <= 1.0,
+ # (C) is False and we take the elif branch, (E). For it to be True,
+ # so that the break is executed, we need that:
+ # r2 = random() <= _exp(-x)
+ # r2 <= _exp(-(p ** (1.0/alpha)))
+ # r2 <= _exp(-((b*r1) ** (1.0/alpha)))
+
+ _e = random._e
+ _exp = random._exp
+ _log = random._log
+ alpha = 0.35
+ beta = 1.45
+ b = (_e + alpha)/_e
+ epsilon = 0.01
+
+ r1 = 0.8859296441566 # 1.0 / b
+ r2 = 0.3678794411714 # _exp(-((b*r1) ** (1.0/alpha)))
+
+ # These four "random" values result in the following trace:
+ # (A) True, (E) False --> [next iteration of while]
+ # (A) True, (E) True --> [while loop breaks]
+ random_mock.side_effect = [r1, r2 + epsilon, r1, r2]
+ returned_value = random.gammavariate(alpha, beta)
+ self.assertAlmostEqual(returned_value, 1.4499999999997544)
+
+ # Let's now make (A) be False. If this is the case, when we get to the
+ # second if-else 'p' is greater than 1, so (C) evaluates to True. We
+ # now encounter a second if statement, (D), which in order to execute
+ # must satisfy the following condition:
+ # r2 <= x ** (alpha - 1.0)
+ # r2 <= (-_log((b-p)/alpha)) ** (alpha - 1.0)
+ # r2 <= (-_log((b-(b*r1))/alpha)) ** (alpha - 1.0)
+ r1 = 0.8959296441566 # (1.0 / b) + epsilon -- so that (A) is False
+ r2 = 0.9445400408898141
+
+ # And these four values result in the following trace:
+ # (B) and (C) True, (D) False --> [next iteration of while]
+ # (B) and (C) True, (D) True [while loop breaks]
+ random_mock.side_effect = [r1, r2 + epsilon, r1, r2]
+ returned_value = random.gammavariate(alpha, beta)
+ self.assertAlmostEqual(returned_value, 1.5830349561760781)
+
+ @unittest.mock.patch('random.Random.gammavariate')
+ def test_betavariate_return_zero(self, gammavariate_mock):
+ # betavariate() returns zero when the Gamma distribution
+ # that it uses internally returns this same value.
+ gammavariate_mock.return_value = 0.0
+ self.assertEqual(0.0, random.betavariate(2.71828, 3.14159))
class TestModule(unittest.TestCase):
def testMagicConstants(self):
diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py
index aab98ab..fecda1b 100644
--- a/Lib/test/test_range.py
+++ b/Lib/test/test_range.py
@@ -313,7 +313,7 @@ class RangeTest(unittest.TestCase):
self.assertRaises(TypeError, range, IN())
# Test use of user-defined classes in slice indices.
- self.assertEqual(list(range(10)[:I(5)]), list(range(5)))
+ self.assertEqual(range(10)[:I(5)], range(5))
with self.assertRaises(RuntimeError):
range(0, 10)[:IX()]
@@ -353,9 +353,10 @@ class RangeTest(unittest.TestCase):
(13, 21, 3), (-2, 2, 2), (2**65, 2**65+2)]
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
for t in testcases:
- r = range(*t)
- self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
- list(r))
+ with self.subTest(proto=proto, test=t):
+ r = range(*t)
+ self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
+ list(r))
def test_iterator_pickling(self):
testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
index 5466b20..33ccd15 100644
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -3,10 +3,12 @@ from test.support import verbose, run_unittest, gc_collect, bigmemtest, _2G, \
import io
import re
from re import Scanner
+import sre_compile
import sre_constants
import sys
import string
import traceback
+import unittest
from weakref import proxy
# Misc tests from Tim Peters' re.doc
@@ -15,10 +17,26 @@ from weakref import proxy
# what you're doing. Some of these tests were carefully modeled to
# cover most of the code.
-import unittest
+class S(str):
+ def __getitem__(self, index):
+ return S(super().__getitem__(index))
+
+class B(bytes):
+ def __getitem__(self, index):
+ return B(super().__getitem__(index))
class ReTests(unittest.TestCase):
+ def assertTypedEqual(self, actual, expect, msg=None):
+ self.assertEqual(actual, expect, msg)
+ def recurse(actual, expect):
+ if isinstance(expect, (tuple, list)):
+ for x, y in zip(actual, expect):
+ recurse(x, y)
+ else:
+ self.assertIs(type(actual), type(expect), msg)
+ recurse(actual, expect)
+
def test_keep_buffer(self):
# See bug 14212
b = bytearray(b'x')
@@ -53,6 +71,15 @@ class ReTests(unittest.TestCase):
return str(int_value + 1)
def test_basic_re_sub(self):
+ self.assertTypedEqual(re.sub('y', 'a', 'xyz'), 'xaz')
+ self.assertTypedEqual(re.sub('y', S('a'), S('xyz')), 'xaz')
+ self.assertTypedEqual(re.sub(b'y', b'a', b'xyz'), b'xaz')
+ self.assertTypedEqual(re.sub(b'y', B(b'a'), B(b'xyz')), b'xaz')
+ self.assertTypedEqual(re.sub(b'y', bytearray(b'a'), bytearray(b'xyz')), b'xaz')
+ self.assertTypedEqual(re.sub(b'y', memoryview(b'a'), memoryview(b'xyz')), b'xaz')
+ for y in ("\xe0", "\u0430", "\U0001d49c"):
+ self.assertEqual(re.sub(y, 'a', 'x%sz' % y), 'xaz')
+
self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x')
self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'),
'9.3 -3 24x100y')
@@ -210,10 +237,29 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2))
def test_re_split(self):
- self.assertEqual(re.split(":", ":a:b::c"), ['', 'a', 'b', '', 'c'])
- self.assertEqual(re.split(":*", ":a:b::c"), ['', 'a', 'b', 'c'])
- self.assertEqual(re.split("(:*)", ":a:b::c"),
- ['', ':', 'a', ':', 'b', '::', 'c'])
+ for string in ":a:b::c", S(":a:b::c"):
+ self.assertTypedEqual(re.split(":", string),
+ ['', 'a', 'b', '', 'c'])
+ self.assertTypedEqual(re.split(":*", string),
+ ['', 'a', 'b', 'c'])
+ self.assertTypedEqual(re.split("(:*)", string),
+ ['', ':', 'a', ':', 'b', '::', 'c'])
+ for string in (b":a:b::c", B(b":a:b::c"), bytearray(b":a:b::c"),
+ memoryview(b":a:b::c")):
+ self.assertTypedEqual(re.split(b":", string),
+ [b'', b'a', b'b', b'', b'c'])
+ self.assertTypedEqual(re.split(b":*", string),
+ [b'', b'a', b'b', b'c'])
+ self.assertTypedEqual(re.split(b"(:*)", string),
+ [b'', b':', b'a', b':', b'b', b'::', b'c'])
+ for a, b, c in ("\xe0\xdf\xe7", "\u0430\u0431\u0432",
+ "\U0001d49c\U0001d49e\U0001d4b5"):
+ string = ":%s:%s::%s" % (a, b, c)
+ self.assertEqual(re.split(":", string), ['', a, b, '', c])
+ self.assertEqual(re.split(":*", string), ['', a, b, c])
+ self.assertEqual(re.split("(:*)", string),
+ ['', ':', a, ':', b, '::', c])
+
self.assertEqual(re.split("(?::*)", ":a:b::c"), ['', 'a', 'b', 'c'])
self.assertEqual(re.split("(:)*", ":a:b::c"),
['', ':', 'a', ':', 'b', ':', 'c'])
@@ -235,22 +281,53 @@ class ReTests(unittest.TestCase):
def test_re_findall(self):
self.assertEqual(re.findall(":+", "abc"), [])
- self.assertEqual(re.findall(":+", "a:b::c:::d"), [":", "::", ":::"])
- self.assertEqual(re.findall("(:+)", "a:b::c:::d"), [":", "::", ":::"])
- self.assertEqual(re.findall("(:)(:*)", "a:b::c:::d"), [(":", ""),
- (":", ":"),
- (":", "::")])
+ for string in "a:b::c:::d", S("a:b::c:::d"):
+ self.assertTypedEqual(re.findall(":+", string),
+ [":", "::", ":::"])
+ self.assertTypedEqual(re.findall("(:+)", string),
+ [":", "::", ":::"])
+ self.assertTypedEqual(re.findall("(:)(:*)", string),
+ [(":", ""), (":", ":"), (":", "::")])
+ for string in (b"a:b::c:::d", B(b"a:b::c:::d"), bytearray(b"a:b::c:::d"),
+ memoryview(b"a:b::c:::d")):
+ self.assertTypedEqual(re.findall(b":+", string),
+ [b":", b"::", b":::"])
+ self.assertTypedEqual(re.findall(b"(:+)", string),
+ [b":", b"::", b":::"])
+ self.assertTypedEqual(re.findall(b"(:)(:*)", string),
+ [(b":", b""), (b":", b":"), (b":", b"::")])
+ for x in ("\xe0", "\u0430", "\U0001d49c"):
+ xx = x * 2
+ xxx = x * 3
+ string = "a%sb%sc%sd" % (x, xx, xxx)
+ self.assertEqual(re.findall("%s+" % x, string), [x, xx, xxx])
+ self.assertEqual(re.findall("(%s+)" % x, string), [x, xx, xxx])
+ self.assertEqual(re.findall("(%s)(%s*)" % (x, x), string),
+ [(x, ""), (x, x), (x, xx)])
def test_bug_117612(self):
self.assertEqual(re.findall(r"(a|(b))", "aba"),
[("a", ""),("b", "b"),("a", "")])
def test_re_match(self):
- self.assertEqual(re.match('a', 'a').groups(), ())
- self.assertEqual(re.match('(a)', 'a').groups(), ('a',))
- self.assertEqual(re.match(r'(a)', 'a').group(0), 'a')
- self.assertEqual(re.match(r'(a)', 'a').group(1), 'a')
- self.assertEqual(re.match(r'(a)', 'a').group(1, 1), ('a', 'a'))
+ for string in 'a', S('a'):
+ self.assertEqual(re.match('a', string).groups(), ())
+ self.assertEqual(re.match('(a)', string).groups(), ('a',))
+ self.assertEqual(re.match('(a)', string).group(0), 'a')
+ self.assertEqual(re.match('(a)', string).group(1), 'a')
+ self.assertEqual(re.match('(a)', string).group(1, 1), ('a', 'a'))
+ for string in b'a', B(b'a'), bytearray(b'a'), memoryview(b'a'):
+ self.assertEqual(re.match(b'a', string).groups(), ())
+ self.assertEqual(re.match(b'(a)', string).groups(), (b'a',))
+ self.assertEqual(re.match(b'(a)', string).group(0), b'a')
+ self.assertEqual(re.match(b'(a)', string).group(1), b'a')
+ self.assertEqual(re.match(b'(a)', string).group(1, 1), (b'a', b'a'))
+ for a in ("\xe0", "\u0430", "\U0001d49c"):
+ self.assertEqual(re.match(a, a).groups(), ())
+ self.assertEqual(re.match('(%s)' % a, a).groups(), (a,))
+ self.assertEqual(re.match('(%s)' % a, a).group(0), a)
+ self.assertEqual(re.match('(%s)' % a, a).group(1), a)
+ self.assertEqual(re.match('(%s)' % a, a).group(1, 1), (a, a))
pat = re.compile('((a)|(b))(c)?')
self.assertEqual(pat.match('a').groups(), ('a', 'a', None, None))
@@ -272,6 +349,36 @@ class ReTests(unittest.TestCase):
(None, 'b', None))
self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c'))
+ def test_re_fullmatch(self):
+ # Issue 16203: Proposal: add re.fullmatch() method.
+ self.assertEqual(re.fullmatch(r"a", "a").span(), (0, 1))
+ for string in "ab", S("ab"):
+ self.assertEqual(re.fullmatch(r"a|ab", string).span(), (0, 2))
+ for string in b"ab", B(b"ab"), bytearray(b"ab"), memoryview(b"ab"):
+ self.assertEqual(re.fullmatch(br"a|ab", string).span(), (0, 2))
+ for a, b in "\xe0\xdf", "\u0430\u0431", "\U0001d49c\U0001d49e":
+ r = r"%s|%s" % (a, a + b)
+ self.assertEqual(re.fullmatch(r, a + b).span(), (0, 2))
+ self.assertEqual(re.fullmatch(r".*?$", "abc").span(), (0, 3))
+ self.assertEqual(re.fullmatch(r".*?", "abc").span(), (0, 3))
+ self.assertEqual(re.fullmatch(r"a.*?b", "ab").span(), (0, 2))
+ self.assertEqual(re.fullmatch(r"a.*?b", "abb").span(), (0, 3))
+ self.assertEqual(re.fullmatch(r"a.*?b", "axxb").span(), (0, 4))
+ self.assertIsNone(re.fullmatch(r"a+", "ab"))
+ self.assertIsNone(re.fullmatch(r"abc$", "abc\n"))
+ self.assertIsNone(re.fullmatch(r"abc\Z", "abc\n"))
+ self.assertIsNone(re.fullmatch(r"(?m)abc$", "abc\n"))
+ self.assertEqual(re.fullmatch(r"ab(?=c)cd", "abcd").span(), (0, 4))
+ self.assertEqual(re.fullmatch(r"ab(?<=b)cd", "abcd").span(), (0, 4))
+ self.assertEqual(re.fullmatch(r"(?=a|ab)ab", "ab").span(), (0, 2))
+
+ self.assertEqual(
+ re.compile(r"bc").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3))
+ self.assertEqual(
+ re.compile(r".*?$").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3))
+ self.assertEqual(
+ re.compile(r".*?").fullmatch("abcd", pos=1, endpos=3).span(), (1, 3))
+
def test_re_groupref_exists(self):
self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups(),
('(', 'a'))
@@ -1053,6 +1160,28 @@ class ReTests(unittest.TestCase):
self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'),
[b'xyz'], msg=pattern)
+ def test_match_repr(self):
+ for string in '[abracadabra]', S('[abracadabra]'):
+ m = re.search(r'(.+)(.*?)\1', string)
+ self.assertEqual(repr(m), "<%s.%s object; "
+ "span=(1, 12), match='abracadabra'>" %
+ (type(m).__module__, type(m).__qualname__))
+ for string in (b'[abracadabra]', B(b'[abracadabra]'),
+ bytearray(b'[abracadabra]'),
+ memoryview(b'[abracadabra]')):
+ m = re.search(rb'(.+)(.*?)\1', string)
+ self.assertEqual(repr(m), "<%s.%s object; "
+ "span=(1, 12), match=b'abracadabra'>" %
+ (type(m).__module__, type(m).__qualname__))
+
+ first, second = list(re.finditer("(aa)|(bb)", "aa bb"))
+ self.assertEqual(repr(first), "<%s.%s object; "
+ "span=(0, 2), match='aa'>" %
+ (type(second).__module__, type(first).__qualname__))
+ self.assertEqual(repr(second), "<%s.%s object; "
+ "span=(3, 5), match='bb'>" %
+ (type(second).__module__, type(second).__qualname__))
+
def test_bug_2537(self):
# issue 2537: empty submatches
@@ -1082,6 +1211,8 @@ class ReTests(unittest.TestCase):
self.assertEqual(
pat.match(string='abracadabra', pos=7, endpos=10).span(), (7, 9))
self.assertEqual(
+ pat.fullmatch(string='abracadabra', pos=7, endpos=9).span(), (7, 9))
+ self.assertEqual(
pat.search(string='abracadabra', pos=3, endpos=10).span(), (7, 9))
self.assertEqual(
pat.findall(string='abracadabra', pos=3, endpos=10), ['ab'])
@@ -1093,6 +1224,83 @@ class ReTests(unittest.TestCase):
(7, 9))
+class PatternReprTests(unittest.TestCase):
+ def check(self, pattern, expected):
+ self.assertEqual(repr(re.compile(pattern)), expected)
+
+ def check_flags(self, pattern, flags, expected):
+ self.assertEqual(repr(re.compile(pattern, flags)), expected)
+
+ def test_without_flags(self):
+ self.check('random pattern',
+ "re.compile('random pattern')")
+
+ def test_single_flag(self):
+ self.check_flags('random pattern', re.IGNORECASE,
+ "re.compile('random pattern', re.IGNORECASE)")
+
+ def test_multiple_flags(self):
+ self.check_flags('random pattern', re.I|re.S|re.X,
+ "re.compile('random pattern', "
+ "re.IGNORECASE|re.DOTALL|re.VERBOSE)")
+
+ def test_unicode_flag(self):
+ self.check_flags('random pattern', re.U,
+ "re.compile('random pattern')")
+ self.check_flags('random pattern', re.I|re.S|re.U,
+ "re.compile('random pattern', "
+ "re.IGNORECASE|re.DOTALL)")
+
+ def test_inline_flags(self):
+ self.check('(?i)pattern',
+ "re.compile('(?i)pattern', re.IGNORECASE)")
+
+ def test_unknown_flags(self):
+ self.check_flags('random pattern', 0x123000,
+ "re.compile('random pattern', 0x123000)")
+ self.check_flags('random pattern', 0x123000|re.I,
+ "re.compile('random pattern', re.IGNORECASE|0x123000)")
+
+ def test_bytes(self):
+ self.check(b'bytes pattern',
+ "re.compile(b'bytes pattern')")
+ self.check_flags(b'bytes pattern', re.A,
+ "re.compile(b'bytes pattern', re.ASCII)")
+
+ def test_quotes(self):
+ self.check('random "double quoted" pattern',
+ '''re.compile('random "double quoted" pattern')''')
+ self.check("random 'single quoted' pattern",
+ '''re.compile("random 'single quoted' pattern")''')
+ self.check('''both 'single' and "double" quotes''',
+ '''re.compile('both \\'single\\' and "double" quotes')''')
+
+ def test_long_pattern(self):
+ pattern = 'Very %spattern' % ('long ' * 1000)
+ r = repr(re.compile(pattern))
+ self.assertLess(len(r), 300)
+ self.assertEqual(r[:30], "re.compile('Very long long lon")
+ r = repr(re.compile(pattern, re.I))
+ self.assertLess(len(r), 300)
+ self.assertEqual(r[:30], "re.compile('Very long long lon")
+ self.assertEqual(r[-16:], ", re.IGNORECASE)")
+
+
+class ImplementationTest(unittest.TestCase):
+ """
+ Test implementation details of the re module.
+ """
+
+ def test_overlap_table(self):
+ f = sre_compile._generate_overlap_table
+ self.assertEqual(f(""), [])
+ self.assertEqual(f("a"), [0])
+ self.assertEqual(f("abcd"), [0, 0, 0, 0])
+ self.assertEqual(f("aaaa"), [0, 1, 2, 3])
+ self.assertEqual(f("ababba"), [0, 0, 1, 2, 0, 1])
+ self.assertEqual(f("abcabdac"), [0, 0, 0, 1, 2, 0, 1, 0])
+
+
def run_re_tests():
from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR
if verbose:
@@ -1222,7 +1430,7 @@ def run_re_tests():
def test_main():
- run_unittest(ReTests)
+ run_unittest(__name__)
run_re_tests()
if __name__ == "__main__":
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
new file mode 100644
index 0000000..a398a4f
--- /dev/null
+++ b/Lib/test/test_regrtest.py
@@ -0,0 +1,275 @@
+"""
+Tests of regrtest.py.
+"""
+
+import argparse
+import faulthandler
+import getopt
+import os.path
+import unittest
+from test import regrtest, support
+
+class ParseArgsTestCase(unittest.TestCase):
+
+ """Test regrtest's argument parsing."""
+
+ def checkError(self, args, msg):
+ with support.captured_stderr() as err, self.assertRaises(SystemExit):
+ regrtest._parse_args(args)
+ self.assertIn(msg, err.getvalue())
+
+ def test_help(self):
+ for opt in '-h', '--help':
+ with self.subTest(opt=opt):
+ with support.captured_stdout() as out, \
+ self.assertRaises(SystemExit):
+ regrtest._parse_args([opt])
+ self.assertIn('Run Python regression tests.', out.getvalue())
+
+ @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'),
+ "faulthandler.dump_traceback_later() required")
+ def test_timeout(self):
+ ns = regrtest._parse_args(['--timeout', '4.2'])
+ self.assertEqual(ns.timeout, 4.2)
+ self.checkError(['--timeout'], 'expected one argument')
+ self.checkError(['--timeout', 'foo'], 'invalid float value')
+
+ def test_wait(self):
+ ns = regrtest._parse_args(['--wait'])
+ self.assertTrue(ns.wait)
+
+ def test_slaveargs(self):
+ ns = regrtest._parse_args(['--slaveargs', '[[], {}]'])
+ self.assertEqual(ns.slaveargs, '[[], {}]')
+ self.checkError(['--slaveargs'], 'expected one argument')
+
+ def test_start(self):
+ for opt in '-S', '--start':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, 'foo'])
+ self.assertEqual(ns.start, 'foo')
+ self.checkError([opt], 'expected one argument')
+
+ def test_verbose(self):
+ ns = regrtest._parse_args(['-v'])
+ self.assertEqual(ns.verbose, 1)
+ ns = regrtest._parse_args(['-vvv'])
+ self.assertEqual(ns.verbose, 3)
+ ns = regrtest._parse_args(['--verbose'])
+ self.assertEqual(ns.verbose, 1)
+ ns = regrtest._parse_args(['--verbose'] * 3)
+ self.assertEqual(ns.verbose, 3)
+ ns = regrtest._parse_args([])
+ self.assertEqual(ns.verbose, 0)
+
+ def test_verbose2(self):
+ for opt in '-w', '--verbose2':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.verbose2)
+
+ def test_verbose3(self):
+ for opt in '-W', '--verbose3':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.verbose3)
+
+ def test_quiet(self):
+ for opt in '-q', '--quiet':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.quiet)
+ self.assertEqual(ns.verbose, 0)
+
+ def test_slow(self):
+ for opt in '-o', '--slow':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.print_slow)
+
+ def test_header(self):
+ ns = regrtest._parse_args(['--header'])
+ self.assertTrue(ns.header)
+
+ def test_randomize(self):
+ for opt in '-r', '--randomize':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.randomize)
+
+ def test_randseed(self):
+ ns = regrtest._parse_args(['--randseed', '12345'])
+ self.assertEqual(ns.random_seed, 12345)
+ self.assertTrue(ns.randomize)
+ self.checkError(['--randseed'], 'expected one argument')
+ self.checkError(['--randseed', 'foo'], 'invalid int value')
+
+ def test_fromfile(self):
+ for opt in '-f', '--fromfile':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, 'foo'])
+ self.assertEqual(ns.fromfile, 'foo')
+ self.checkError([opt], 'expected one argument')
+ self.checkError([opt, 'foo', '-s'], "don't go together")
+
+ def test_exclude(self):
+ for opt in '-x', '--exclude':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.exclude)
+
+ def test_single(self):
+ for opt in '-s', '--single':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.single)
+ self.checkError([opt, '-f', 'foo'], "don't go together")
+
+ def test_match(self):
+ for opt in '-m', '--match':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, 'pattern'])
+ self.assertEqual(ns.match_tests, 'pattern')
+ self.checkError([opt], 'expected one argument')
+
+ def test_failfast(self):
+ for opt in '-G', '--failfast':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, '-v'])
+ self.assertTrue(ns.failfast)
+ ns = regrtest._parse_args([opt, '-W'])
+ self.assertTrue(ns.failfast)
+ self.checkError([opt], '-G/--failfast needs either -v or -W')
+
+ def test_use(self):
+ for opt in '-u', '--use':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, 'gui,network'])
+ self.assertEqual(ns.use_resources, ['gui', 'network'])
+ ns = regrtest._parse_args([opt, 'gui,none,network'])
+ self.assertEqual(ns.use_resources, ['network'])
+ expected = list(regrtest.RESOURCE_NAMES)
+ expected.remove('gui')
+ ns = regrtest._parse_args([opt, 'all,-gui'])
+ self.assertEqual(ns.use_resources, expected)
+ self.checkError([opt], 'expected one argument')
+ self.checkError([opt, 'foo'], 'invalid resource')
+
+ def test_memlimit(self):
+ for opt in '-M', '--memlimit':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, '4G'])
+ self.assertEqual(ns.memlimit, '4G')
+ self.checkError([opt], 'expected one argument')
+
+ def test_testdir(self):
+ ns = regrtest._parse_args(['--testdir', 'foo'])
+ self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo'))
+ self.checkError(['--testdir'], 'expected one argument')
+
+ def test_runleaks(self):
+ for opt in '-L', '--runleaks':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.runleaks)
+
+ def test_huntrleaks(self):
+ for opt in '-R', '--huntrleaks':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, ':'])
+ self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt'))
+ ns = regrtest._parse_args([opt, '6:'])
+ self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt'))
+ ns = regrtest._parse_args([opt, ':3'])
+ self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt'))
+ ns = regrtest._parse_args([opt, '6:3:leaks.log'])
+ self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log'))
+ self.checkError([opt], 'expected one argument')
+ self.checkError([opt, '6'],
+ 'needs 2 or 3 colon-separated arguments')
+ self.checkError([opt, 'foo:'], 'invalid huntrleaks value')
+ self.checkError([opt, '6:foo'], 'invalid huntrleaks value')
+
+ def test_multiprocess(self):
+ for opt in '-j', '--multiprocess':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, '2'])
+ self.assertEqual(ns.use_mp, 2)
+ self.checkError([opt], 'expected one argument')
+ self.checkError([opt, 'foo'], 'invalid int value')
+ self.checkError([opt, '2', '-T'], "don't go together")
+ self.checkError([opt, '2', '-l'], "don't go together")
+ self.checkError([opt, '2', '-M', '4G'], "don't go together")
+
+ def test_coverage(self):
+ for opt in '-T', '--coverage':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.trace)
+
+ def test_coverdir(self):
+ for opt in '-D', '--coverdir':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, 'foo'])
+ self.assertEqual(ns.coverdir,
+ os.path.join(support.SAVEDCWD, 'foo'))
+ self.checkError([opt], 'expected one argument')
+
+ def test_nocoverdir(self):
+ for opt in '-N', '--nocoverdir':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertIsNone(ns.coverdir)
+
+ def test_threshold(self):
+ for opt in '-t', '--threshold':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt, '1000'])
+ self.assertEqual(ns.threshold, 1000)
+ self.checkError([opt], 'expected one argument')
+ self.checkError([opt, 'foo'], 'invalid int value')
+
+ def test_nowindows(self):
+ for opt in '-n', '--nowindows':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.nowindows)
+
+ def test_forever(self):
+ for opt in '-F', '--forever':
+ with self.subTest(opt=opt):
+ ns = regrtest._parse_args([opt])
+ self.assertTrue(ns.forever)
+
+
+ def test_unrecognized_argument(self):
+ self.checkError(['--xxx'], 'usage:')
+
+ def test_long_option__partial(self):
+ ns = regrtest._parse_args(['--qui'])
+ self.assertTrue(ns.quiet)
+ self.assertEqual(ns.verbose, 0)
+
+ def test_two_options(self):
+ ns = regrtest._parse_args(['--quiet', '--exclude'])
+ self.assertTrue(ns.quiet)
+ self.assertEqual(ns.verbose, 0)
+ self.assertTrue(ns.exclude)
+
+ def test_option_with_empty_string_value(self):
+ ns = regrtest._parse_args(['--start', ''])
+ self.assertEqual(ns.start, '')
+
+ def test_arg(self):
+ ns = regrtest._parse_args(['foo'])
+ self.assertEqual(ns.args, ['foo'])
+
+ def test_option_and_arg(self):
+ ns = regrtest._parse_args(['--quiet', 'foo'])
+ self.assertTrue(ns.quiet)
+ self.assertEqual(ns.verbose, 0)
+ self.assertEqual(ns.args, ['foo'])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py
index c233b1d..ae67f06 100644
--- a/Lib/test/test_reprlib.py
+++ b/Lib/test/test_reprlib.py
@@ -3,11 +3,11 @@
Nick Mathewson
"""
-import imp
import sys
import os
import shutil
import importlib
+import importlib.util
import unittest
from test.support import run_unittest, create_empty_file, verbose
@@ -248,7 +248,8 @@ class LongReprTest(unittest.TestCase):
source_path_len += 2 * (len(self.longname) + 1)
# a path separator + `module_name` + ".py"
source_path_len += len(module_name) + 1 + len(".py")
- cached_path_len = source_path_len + len(imp.cache_from_source("x.py")) - len("x.py")
+ cached_path_len = (source_path_len +
+ len(importlib.util.cache_from_source("x.py")) - len("x.py"))
if os.name == 'nt' and cached_path_len >= 258:
# Under Windows, the max path len is 260 including C's terminating
# NUL character.
@@ -259,6 +260,7 @@ class LongReprTest(unittest.TestCase):
print("cached_path_len =", cached_path_len)
def test_module(self):
+ self.maxDiff = None
self._check_path_limitations(self.pkgname)
create_empty_file(os.path.join(self.subpkgname, self.pkgname + '.py'))
importlib.invalidate_caches()
diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py
index 0cf61cb..2ecae0f 100644
--- a/Lib/test/test_resource.py
+++ b/Lib/test/test_resource.py
@@ -1,3 +1,6 @@
+import contextlib
+import sys
+import os
import unittest
from test import support
import time
@@ -61,7 +64,7 @@ class ResourceTest(unittest.TestCase):
for i in range(5):
time.sleep(.1)
f.flush()
- except IOError:
+ except OSError:
if not limit_set:
raise
if limit_set:
@@ -129,6 +132,33 @@ class ResourceTest(unittest.TestCase):
self.assertIsInstance(pagesize, int)
self.assertGreaterEqual(pagesize, 0)
+ @unittest.skipUnless(sys.platform == 'linux', 'test requires Linux')
+ def test_linux_constants(self):
+ for attr in ['MSGQUEUE', 'NICE', 'RTPRIO', 'RTTIME', 'SIGPENDING']:
+ with contextlib.suppress(AttributeError):
+ self.assertIsInstance(getattr(resource, 'RLIMIT_' + attr), int)
+
+ @support.requires_freebsd_version(9)
+ def test_freebsd_contants(self):
+ for attr in ['SWAP', 'SBSIZE', 'NPTS']:
+ with contextlib.suppress(AttributeError):
+ self.assertIsInstance(getattr(resource, 'RLIMIT_' + attr), int)
+
+ @unittest.skipUnless(hasattr(resource, 'prlimit'), 'no prlimit')
+ @support.requires_linux_version(2, 6, 36)
+ def test_prlimit(self):
+ self.assertRaises(TypeError, resource.prlimit)
+ if os.geteuid() != 0:
+ self.assertRaises(PermissionError, resource.prlimit,
+ 1, resource.RLIMIT_AS)
+ self.assertRaises(ProcessLookupError, resource.prlimit,
+ -1, resource.RLIMIT_AS)
+ limit = resource.getrlimit(resource.RLIMIT_AS)
+ self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS), limit)
+ self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, limit),
+ limit)
+
+
def test_main(verbose=None):
support.run_unittest(ResourceTest)
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index 2ddba34..786b813 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,7 +5,7 @@ import os.path
import sys
import re
import tempfile
-import importlib
+import importlib, importlib.machinery, importlib.util
import py_compile
from test.support import (
forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -47,6 +47,7 @@ implicit_namespace = {
"__cached__": None,
"__package__": None,
"__doc__": None,
+ "__spec__": None
}
example_namespace = {
"sys": sys,
@@ -67,11 +68,19 @@ class CodeExecutionMixin:
# testing occurs at those upper layers as well, not just at the utility
# layer
+ # Figuring out the loader details in advance is hard to do, so we skip
+ # checking the full details of loader and loader_state
+ CHECKED_SPEC_ATTRIBUTES = ["name", "parent", "origin", "cached",
+ "has_location", "submodule_search_locations"]
+
def assertNamespaceMatches(self, result_ns, expected_ns):
"""Check two namespaces match.
Ignores any unspecified interpreter created names
"""
+ # Avoid side effects
+ result_ns = result_ns.copy()
+ expected_ns = expected_ns.copy()
# Impls are permitted to add extra names, so filter them out
for k in list(result_ns):
if k.startswith("__") and k.endswith("__"):
@@ -79,7 +88,25 @@ class CodeExecutionMixin:
result_ns.pop(k)
if k not in expected_ns["nested"]:
result_ns["nested"].pop(k)
- # Don't use direct dict comparison - the diffs are too hard to debug
+ # Spec equality includes the loader, so we take the spec out of the
+ # result namespace and check that separately
+ result_spec = result_ns.pop("__spec__")
+ expected_spec = expected_ns.pop("__spec__")
+ if expected_spec is None:
+ self.assertIsNone(result_spec)
+ else:
+ # If an expected loader is set, we just check we got the right
+ # type, rather than checking for full equality
+ if expected_spec.loader is not None:
+ self.assertEqual(type(result_spec.loader),
+ type(expected_spec.loader))
+ for attr in self.CHECKED_SPEC_ATTRIBUTES:
+ k = "__spec__." + attr
+ actual = (k, getattr(result_spec, attr))
+ expected = (k, getattr(expected_spec, attr))
+ self.assertEqual(actual, expected)
+ # For the rest, we still don't use direct dict comparison on the
+ # namespace, as the diffs are too hard to debug if anything breaks
self.assertEqual(set(result_ns), set(expected_ns))
for k in result_ns:
actual = (k, result_ns[k])
@@ -130,12 +157,16 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
mod_fname = "Some other nonsense"
mod_loader = "Now you're just being silly"
mod_package = '' # Treat as a top level module
+ mod_spec = importlib.machinery.ModuleSpec(mod_name,
+ origin=mod_fname,
+ loader=mod_loader)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
"__loader__": mod_loader,
"__package__": mod_package,
+ "__spec__": mod_spec,
"run_argv0": mod_fname,
"run_name_in_sys_modules": True,
"module_in_sys_modules": True,
@@ -144,9 +175,7 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin):
return _run_module_code(example_source,
init_globals,
mod_name,
- mod_fname,
- mod_loader,
- mod_package)
+ mod_spec)
self.check_code_execution(create_ns, expected_ns)
# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
@@ -176,31 +205,43 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
def test_library_module(self):
self.assertEqual(run_module("runpy")["__name__"], "runpy")
- def _add_pkg_dir(self, pkg_dir):
+ def _add_pkg_dir(self, pkg_dir, namespace=False):
os.mkdir(pkg_dir)
+ if namespace:
+ return None
pkg_fname = os.path.join(pkg_dir, "__init__.py")
create_empty_file(pkg_fname)
return pkg_fname
- def _make_pkg(self, source, depth, mod_base="runpy_test"):
+ def _make_pkg(self, source, depth, mod_base="runpy_test",
+ *, namespace=False, parent_namespaces=False):
+ # Enforce a couple of internal sanity checks on test cases
+ if (namespace or parent_namespaces) and not depth:
+ raise RuntimeError("Can't mark top level module as a "
+ "namespace package")
pkg_name = "__runpy_pkg__"
test_fname = mod_base+os.extsep+"py"
pkg_dir = sub_dir = os.path.realpath(tempfile.mkdtemp())
if verbose > 1: print(" Package tree in:", sub_dir)
sys.path.insert(0, pkg_dir)
if verbose > 1: print(" Updated sys.path:", sys.path[0])
- for i in range(depth):
- sub_dir = os.path.join(sub_dir, pkg_name)
- pkg_fname = self._add_pkg_dir(sub_dir)
- if verbose > 1: print(" Next level in:", sub_dir)
- if verbose > 1: print(" Created:", pkg_fname)
+ if depth:
+ namespace_flags = [parent_namespaces] * depth
+ namespace_flags[-1] = namespace
+ for namespace_flag in namespace_flags:
+ sub_dir = os.path.join(sub_dir, pkg_name)
+ pkg_fname = self._add_pkg_dir(sub_dir, namespace_flag)
+ if verbose > 1: print(" Next level in:", sub_dir)
+ if verbose > 1: print(" Created:", pkg_fname)
mod_fname = os.path.join(sub_dir, test_fname)
mod_file = open(mod_fname, "w")
mod_file.write(source)
mod_file.close()
if verbose > 1: print(" Created:", mod_fname)
mod_name = (pkg_name+".")*depth + mod_base
- return pkg_dir, mod_fname, mod_name
+ mod_spec = importlib.util.spec_from_file_location(mod_name,
+ mod_fname)
+ return pkg_dir, mod_fname, mod_name, mod_spec
def _del_pkg(self, top, depth, mod_name):
for entry in list(sys.modules):
@@ -230,19 +271,29 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
def _fix_ns_for_legacy_pyc(self, ns, alter_sys):
char_to_add = "c" if __debug__ else "o"
ns["__file__"] += char_to_add
+ ns["__cached__"] = ns["__file__"]
+ spec = ns["__spec__"]
+ new_spec = importlib.util.spec_from_file_location(spec.name,
+ ns["__file__"])
+ ns["__spec__"] = new_spec
if alter_sys:
ns["run_argv0"] += char_to_add
- def _check_module(self, depth, alter_sys=False):
- pkg_dir, mod_fname, mod_name = (
- self._make_pkg(example_source, depth))
+ def _check_module(self, depth, alter_sys=False,
+ *, namespace=False, parent_namespaces=False):
+ pkg_dir, mod_fname, mod_name, mod_spec = (
+ self._make_pkg(example_source, depth,
+ namespace=namespace,
+ parent_namespaces=parent_namespaces))
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
+ "__cached__": mod_spec.cached,
"__package__": mod_name.rpartition(".")[0],
+ "__spec__": mod_spec,
})
if alter_sys:
expected_ns.update({
@@ -269,16 +320,21 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
self._del_pkg(pkg_dir, depth, mod_name)
if verbose > 1: print("Module executed successfully")
- def _check_package(self, depth, alter_sys=False):
- pkg_dir, mod_fname, mod_name = (
- self._make_pkg(example_source, depth, "__main__"))
+ def _check_package(self, depth, alter_sys=False,
+ *, namespace=False, parent_namespaces=False):
+ pkg_dir, mod_fname, mod_name, mod_spec = (
+ self._make_pkg(example_source, depth, "__main__",
+ namespace=namespace,
+ parent_namespaces=parent_namespaces))
pkg_name = mod_name.rpartition(".")[0]
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
+ "__cached__": importlib.util.cache_from_source(mod_fname),
"__package__": pkg_name,
+ "__spec__": mod_spec,
})
if alter_sys:
expected_ns.update({
@@ -334,7 +390,7 @@ from __future__ import absolute_import
from . import sibling
from ..uncle.cousin import nephew
"""
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg(contents, depth))
if run_name is None:
expected_name = mod_name
@@ -373,11 +429,31 @@ from ..uncle.cousin import nephew
if verbose > 1: print("Testing package depth:", depth)
self._check_module(depth)
+ def test_run_module_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_module(depth, namespace=True, parent_namespaces=True)
+
def test_run_package(self):
for depth in range(1, 4):
if verbose > 1: print("Testing package depth:", depth)
self._check_package(depth)
+ def test_run_package_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, parent_namespaces=True)
+
+ def test_run_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, namespace=True)
+
+ def test_run_namespace_package_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, namespace=True, parent_namespaces=True)
+
def test_run_module_alter_sys(self):
for depth in range(4):
if verbose > 1: print("Testing package depth:", depth)
@@ -401,14 +477,16 @@ from ..uncle.cousin import nephew
def test_run_name(self):
depth = 1
run_name = "And now for something completely different"
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg(example_source, depth))
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": run_name,
"__file__": mod_fname,
+ "__cached__": importlib.util.cache_from_source(mod_fname),
"__package__": mod_name.rpartition(".")[0],
+ "__spec__": mod_spec,
})
def create_ns(init_globals):
return run_module(mod_name, init_globals, run_name)
@@ -437,7 +515,7 @@ from ..uncle.cousin import nephew
pkg_name = ".".join([base_name] * max_depth)
expected_packages.add(pkg_name)
expected_modules.add(pkg_name + ".runpy_test")
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg("", max_depth))
self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
for depth in range(2, max_depth+1):
@@ -455,21 +533,39 @@ from ..uncle.cousin import nephew
class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_path"""
- def _make_test_script(self, script_dir, script_basename, source=None):
+ def _make_test_script(self, script_dir, script_basename,
+ source=None, omit_suffix=False):
if source is None:
source = example_source
- return make_script(script_dir, script_basename, source)
+ return make_script(script_dir, script_basename,
+ source, omit_suffix)
def _check_script(self, script_name, expected_name, expected_file,
- expected_argv0):
+ expected_argv0, mod_name=None,
+ expect_spec=True, check_loader=True):
# First check is without run_name
def create_ns(init_globals):
return run_path(script_name, init_globals)
expected_ns = example_namespace.copy()
+ if mod_name is None:
+ spec_name = expected_name
+ else:
+ spec_name = mod_name
+ if expect_spec:
+ mod_spec = importlib.util.spec_from_file_location(spec_name,
+ expected_file)
+ mod_cached = mod_spec.cached
+ if not check_loader:
+ mod_spec.loader = None
+ else:
+ mod_spec = mod_cached = None
+
expected_ns.update({
"__name__": expected_name,
"__file__": expected_file,
+ "__cached__": mod_cached,
"__package__": "",
+ "__spec__": mod_spec,
"run_argv0": expected_argv0,
"run_name_in_sys_modules": True,
"module_in_sys_modules": True,
@@ -479,6 +575,12 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
run_name = "prove.issue15230.is.fixed"
def create_ns(init_globals):
return run_path(script_name, init_globals, run_name)
+ if expect_spec and mod_name is None:
+ mod_spec = importlib.util.spec_from_file_location(run_name,
+ expected_file)
+ if not check_loader:
+ mod_spec.loader = None
+ expected_ns["__spec__"] = mod_spec
expected_ns["__name__"] = run_name
expected_ns["__package__"] = run_name.rpartition(".")[0]
self.check_code_execution(create_ns, expected_ns)
@@ -492,7 +594,15 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
mod_name = 'script'
script_name = self._make_test_script(script_dir, mod_name)
self._check_script(script_name, "<run_path>", script_name,
- script_name)
+ script_name, expect_spec=False)
+
+ def test_basic_script_no_suffix(self):
+ with temp_dir() as script_dir:
+ mod_name = 'script'
+ script_name = self._make_test_script(script_dir, mod_name,
+ omit_suffix=True)
+ self._check_script(script_name, "<run_path>", script_name,
+ script_name, expect_spec=False)
def test_script_compiled(self):
with temp_dir() as script_dir:
@@ -501,14 +611,14 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
self._check_script(compiled_name, "<run_path>", compiled_name,
- compiled_name)
+ compiled_name, expect_spec=False)
def test_directory(self):
with temp_dir() as script_dir:
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
self._check_script(script_dir, "<run_path>", script_name,
- script_dir)
+ script_dir, mod_name=mod_name)
def test_directory_compiled(self):
with temp_dir() as script_dir:
@@ -519,7 +629,7 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
if not sys.dont_write_bytecode:
legacy_pyc = make_legacy_pyc(script_name)
self._check_script(script_dir, "<run_path>", legacy_pyc,
- script_dir)
+ script_dir, mod_name=mod_name)
def test_directory_error(self):
with temp_dir() as script_dir:
@@ -533,7 +643,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name)
- self._check_script(zip_name, "<run_path>", fname, zip_name)
+ self._check_script(zip_name, "<run_path>", fname, zip_name,
+ mod_name=mod_name, check_loader=False)
def test_zipfile_compiled(self):
with temp_dir() as script_dir:
@@ -542,7 +653,8 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
compiled_name = py_compile.compile(script_name, doraise=True)
zip_name, fname = make_zip_script(script_dir, 'test_zip',
compiled_name)
- self._check_script(zip_name, "<run_path>", fname, zip_name)
+ self._check_script(zip_name, "<run_path>", fname, zip_name,
+ mod_name=mod_name, check_loader=False)
def test_zipfile_error(self):
with temp_dir() as script_dir:
@@ -575,12 +687,5 @@ s = "non-ASCII: h\xe9"
self.assertEqual(result['s'], "non-ASCII: h\xe9")
-def test_main():
- run_unittest(
- ExecutionLayerTestCase,
- RunModuleTestCase,
- RunPathTestCase
- )
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py
index 79fa7d3..fe8e785 100644
--- a/Lib/test/test_sched.py
+++ b/Lib/test/test_sched.py
@@ -195,8 +195,5 @@ class TestCase(unittest.TestCase):
self.assertEqual(l, [])
-def test_main():
- support.run_unittest(TestCase)
-
if __name__ == "__main__":
- test_main()
+ unittest.main()
diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py
index 26ce042..b325545 100644
--- a/Lib/test/test_scope.py
+++ b/Lib/test/test_scope.py
@@ -715,6 +715,19 @@ class ScopeTests(unittest.TestCase):
def b():
global a
+ def testClassNamespaceOverridesClosure(self):
+ # See #17853.
+ x = 42
+ class X:
+ locals()["x"] = 43
+ y = x
+ self.assertEqual(X.y, 43)
+ class X:
+ locals()["x"] = 43
+ del x
+ self.assertFalse(hasattr(X, "x"))
+ self.assertEqual(x, 42)
+
@cpython_only
def testCellLeak(self):
# Issue 17927.
@@ -743,10 +756,6 @@ class ScopeTests(unittest.TestCase):
del tester
self.assertIsNone(ref())
- def test__Class__Global(self):
- s = "class X:\n global __class__\n def f(self): super()"
- self.assertRaises(SyntaxError, exec, s)
-
def test_main():
run_unittest(ScopeTests)
diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py
index ddb9a0f..8f9a1c9 100644
--- a/Lib/test/test_select.py
+++ b/Lib/test/test_select.py
@@ -5,7 +5,7 @@ import sys
import unittest
from test import support
-@unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'),
+@unittest.skipIf((sys.platform[:3]=='win'),
"can't easily test on this system")
class SelectTestCase(unittest.TestCase):
@@ -32,7 +32,7 @@ class SelectTestCase(unittest.TestCase):
fp.close()
try:
select.select([fd], [], [], 0)
- except select.error as err:
+ except OSError as err:
self.assertEqual(err.errno, errno.EBADF)
else:
self.fail("exception not raised")
diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py
new file mode 100644
index 0000000..34edd76
--- /dev/null
+++ b/Lib/test/test_selectors.py
@@ -0,0 +1,453 @@
+import errno
+import os
+import random
+import selectors
+import signal
+import socket
+from test import support
+from time import sleep
+import unittest
+import unittest.mock
+try:
+ from time import monotonic as time
+except ImportError:
+ from time import time as time
+try:
+ import resource
+except ImportError:
+ resource = None
+
+
+if hasattr(socket, 'socketpair'):
+ socketpair = socket.socketpair
+else:
+ def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
+ with socket.socket(family, type, proto) as l:
+ l.bind((support.HOST, 0))
+ l.listen(3)
+ c = socket.socket(family, type, proto)
+ try:
+ c.connect(l.getsockname())
+ caddr = c.getsockname()
+ while True:
+ a, addr = l.accept()
+ # check that we've got the correct client
+ if addr == caddr:
+ return c, a
+ a.close()
+ except OSError:
+ c.close()
+ raise
+
+
+def find_ready_matching(ready, flag):
+ match = []
+ for key, events in ready:
+ if events & flag:
+ match.append(key.fileobj)
+ return match
+
+
+class BaseSelectorTestCase(unittest.TestCase):
+
+ def make_socketpair(self):
+ rd, wr = socketpair()
+ self.addCleanup(rd.close)
+ self.addCleanup(wr.close)
+ return rd, wr
+
+ def test_register(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ key = s.register(rd, selectors.EVENT_READ, "data")
+ self.assertIsInstance(key, selectors.SelectorKey)
+ self.assertEqual(key.fileobj, rd)
+ self.assertEqual(key.fd, rd.fileno())
+ self.assertEqual(key.events, selectors.EVENT_READ)
+ self.assertEqual(key.data, "data")
+
+ # register an unknown event
+ self.assertRaises(ValueError, s.register, 0, 999999)
+
+ # register an invalid FD
+ self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ)
+
+ # register twice
+ self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ)
+
+ # register the same FD, but with a different object
+ self.assertRaises(KeyError, s.register, rd.fileno(),
+ selectors.EVENT_READ)
+
+ def test_unregister(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ s.register(rd, selectors.EVENT_READ)
+ s.unregister(rd)
+
+ # unregister an unknown file obj
+ self.assertRaises(KeyError, s.unregister, 999999)
+
+ # unregister twice
+ self.assertRaises(KeyError, s.unregister, rd)
+
+ def test_unregister_after_fd_close(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+ rd, wr = self.make_socketpair()
+ r, w = rd.fileno(), wr.fileno()
+ s.register(r, selectors.EVENT_READ)
+ s.register(w, selectors.EVENT_WRITE)
+ rd.close()
+ wr.close()
+ s.unregister(r)
+ s.unregister(w)
+
+ @unittest.skipUnless(os.name == 'posix', "requires posix")
+ def test_unregister_after_fd_close_and_reuse(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+ rd, wr = self.make_socketpair()
+ r, w = rd.fileno(), wr.fileno()
+ s.register(r, selectors.EVENT_READ)
+ s.register(w, selectors.EVENT_WRITE)
+ rd2, wr2 = self.make_socketpair()
+ rd.close()
+ wr.close()
+ os.dup2(rd2.fileno(), r)
+ os.dup2(wr2.fileno(), w)
+ self.addCleanup(os.close, r)
+ self.addCleanup(os.close, w)
+ s.unregister(r)
+ s.unregister(w)
+
+ def test_unregister_after_socket_close(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+ rd, wr = self.make_socketpair()
+ s.register(rd, selectors.EVENT_READ)
+ s.register(wr, selectors.EVENT_WRITE)
+ rd.close()
+ wr.close()
+ s.unregister(rd)
+ s.unregister(wr)
+
+ def test_modify(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ key = s.register(rd, selectors.EVENT_READ)
+
+ # modify events
+ key2 = s.modify(rd, selectors.EVENT_WRITE)
+ self.assertNotEqual(key.events, key2.events)
+ self.assertEqual(key2, s.get_key(rd))
+
+ s.unregister(rd)
+
+ # modify data
+ d1 = object()
+ d2 = object()
+
+ key = s.register(rd, selectors.EVENT_READ, d1)
+ key2 = s.modify(rd, selectors.EVENT_READ, d2)
+ self.assertEqual(key.events, key2.events)
+ self.assertNotEqual(key.data, key2.data)
+ self.assertEqual(key2, s.get_key(rd))
+ self.assertEqual(key2.data, d2)
+
+ # modify unknown file obj
+ self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ)
+
+ # modify use a shortcut
+ d3 = object()
+ s.register = unittest.mock.Mock()
+ s.unregister = unittest.mock.Mock()
+
+ s.modify(rd, selectors.EVENT_READ, d3)
+ self.assertFalse(s.register.called)
+ self.assertFalse(s.unregister.called)
+
+ def test_close(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ s.register(rd, selectors.EVENT_READ)
+ s.register(wr, selectors.EVENT_WRITE)
+
+ s.close()
+ self.assertRaises(KeyError, s.get_key, rd)
+ self.assertRaises(KeyError, s.get_key, wr)
+
+ def test_get_key(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ key = s.register(rd, selectors.EVENT_READ, "data")
+ self.assertEqual(key, s.get_key(rd))
+
+ # unknown file obj
+ self.assertRaises(KeyError, s.get_key, 999999)
+
+ def test_get_map(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ keys = s.get_map()
+ self.assertFalse(keys)
+ self.assertEqual(len(keys), 0)
+ self.assertEqual(list(keys), [])
+ key = s.register(rd, selectors.EVENT_READ, "data")
+ self.assertIn(rd, keys)
+ self.assertEqual(key, keys[rd])
+ self.assertEqual(len(keys), 1)
+ self.assertEqual(list(keys), [rd.fileno()])
+ self.assertEqual(list(keys.values()), [key])
+
+ # unknown file obj
+ with self.assertRaises(KeyError):
+ keys[999999]
+
+ # Read-only mapping
+ with self.assertRaises(TypeError):
+ del keys[rd]
+
+ def test_select(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ s.register(rd, selectors.EVENT_READ)
+ wr_key = s.register(wr, selectors.EVENT_WRITE)
+
+ result = s.select()
+ for key, events in result:
+ self.assertTrue(isinstance(key, selectors.SelectorKey))
+ self.assertTrue(events)
+ self.assertFalse(events & ~(selectors.EVENT_READ |
+ selectors.EVENT_WRITE))
+
+ self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result)
+
+ def test_context_manager(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ with s as sel:
+ sel.register(rd, selectors.EVENT_READ)
+ sel.register(wr, selectors.EVENT_WRITE)
+
+ self.assertRaises(KeyError, s.get_key, rd)
+ self.assertRaises(KeyError, s.get_key, wr)
+
+ def test_fileno(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ if hasattr(s, 'fileno'):
+ fd = s.fileno()
+ self.assertTrue(isinstance(fd, int))
+ self.assertGreaterEqual(fd, 0)
+
+ def test_selector(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ NUM_SOCKETS = 12
+ MSG = b" This is a test."
+ MSG_LEN = len(MSG)
+ readers = []
+ writers = []
+ r2w = {}
+ w2r = {}
+
+ for i in range(NUM_SOCKETS):
+ rd, wr = self.make_socketpair()
+ s.register(rd, selectors.EVENT_READ)
+ s.register(wr, selectors.EVENT_WRITE)
+ readers.append(rd)
+ writers.append(wr)
+ r2w[rd] = wr
+ w2r[wr] = rd
+
+ bufs = []
+
+ while writers:
+ ready = s.select()
+ ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE)
+ if not ready_writers:
+ self.fail("no sockets ready for writing")
+ wr = random.choice(ready_writers)
+ wr.send(MSG)
+
+ for i in range(10):
+ ready = s.select()
+ ready_readers = find_ready_matching(ready,
+ selectors.EVENT_READ)
+ if ready_readers:
+ break
+ # there might be a delay between the write to the write end and
+ # the read end is reported ready
+ sleep(0.1)
+ else:
+ self.fail("no sockets ready for reading")
+ self.assertEqual([w2r[wr]], ready_readers)
+ rd = ready_readers[0]
+ buf = rd.recv(MSG_LEN)
+ self.assertEqual(len(buf), MSG_LEN)
+ bufs.append(buf)
+ s.unregister(r2w[rd])
+ s.unregister(rd)
+ writers.remove(r2w[rd])
+
+ self.assertEqual(bufs, [MSG] * NUM_SOCKETS)
+
+ def test_timeout(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ s.register(wr, selectors.EVENT_WRITE)
+ t = time()
+ self.assertEqual(1, len(s.select(0)))
+ self.assertEqual(1, len(s.select(-1)))
+ self.assertLess(time() - t, 0.5)
+
+ s.unregister(wr)
+ s.register(rd, selectors.EVENT_READ)
+ t = time()
+ self.assertFalse(s.select(0))
+ self.assertFalse(s.select(-1))
+ self.assertLess(time() - t, 0.5)
+
+ t0 = time()
+ self.assertFalse(s.select(1))
+ t1 = time()
+ dt = t1 - t0
+ self.assertTrue(0.8 <= dt <= 1.6, dt)
+
+ @unittest.skipUnless(hasattr(signal, "alarm"),
+ "signal.alarm() required for this test")
+ def test_select_interrupt(self):
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ rd, wr = self.make_socketpair()
+
+ orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None)
+ self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler)
+ self.addCleanup(signal.alarm, 0)
+
+ signal.alarm(1)
+
+ s.register(rd, selectors.EVENT_READ)
+ t = time()
+ self.assertFalse(s.select(2))
+ self.assertLess(time() - t, 2.5)
+
+
+class ScalableSelectorMixIn:
+
+ # see issue #18963 for why it's skipped on older OS X versions
+ @support.requires_mac_ver(10, 5)
+ @unittest.skipUnless(resource, "Test needs resource module")
+ def test_above_fd_setsize(self):
+ # A scalable implementation should have no problem with more than
+ # FD_SETSIZE file descriptors. Since we don't know the value, we just
+ # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling.
+ soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+ try:
+ resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
+ self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE,
+ (soft, hard))
+ NUM_FDS = hard
+ except (OSError, ValueError):
+ NUM_FDS = soft
+
+ # guard for already allocated FDs (stdin, stdout...)
+ NUM_FDS -= 32
+
+ s = self.SELECTOR()
+ self.addCleanup(s.close)
+
+ for i in range(NUM_FDS // 2):
+ try:
+ rd, wr = self.make_socketpair()
+ except OSError:
+ # too many FDs, skip - note that we should only catch EMFILE
+ # here, but apparently *BSD and Solaris can fail upon connect()
+ # or bind() with EADDRNOTAVAIL, so let's be safe
+ self.skipTest("FD limit reached")
+
+ try:
+ s.register(rd, selectors.EVENT_READ)
+ s.register(wr, selectors.EVENT_WRITE)
+ except OSError as e:
+ if e.errno == errno.ENOSPC:
+ # this can be raised by epoll if we go over
+ # fs.epoll.max_user_watches sysctl
+ self.skipTest("FD limit reached")
+ raise
+
+ self.assertEqual(NUM_FDS // 2, len(s.select()))
+
+
+class DefaultSelectorTestCase(BaseSelectorTestCase):
+
+ SELECTOR = selectors.DefaultSelector
+
+
+class SelectSelectorTestCase(BaseSelectorTestCase):
+
+ SELECTOR = selectors.SelectSelector
+
+
+@unittest.skipUnless(hasattr(selectors, 'PollSelector'),
+ "Test needs selectors.PollSelector")
+class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn):
+
+ SELECTOR = getattr(selectors, 'PollSelector', None)
+
+
+@unittest.skipUnless(hasattr(selectors, 'EpollSelector'),
+ "Test needs selectors.EpollSelector")
+class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn):
+
+ SELECTOR = getattr(selectors, 'EpollSelector', None)
+
+
+@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'),
+ "Test needs selectors.KqueueSelector)")
+class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn):
+
+ SELECTOR = getattr(selectors, 'KqueueSelector', None)
+
+
+def test_main():
+ tests = [DefaultSelectorTestCase, SelectSelectorTestCase,
+ PollSelectorTestCase, EpollSelectorTestCase,
+ KqueueSelectorTestCase]
+ support.run_unittest(*tests)
+ support.reap_children()
+
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py
index c0bca2f..bfef621 100644
--- a/Lib/test/test_set.py
+++ b/Lib/test/test_set.py
@@ -848,8 +848,6 @@ class TestBasicOps:
for v in self.set:
self.assertIn(v, self.values)
setiter = iter(self.set)
- # note: __length_hint__ is an internal undocumented API,
- # don't rely on it in your own programs
self.assertEqual(setiter.__length_hint__(), len(self.set))
def test_pickling(self):
diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py
index 13c1265..bd51d86 100644
--- a/Lib/test/test_shelve.py
+++ b/Lib/test/test_shelve.py
@@ -148,6 +148,19 @@ class TestCase(unittest.TestCase):
p2 = d[encodedkey]
self.assertNotEqual(p1, p2) # Write creates new object in store
+ def test_with(self):
+ d1 = {}
+ with shelve.Shelf(d1, protocol=2, writeback=False) as s:
+ s['key1'] = [1,2,3,4]
+ self.assertEqual(s['key1'], [1,2,3,4])
+ self.assertEqual(len(s), 1)
+ self.assertRaises(ValueError, len, s)
+ try:
+ s['key1']
+ except ValueError:
+ pass
+ else:
+ self.fail('Closed shelf should not find a key')
from test import mapping_tests
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index c3fb8a7..a483fe1 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py