From a4e2d4fcf15e5973df1f0035a94b086b72b55443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Mon, 2 May 2011 22:59:15 +0200 Subject: Add missing name in pkgutil.__all__ --- Lib/pkgutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 0ec6ec5..e2d630a 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -8,7 +8,7 @@ from types import ModuleType __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', - 'walk_packages', 'iter_modules', + 'walk_packages', 'iter_modules', 'get_data', 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', ] -- cgit v0.12 From 39f180bb1f5407b89ed40c840fe0356566995da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 4 May 2011 15:55:47 +0200 Subject: Fix display of html.parser.HTMLParser.feed docstring --- Lib/html/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 3f68e18..bef5c3c 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -99,7 +99,7 @@ class HTMLParser(_markupbase.ParserBase): _markupbase.ParserBase.reset(self) def feed(self, data): - """Feed data to the parser. + r"""Feed data to the parser. Call this as often as you want, with as little or as much text as you want (may include '\n'). -- cgit v0.12 From 1e794f66f4503561f651547d08653d8ee3a87c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Thu, 5 May 2011 20:18:16 +0200 Subject: Remove obsolete 'rU' mode in Tools/scripts too (see be5b8d1ded34) --- Tools/scripts/findnocoding.py | 4 ++-- Tools/scripts/pysource.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index 77607ce..c42fa7c 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -2,7 +2,7 @@ """List all those Python files that require a coding directive -Usage: nocoding.py dir1 [dir2...] +Usage: findnocoding.py dir1 [dir2...] """ __author__ = "Oleg Broytmann, Georg Brandl" @@ -50,7 +50,7 @@ def has_correct_encoding(text, codec): def needs_declaration(fullpath): try: - infile = open(fullpath, 'rU') + infile = open(fullpath) except IOError: # Oops, the file was removed - ignore it return None diff --git a/Tools/scripts/pysource.py b/Tools/scripts/pysource.py index 048131e..c7dbe60 100644 --- a/Tools/scripts/pysource.py +++ b/Tools/scripts/pysource.py @@ -42,7 +42,7 @@ def _open(fullpath): return None try: - return open(fullpath, 'rU') + return open(fullpath) except IOError as err: # Access denied, or a special file - ignore it print_debug("%s: access denied: %s" % (fullpath, err)) return None -- cgit v0.12 From 17b60f0367db0ae8cd6d4ce6a6ea8330b4802029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Fri, 6 May 2011 18:50:08 +0200 Subject: Also add versionchanged directive to the function doc (#11591) --- Doc/library/site.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 23ce465..46bd5dc 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -129,6 +129,10 @@ empty, and the path manipulations are skipped; however the import of unless the :program:`python` interpreter was started with the :option:`-S` flag. + .. versionchanged:: 3.3 + This function used to be called unconditionnally. + + .. function:: addsitedir(sitedir, known_paths=None) Adds a directory to sys.path and processes its pth files. -- cgit v0.12 From 3c8d12d6b870bd907096da04d1bdb4d917639309 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 May 2011 22:12:27 +0200 Subject: Issue #10801: Remove a debug print() from test_zipfile --- Lib/test/test_zipfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 825fba1..a36b010 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -403,7 +403,6 @@ class TestsWithSourceFile(unittest.TestCase): mydir = os.path.dirname(myfile) or os.curdir fname = os.path.join(mydir, 'zip_cp437_header.zip') - print(fname) zipfp = zipfile.ZipFile(fname) try: zipfp.extractall() -- cgit v0.12 From 9b49304790a9ada10c4d0d13422ee350353cbeb0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 23 May 2011 12:29:10 +0200 Subject: Close #12153: faulthandler, mark stack_overflow() as static --- Modules/faulthandler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 72dbe1e..83c47ce 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -854,7 +854,7 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) } #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) -void* +static void* stack_overflow(void *min_sp, void *max_sp, size_t *depth) { /* allocate 4096 bytes on the stack at each call */ -- cgit v0.12 From cc6491a0faab5e57e6e004fd07a5b5ecf2762786 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 23 May 2011 12:54:52 +0200 Subject: Issue #12105: test_posix, add the value of O_CLOEXEC in the error message --- Lib/test/test_posix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9c2cac3..07e6b08 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -312,7 +312,8 @@ class PosixTester(unittest.TestCase): def test_oscloexec(self): fd = os.open(support.TESTFN, os.O_RDONLY|os.O_CLOEXEC) self.addCleanup(os.close, fd) - self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 'CLOEXEC flag not set (O_CLOEXEC=0x%x)' % os.O_CLOEXEC) def test_osexlock(self): if hasattr(posix, "O_EXLOCK"): -- cgit v0.12 From 2cca0572848817791537efc9ee5df66d4ceb2d42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 23 May 2011 14:51:42 +0200 Subject: test_codecs now removes the temporay file (created by the test) --- Lib/test/test_codecs.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 93cdf91..e412a64 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -515,13 +515,11 @@ class UTF16Test(ReadTest): s1 = 'Hello\r\nworld\r\n' s = s1.encode(self.encoding) - try: - with open(support.TESTFN, 'wb') as fp: - fp.write(s) - with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader: - self.assertEqual(reader.read(), s1) - finally: - support.unlink(support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, 'wb') as fp: + fp.write(s) + with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader: + self.assertEqual(reader.read(), s1) class UTF16LETest(ReadTest): encoding = "utf-16-le" @@ -1628,6 +1626,7 @@ class BomTest(unittest.TestCase): "utf-32", "utf-32-le", "utf-32-be") + self.addCleanup(support.unlink, support.TESTFN) for encoding in tests: # Check if the BOM is written only once with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: -- cgit v0.12 From a1ae533ac9e1535003daeaf8928018ab1e63c51a Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 23 May 2011 17:35:20 +0200 Subject: make sure build_dir.build_lib is synced to the faked build location --- Lib/packaging/tests/test_command_install_lib.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/packaging/tests/test_command_install_lib.py b/Lib/packaging/tests/test_command_install_lib.py index 99d47dd..96749e3 100644 --- a/Lib/packaging/tests/test_command_install_lib.py +++ b/Lib/packaging/tests/test_command_install_lib.py @@ -67,6 +67,10 @@ class InstallLibTestCase(support.TempdirManager, cmd.distribution.packages = [pkg_dir] cmd.distribution.script_name = 'setup.py' + # make sure the build_lib is set the temp dir + build_dir = os.path.split(pkg_dir)[0] + cmd.get_finalized_command('build_py').build_lib = build_dir + # get_output should return 4 elements self.assertEqual(len(cmd.get_outputs()), 4) -- cgit v0.12 From 7870bdff5a16c7440a6db5ef5f91082d27a4c482 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 23 May 2011 18:12:52 +0200 Subject: Issue #6501: os.device_encoding() returns None on Windows if the application has no console. --- Misc/NEWS | 3 +++ Modules/posixmodule.c | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 6ebb33a..9e2c14d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -153,6 +153,9 @@ Core and Builtins Library ------- +- Issue #6501: os.device_encoding() returns None on Windows if the application + has no console. + - Issue #12132: Skip test_build_ext in case the xxmodule is not found. - Issue #12105: Add O_CLOEXEC to the os module. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9c19ed0..add3b35 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8495,6 +8495,9 @@ static PyObject * device_encoding(PyObject *self, PyObject *args) { int fd; +#if defined(MS_WINDOWS) || defined(MS_WIN64) + UINT cp; +#endif if (!PyArg_ParseTuple(args, "i:device_encoding", &fd)) return NULL; if (!_PyVerify_fd(fd) || !isatty(fd)) { @@ -8502,16 +8505,16 @@ device_encoding(PyObject *self, PyObject *args) return Py_None; } #if defined(MS_WINDOWS) || defined(MS_WIN64) - if (fd == 0) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleCP()); - return PyUnicode_FromString(buf); - } - if (fd == 1 || fd == 2) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleOutputCP()); - return PyUnicode_FromString(buf); - } + if (fd == 0) + cp = GetConsoleCP(); + else if (fd == 1 || fd == 2) + cp = GetConsoleOutputCP(); + else + cp = 0; + /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application + has no console */ + if (cp != 0) + return PyUnicode_FromFormat("cp%u", (unsigned int)cp); #elif defined(CODESET) { char *codeset = nl_langinfo(CODESET); -- cgit v0.12 From eb64b61bf5c774fc0694287d5ad70061e1b94614 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 23 May 2011 18:31:20 +0200 Subject: make sure the cwd is restored on tearDown --- Lib/packaging/tests/test_config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/packaging/tests/test_config.py b/Lib/packaging/tests/test_config.py index 6f70817..a276730 100644 --- a/Lib/packaging/tests/test_config.py +++ b/Lib/packaging/tests/test_config.py @@ -176,9 +176,14 @@ class ConfigTestCase(support.TempdirManager, self.addCleanup(os.chdir, os.getcwd()) tempdir = self.mkdtemp() + self.working_dir = os.getcwd() os.chdir(tempdir) self.tempdir = tempdir + def tearDown(self): + os.chdir(self.working_dir) + super(ConfigTestCase, self).tearDown() + def write_setup(self, kwargs=None): opts = {'description-file': 'README', 'extra-files': '', 'setup-hook': 'packaging.tests.test_config.hook'} -- cgit v0.12 From fabc30833f9bab8d1d028c1295259d85be96a789 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 23 May 2011 18:47:27 +0200 Subject: fixed the cwd cleanup in packaging test_util --- Lib/packaging/tests/test_util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py index 29f5379..61b4ec7 100644 --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -526,11 +526,18 @@ class GlobTestCaseBase(support.TempdirManager, class GlobTestCase(GlobTestCaseBase): + def setUp(self): + super(GlobTestCase, self).setUp() + self.cwd = os.getcwd() + + def tearDown(self): + os.chdir(self.cwd) + super(GlobTestCase, self).tearDown() + def assertGlobMatch(self, glob, spec): """""" tempdir = self.build_files_tree(spec) expected = self.clean_tree(spec) - self.addCleanup(os.chdir, os.getcwd()) os.chdir(tempdir) result = list(iglob(glob)) self.assertCountEqual(expected, result) -- cgit v0.12 From e1177d9d17aa5ec9c1953e37fbb39f97b0a6f2ec Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 23 May 2011 19:07:56 +0200 Subject: Issue #12125: fixed the failures under Solaris due to improper test cleanup. --- Lib/packaging/tests/test_create.py | 2 +- Lib/packaging/tests/test_manifest.py | 8 ++++++++ Lib/packaging/tests/test_uninstall.py | 5 +++++ Misc/NEWS | 2 ++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/tests/test_create.py b/Lib/packaging/tests/test_create.py index 42cf34b..906ca8f 100644 --- a/Lib/packaging/tests/test_create.py +++ b/Lib/packaging/tests/test_create.py @@ -31,11 +31,11 @@ class CreateTestCase(support.TempdirManager, 'doc': sys.prefix + '/share/doc/pyxfoil', } def tearDown(self): - super(CreateTestCase, self).tearDown() sys.stdin = self._stdin sys.stdout = self._stdout os.chdir(self._cwd) sysconfig.get_paths = self._old_get_paths + super(CreateTestCase, self).tearDown() def test_ask_yn(self): sys.stdin.write('y\n') diff --git a/Lib/packaging/tests/test_manifest.py b/Lib/packaging/tests/test_manifest.py index 21a42c3..9fb8b63 100644 --- a/Lib/packaging/tests/test_manifest.py +++ b/Lib/packaging/tests/test_manifest.py @@ -26,6 +26,14 @@ class ManifestTestCase(support.TempdirManager, support.LoggingCatcher, unittest.TestCase): + def setUp(self): + super(ManifestTestCase, self).setUp() + self.cwd = os.getcwd() + + def tearDown(self): + os.chdir(self.cwd) + super(ManifestTestCase, self).tearDown() + def test_manifest_reader(self): tmpdir = self.mkdtemp() MANIFEST = os.path.join(tmpdir, 'MANIFEST.in') diff --git a/Lib/packaging/tests/test_uninstall.py b/Lib/packaging/tests/test_uninstall.py index 2d0e896..578b10d 100644 --- a/Lib/packaging/tests/test_uninstall.py +++ b/Lib/packaging/tests/test_uninstall.py @@ -36,8 +36,13 @@ class UninstallTestCase(support.TempdirManager, self.addCleanup(os.chdir, os.getcwd()) self.addCleanup(enable_cache) self.root_dir = self.mkdtemp() + self.cwd = os.getcwd() disable_cache() + def tearDown(self): + os.chdir(self.cwd) + super(UninstallTestCase, self).tearDown() + def run_setup(self, *args): # run setup with args args = ['run'] + list(args) diff --git a/Misc/NEWS b/Misc/NEWS index 9e2c14d..01b95e2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -153,6 +153,8 @@ Core and Builtins Library ------- +- Issue #12125: fixed the failures under Solaris due to improper test cleanup. + - Issue #6501: os.device_encoding() returns None on Windows if the application has no console. -- cgit v0.12 From 4b244ef2554bd6e35fb432ad9b55eae90f37b4db Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 23 May 2011 12:45:34 -0700 Subject: Clean-up example. --- Doc/library/re.rst | 72 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index e10c794..3046755 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1298,24 +1298,27 @@ The text categories are specified with regular expressions. The technique is to combine those into a single master regular expression and to loop over successive matches:: - Token = collections.namedtuple('Token', 'typ value line column') + import collections + import re + + Token = collections.namedtuple('Token', ['typ', 'value', 'line', 'column']) def tokenize(s): - keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} - tok_spec = [ - ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number - ('ASSIGN', r':='), # Assignment operator - ('END', ';'), # Statement terminator - ('ID', r'[A-Za-z]+'), # Identifiers - ('OP', r'[+*\/\-]'), # Arithmetic operators - ('NEWLINE', r'\n'), # Line endings - ('SKIP', r'[ \t]'), # Skip over spaces and tabs + keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} + token_specification = [ + ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number + ('ASSIGN', r':='), # Assignment operator + ('END', r';'), # Statement terminator + ('ID', r'[A-Za-z]+'), # Identifiers + ('OP', r'[+*\/\-]'), # Arithmetic operators + ('NEWLINE', r'\n'), # Line endings + ('SKIP', r'[ \t]'), # Skip over spaces and tabs ] - tok_re = '|'.join('(?P<%s>%s)' % pair for pair in tok_spec) - gettok = re.compile(tok_re).match + tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification) + get_token = re.compile(tok_regex).match line = 1 pos = line_start = 0 - mo = gettok(s) + mo = get_token(s) while mo is not None: typ = mo.lastgroup if typ == 'NEWLINE': @@ -1327,13 +1330,15 @@ successive matches:: typ = val yield Token(typ, val, line, mo.start()-line_start) pos = mo.end() - mo = gettok(s, pos) + mo = get_token(s, pos) if pos != len(s): raise RuntimeError('Unexpected character %r on line %d' %(s[pos], line)) - statements = '''\ - total := total + price * quantity; - tax := price * 0.05; + statements = ''' + IF quantity THEN + total := total + price * quantity; + tax := price * 0.05; + ENDIF; ''' for token in tokenize(statements): @@ -1341,17 +1346,22 @@ successive matches:: The tokenizer produces the following output:: - Token(typ='ID', value='total', line=1, column=8) - Token(typ='ASSIGN', value=':=', line=1, column=14) - Token(typ='ID', value='total', line=1, column=17) - Token(typ='OP', value='+', line=1, column=23) - Token(typ='ID', value='price', line=1, column=25) - Token(typ='OP', value='*', line=1, column=31) - Token(typ='ID', value='quantity', line=1, column=33) - Token(typ='END', value=';', line=1, column=41) - Token(typ='ID', value='tax', line=2, column=9) - Token(typ='ASSIGN', value=':=', line=2, column=13) - Token(typ='ID', value='price', line=2, column=16) - Token(typ='OP', value='*', line=2, column=22) - Token(typ='NUMBER', value='0.05', line=2, column=24) - Token(typ='END', value=';', line=2, column=28) + Token(typ='IF', value='IF', line=2, column=5) + Token(typ='ID', value='quantity', line=2, column=8) + Token(typ='THEN', value='THEN', line=2, column=17) + Token(typ='ID', value='total', line=3, column=9) + Token(typ='ASSIGN', value=':=', line=3, column=15) + Token(typ='ID', value='total', line=3, column=18) + Token(typ='OP', value='+', line=3, column=24) + Token(typ='ID', value='price', line=3, column=26) + Token(typ='OP', value='*', line=3, column=32) + Token(typ='ID', value='quantity', line=3, column=34) + Token(typ='END', value=';', line=3, column=42) + Token(typ='ID', value='tax', line=4, column=9) + Token(typ='ASSIGN', value=':=', line=4, column=13) + Token(typ='ID', value='price', line=4, column=16) + Token(typ='OP', value='*', line=4, column=22) + Token(typ='NUMBER', value='0.05', line=4, column=24) + Token(typ='END', value=';', line=4, column=28) + Token(typ='ENDIF', value='ENDIF', line=5, column=5) + Token(typ='END', value=';', line=5, column=10) -- cgit v0.12 From ba980db797892e771e9d66e9bd0468cdd2c73115 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 23 May 2011 21:37:54 +0100 Subject: Issue #12151: Added diagnostics to help diagnose intermittent socket errors. --- Lib/test/test_logging.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 3118065..6c66ff1 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -893,14 +893,29 @@ if threading: def handle(self): self.server._handler(self) + + def finish(self): + data = self.wfile.getvalue() + try: + super(DelegatingUDPRequestHandler, self).finish() + except socket.error: + msg = ('Error during finish, while sending %r, ' + 'closed = %s') + print(msg % (data, self._closed), file=sys.stderr) + raise + ThreadingUDPServer.__init__(self, addr, DelegatingUDPRequestHandler, bind_and_activate) ControlMixin.__init__(self, handler, poll_interval) + self._closed = False def server_bind(self): super(TestUDPServer, self).server_bind() self.port = self.socket.getsockname()[1] + def server_close(self): + super(TestUDPServer, self).server_close() + self._closed = True # - end of server_helper section -- cgit v0.12 From acd902be6466729ffc2f2cece9bf5e945d64619b Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 23 May 2011 21:49:09 +0100 Subject: Issue #12151: Correction to diagnostic code. --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 6c66ff1..8166b33 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -901,7 +901,7 @@ if threading: except socket.error: msg = ('Error during finish, while sending %r, ' 'closed = %s') - print(msg % (data, self._closed), file=sys.stderr) + print(msg % (data, self.server._closed), file=sys.stderr) raise ThreadingUDPServer.__init__(self, addr, DelegatingUDPRequestHandler, -- cgit v0.12 From 7963a35b417dfde5d97c52c05b73af738c827ca6 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 23 May 2011 16:11:05 -0500 Subject: correctly lookup __dir__ --- Lib/test/test_descr.py | 1 + Misc/NEWS | 3 +++ Objects/object.c | 11 ++++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index aafe428..c74e232 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1595,6 +1595,7 @@ order (MRO) for bases """ # probably not worth it. # ("__enter__", run_context, iden), # ("__exit__", run_context, iden), + ("__dir__", dir, empty_seq, set(), {}), ] class Checker(object): diff --git a/Misc/NEWS b/Misc/NEWS index 9ae7979..37bbbc1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.1.4? Core and Builtins ----------------- +- Correct lookup of __dir__ on objects. Among other things, this causes errors + besides AttributeError found on lookup to be propagated. + - Issue #12060: Use sig_atomic_t type and volatile keyword in the signal module. Patch written by Charles-François Natali. diff --git a/Objects/object.c b/Objects/object.c index d534273..ac57cd7 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1348,14 +1348,15 @@ error: static PyObject * _dir_object(PyObject *obj) { - PyObject * result = NULL; - PyObject * dirfunc = PyObject_GetAttrString((PyObject*)obj->ob_type, - "__dir__"); + PyObject *result = NULL; + static PyObject *dir_str = NULL; + PyObject *dirfunc = _PyObject_LookupSpecial(obj, "__dir__", &dir_str); assert(obj); if (dirfunc == NULL) { + if (PyErr_Occurred()) + return NULL; /* use default implementation */ - PyErr_Clear(); if (PyModule_Check(obj)) result = _specialized_dir_module(obj); else if (PyType_Check(obj)) @@ -1365,7 +1366,7 @@ _dir_object(PyObject *obj) } else { /* use __dir__ */ - result = PyObject_CallFunctionObjArgs(dirfunc, obj, NULL); + result = PyObject_CallFunctionObjArgs(dirfunc, NULL); Py_DECREF(dirfunc); if (result == NULL) return NULL; -- cgit v0.12 From a2a895c46c8e838a29308dd0da7ab1d364ff4495 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 23 May 2011 23:14:05 +0200 Subject: Issue #12074: regrtest displays also the current number of failures --- Lib/test/regrtest.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index e2e3765..a8726b5 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -580,9 +580,11 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if test is None: finished += 1 continue + accumulate_result(test, result) if not quiet: - print("[{1:{0}}{2}] {3}".format( - test_count_width, test_index, test_count, test)) + print("[{1:{0}}{2}/{3}] {4}".format( + test_count_width, test_index, test_count, + len(bad), test)) if stdout: print(stdout) if stderr: @@ -590,7 +592,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if result[0] == INTERRUPTED: assert result[1] == 'KeyboardInterrupt' raise KeyboardInterrupt # What else? - accumulate_result(test, result) test_index += 1 except KeyboardInterrupt: interrupted = True @@ -600,8 +601,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, else: for test_index, test in enumerate(tests, 1): if not quiet: - print("[{1:{0}}{2}] {3}".format( - test_count_width, test_index, test_count, test)) + print("[{1:{0}}{2}/{3}] {4}".format( + test_count_width, test_index, test_count, len(bad), test)) sys.stdout.flush() if trace: # If we're tracing code coverage, then we don't exit with status -- cgit v0.12 From 84f75c680c2a17954155e71455061599e8778ccb Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Tue, 24 May 2011 01:00:10 +0300 Subject: #12074: remove the /0 when there are no failures. --- Lib/test/regrtest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index a8726b5..08c9916 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -582,7 +582,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, continue accumulate_result(test, result) if not quiet: - print("[{1:{0}}{2}/{3}] {4}".format( + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( test_count_width, test_index, test_count, len(bad), test)) if stdout: @@ -601,7 +602,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, else: for test_index, test in enumerate(tests, 1): if not quiet: - print("[{1:{0}}{2}/{3}] {4}".format( + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( test_count_width, test_index, test_count, len(bad), test)) sys.stdout.flush() if trace: -- cgit v0.12 From 3ef1229b14ef901d19a0a9cc4a4ec70100638b36 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 23 May 2011 23:00:42 +0100 Subject: Issue #12151: Test now ignores datagram socket errors after server is closed. --- Lib/test/test_logging.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 8166b33..e4fc266 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -888,7 +888,8 @@ if threading: before calling :meth:`start`, so that the server will set up the socket and listen on it. """ - def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): class DelegatingUDPRequestHandler(DatagramRequestHandler): def handle(self): @@ -896,15 +897,15 @@ if threading: def finish(self): data = self.wfile.getvalue() - try: - super(DelegatingUDPRequestHandler, self).finish() - except socket.error: - msg = ('Error during finish, while sending %r, ' - 'closed = %s') - print(msg % (data, self.server._closed), file=sys.stderr) - raise - - ThreadingUDPServer.__init__(self, addr, DelegatingUDPRequestHandler, + if data: + try: + super(DelegatingUDPRequestHandler, self).finish() + except socket.error: + if not self.server._closed: + raise + + ThreadingUDPServer.__init__(self, addr, + DelegatingUDPRequestHandler, bind_and_activate) ControlMixin.__init__(self, handler, poll_interval) self._closed = False -- cgit v0.12 From 25000d4d31dfa5842776aa92f234ea6672489c67 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 00:16:16 +0200 Subject: Issue #11377: platform.popen() emits a DeprecationWarning --- Lib/platform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/platform.py b/Lib/platform.py index 1e4abe6..e2a74fe 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -361,6 +361,8 @@ def popen(cmd, mode='r', bufsize=-1): """ Portable popen() interface. """ + import warnings + warnings.warn('use os.popen instead', DeprecationWarning, stacklevel=2) return os.popen(cmd, mode, bufsize) def _norm_version(version, build=''): -- cgit v0.12 From fea0f4d51717fb4b89154e97cfc0084a4965a688 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 00:24:19 +0200 Subject: Issue #12158: Move linux_version() from test_socket to test.support --- Lib/test/support.py | 8 ++++++++ Lib/test/test_socket.py | 14 +++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/support.py b/Lib/test/support.py index 3f60d55..b03069c 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -291,6 +291,14 @@ def requires(resource, msg=None): msg = "Use of the `%s' resource not enabled" % resource raise ResourceDenied(msg) +def linux_version(): + try: + # platform.release() is something like '2.6.33.7-desktop-2mnb' + version_string = platform.release().split('-')[0] + return tuple(map(int, version_string.split('.'))) + except ValueError: + return 0, 0, 0 + HOST = 'localhost' def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index a449f5b..59e3019 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -24,14 +24,6 @@ try: except ImportError: fcntl = False -def linux_version(): - try: - # platform.release() is something like '2.6.33.7-desktop-2mnb' - version_string = platform.release().split('-')[0] - return tuple(map(int, version_string.split('.'))) - except ValueError: - return 0, 0, 0 - HOST = support.HOST MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return @@ -1032,7 +1024,7 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): if hasattr(socket, "SOCK_NONBLOCK"): def testInitNonBlocking(self): - v = linux_version() + v = support.linux_version() if v < (2, 6, 28): self.skipTest("Linux kernel 2.6.28 or higher required, not %s" % ".".join(map(str, v))) @@ -2010,7 +2002,7 @@ class ContextManagersTest(ThreadedTCPSocketTest): @unittest.skipUnless(fcntl, "module fcntl not available") class CloexecConstantTest(unittest.TestCase): def test_SOCK_CLOEXEC(self): - v = linux_version() + v = support.linux_version() if v < (2, 6, 28): self.skipTest("Linux kernel 2.6.28 or higher required, not %s" % ".".join(map(str, v))) @@ -2032,7 +2024,7 @@ class NonblockConstantTest(unittest.TestCase): self.assertEqual(s.gettimeout(), None) def test_SOCK_NONBLOCK(self): - v = linux_version() + v = support.linux_version() if v < (2, 6, 28): self.skipTest("Linux kernel 2.6.28 or higher required, not %s" % ".".join(map(str, v))) -- cgit v0.12 From e36f37525ba57546511376d79ddb0f3e983b8f56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 00:29:43 +0200 Subject: Issue #12105: test_posix skips test_oscloexec() on Linux < 2.6.23 --- Lib/test/test_posix.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 07e6b08..9d9802b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -310,10 +310,13 @@ class PosixTester(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'O_CLOEXEC'), "needs os.O_CLOEXEC") def test_oscloexec(self): + version = support.linux_version() + if sys.platform == 'linux2' and version < (2, 6, 23): + self.skipTest("Linux kernel 2.6.23 or higher required, " + "not %s.%s.%s" % version) fd = os.open(support.TESTFN, os.O_RDONLY|os.O_CLOEXEC) self.addCleanup(os.close, fd) - self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, - 'CLOEXEC flag not set (O_CLOEXEC=0x%x)' % os.O_CLOEXEC) + self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) def test_osexlock(self): if hasattr(posix, "O_EXLOCK"): -- cgit v0.12 From b7b1930fe374317c380e49add9fb4cc2267367e9 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 24 May 2011 07:12:41 +0100 Subject: Updated test_rollover delay to just over a second. --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e4fc266..b27ff6a 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3586,7 +3586,7 @@ class TimedRotatingFileHandlerTest(BaseFileTest): r = logging.makeLogRecord({'msg': 'testing'}) fh.emit(r) self.assertLogFile(self.fn) - time.sleep(1.0) + time.sleep(1.01) # just a little over a second ... fh.emit(r) fh.close() # At this point, we should have a recent rotated file which we -- cgit v0.12 From 99c8b1614319ce0161835ade223cdd395f0126d4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 12:05:19 +0200 Subject: Issue #12049: Add RAND_bytes() and RAND_pseudo_bytes() functions to the ssl module. --- Doc/library/ssl.rst | 20 +++++++++++++--- Doc/whatsnew/3.3.rst | 10 ++++++++ Lib/ssl.py | 2 +- Lib/test/test_ssl.py | 8 +++++++ Misc/NEWS | 3 +++ Modules/_ssl.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 4 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 5ece8cf..a528a03 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -162,6 +162,20 @@ instead. Random generation ^^^^^^^^^^^^^^^^^ +.. function:: RAND_bytes(num) + + Returns *num* cryptographically strong pseudo-random bytes. + + .. versionadded:: 3.3 + +.. function:: RAND_pseudo_bytes(num) + + Returns (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, + is_cryptographic is True if the bytes generated are cryptographically + strong. + + .. versionadded:: 3.3 + .. function:: RAND_status() Returns True if the SSL pseudo-random number generator has been seeded with @@ -171,7 +185,7 @@ Random generation .. function:: RAND_egd(path) - If you are running an entropy-gathering daemon (EGD) somewhere, and ``path`` + If you are running an entropy-gathering daemon (EGD) somewhere, and *path* is the pathname of a socket connection open to it, this will read 256 bytes of randomness from the socket, and add it to the SSL pseudo-random number generator to increase the security of generated secret keys. This is @@ -182,8 +196,8 @@ Random generation .. function:: RAND_add(bytes, entropy) - Mixes the given ``bytes`` into the SSL pseudo-random number generator. The - parameter ``entropy`` (a float) is a lower bound on the entropy contained in + Mixes the given *bytes* into the SSL pseudo-random number generator. The + parameter *entropy* (a float) is a lower bound on the entropy contained in string (so you can always use :const:`0.0`). See :rfc:`1750` for more information on sources of entropy. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index d442665..529665f 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -152,6 +152,16 @@ signal instead of a RuntimeError: OSError has an errno attribute. +ssl +--- + +The :mod:`ssl` module has new functions: + + * :func:`~ssl.RAND_bytes`: generate cryptographically strong + pseudo-random bytes. + * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes. + + Optimizations ============= diff --git a/Lib/ssl.py b/Lib/ssl.py index f938dc9..1a7f599 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -63,7 +63,7 @@ from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION from _ssl import _SSLContext, SSLError from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED from _ssl import OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1 -from _ssl import RAND_status, RAND_egd, RAND_add +from _ssl import RAND_status, RAND_egd, RAND_add, RAND_bytes, RAND_pseudo_bytes from _ssl import ( SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_READ, diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aef51e3..8c21975 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -102,6 +102,14 @@ class BasicSocketTests(unittest.TestCase): sys.stdout.write("\n RAND_status is %d (%s)\n" % (v, (v and "sufficient randomness") or "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + try: ssl.RAND_egd(1) except TypeError: diff --git a/Misc/NEWS b/Misc/NEWS index 22c3cf4..967e2ef 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -156,6 +156,9 @@ Core and Builtins Library ------- +- Issue #12049: Add RAND_bytes() and RAND_pseudo_bytes() functions to the ssl + module. + - Issue #12125: fixed the failures under Solaris due to improper test cleanup. - Issue #6501: os.device_encoding() returns None on Windows if the application diff --git a/Modules/_ssl.c b/Modules/_ssl.c index dc11fc8..3f631e3 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1887,6 +1887,69 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower\n\ bound on the entropy contained in string. See RFC 1750."); static PyObject * +PySSL_RAND(int len, int pseudo) +{ + int ok; + PyObject *bytes; + unsigned long err; + const char *errstr; + PyObject *v; + + bytes = PyBytes_FromStringAndSize(NULL, len); + if (bytes == NULL) + return NULL; + if (pseudo) { + ok = RAND_pseudo_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 0 || ok == 1) + return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); + } + else { + ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 1) + return bytes; + } + Py_DECREF(bytes); + + err = ERR_get_error(); + errstr = ERR_reason_error_string(err); + v = Py_BuildValue("(ks)", err, errstr); + if (v != NULL) { + PyErr_SetObject(PySSLErrorObject, v); + Py_DECREF(v); + } + return NULL; +} + +static PyObject * +PySSL_RAND_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_bytes", &len)) + return NULL; + return PySSL_RAND(len, 0); +} + +PyDoc_STRVAR(PySSL_RAND_bytes_doc, +"RAND_bytes(n) -> bytes\n\ +\n\ +Generate n cryptographically strong pseudo-random bytes."); + +static PyObject * +PySSL_RAND_pseudo_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_pseudo_bytes", &len)) + return NULL; + return PySSL_RAND(len, 1); +} + +PyDoc_STRVAR(PySSL_RAND_pseudo_bytes_doc, +"RAND_pseudo_bytes(n) -> (bytes, is_cryptographic)\n\ +\n\ +Generate n pseudo-random bytes. is_cryptographic is True if the bytes\ +generated are cryptographically strong."); + +static PyObject * PySSL_RAND_status(PyObject *self) { return PyLong_FromLong(RAND_status()); @@ -1939,6 +2002,10 @@ static PyMethodDef PySSL_methods[] = { #ifdef HAVE_OPENSSL_RAND {"RAND_add", PySSL_RAND_add, METH_VARARGS, PySSL_RAND_add_doc}, + {"RAND_bytes", PySSL_RAND_bytes, METH_VARARGS, + PySSL_RAND_bytes_doc}, + {"RAND_pseudo_bytes", PySSL_RAND_pseudo_bytes, METH_VARARGS, + PySSL_RAND_pseudo_bytes_doc}, {"RAND_egd", PySSL_RAND_egd, METH_VARARGS, PySSL_RAND_egd_doc}, {"RAND_status", (PyCFunction)PySSL_RAND_status, METH_NOARGS, -- cgit v0.12 From 79d1b1ce3fdfd6f62d77f6096e1bc61ee2170cdf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 13:59:21 +0200 Subject: Issue #12167: packaging.tests.support, LoggingCatcher restores correctly the log level. Avoid also calls to .addCleanup() in setUp(). --- Lib/packaging/tests/support.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py index 0c85b5d..6d60b9e 100644 --- a/Lib/packaging/tests/support.py +++ b/Lib/packaging/tests/support.py @@ -71,8 +71,8 @@ class LoggingCatcher: def setUp(self): super(LoggingCatcher, self).setUp() self.loghandler = handler = _TestHandler() + self.old_level = logger.level logger.addHandler(handler) - self.addCleanup(logger.setLevel, logger.level) logger.setLevel(logging.DEBUG) # we want all messages def tearDown(self): @@ -84,6 +84,7 @@ class LoggingCatcher: for ref in weakref.getweakrefs(handler): logging._removeHandlerRef(ref) del self.loghandler + logger.setLevel(self.old_level) super(LoggingCatcher, self).tearDown() def get_logs(self, *levels): -- cgit v0.12 From 9bcfacd4fcf64ee7f6e0d05b5d0894c10cd43fc0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 14:01:39 +0200 Subject: Issue #12167: packaging.tests.test_dist unloads the temporary module Fix a reference leak. --- Lib/packaging/tests/test_dist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/packaging/tests/test_dist.py b/Lib/packaging/tests/test_dist.py index 74c727b..fb6d524 100644 --- a/Lib/packaging/tests/test_dist.py +++ b/Lib/packaging/tests/test_dist.py @@ -13,6 +13,7 @@ from packaging.errors import PackagingModuleError, PackagingOptionError from packaging.tests import TESTFN, captured_stdout from packaging.tests import support, unittest from packaging.tests.support import create_distribution +from test.support import unload class test_dist(Command): @@ -224,6 +225,7 @@ class DistributionTestCase(support.TempdirManager, # prepare the call recorders sys.path.append(temp_home) self.addCleanup(sys.path.remove, temp_home) + self.addCleanup(unload, module_name) record = __import__(module_name).record old_run = cmd.run -- cgit v0.12 From 82b00c1d307f7fb39f388760b9ccd519fe3ef63b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 11:09:06 -0500 Subject: move specialized dir implementations into __dir__ methods (closes #12166) --- Lib/test/test_descrtut.py | 1 + Misc/NEWS | 3 + Objects/moduleobject.c | 30 ++++++- Objects/object.c | 195 +++++----------------------------------------- Objects/typeobject.c | 127 ++++++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 178 deletions(-) diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index c3355b9..f495e18 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -170,6 +170,7 @@ You can get the information from the list type: '__contains__', '__delattr__', '__delitem__', + '__dir__', '__doc__', '__eq__', '__format__', diff --git a/Misc/NEWS b/Misc/NEWS index 967e2ef..47b9c32 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12166: Move implementations of dir() specialized for various types into + the __dir__() methods of those types. + - Correct lookup of __dir__ on objects. Among other things, this causes errors besides AttributeError found on lookup to be propagated. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 06f58d8..d922249 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -413,6 +413,34 @@ module_clear(PyModuleObject *m) return 0; } +static PyObject * +module_dir(PyObject *self, PyObject *args) +{ + PyObject *result = NULL; + PyObject *dict = PyObject_GetAttrString(self, "__dict__"); + + if (dict != NULL) { + if (PyDict_Check(dict)) + result = PyDict_Keys(dict); + else { + const char *name = PyModule_GetName(self); + if (name) + PyErr_Format(PyExc_TypeError, + "%.200s.__dict__ is not a dictionary", + name); + } + } + + Py_XDECREF(dict); + return result; +} + +static PyMethodDef module_methods[] = { + {"__dir__", module_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> specialized dir() implementation")}, + {0} +}; + PyDoc_STRVAR(module_doc, "module(name[, doc])\n\ @@ -449,7 +477,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ - 0, /* tp_methods */ + module_methods, /* tp_methods */ module_members, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ diff --git a/Objects/object.c b/Objects/object.c index d8e2ffb..e42c1d9 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1182,66 +1182,6 @@ PyCallable_Check(PyObject *x) return x->ob_type->tp_call != NULL; } -/* ------------------------- PyObject_Dir() helpers ------------------------- */ - -/* Helper for PyObject_Dir. - Merge the __dict__ of aclass into dict, and recursively also all - the __dict__s of aclass's base classes. The order of merging isn't - defined, as it's expected that only the final set of dict keys is - interesting. - Return 0 on success, -1 on error. -*/ - -static int -merge_class_dict(PyObject* dict, PyObject* aclass) -{ - PyObject *classdict; - PyObject *bases; - - assert(PyDict_Check(dict)); - assert(aclass); - - /* Merge in the type's dict (if any). */ - classdict = PyObject_GetAttrString(aclass, "__dict__"); - if (classdict == NULL) - PyErr_Clear(); - else { - int status = PyDict_Update(dict, classdict); - Py_DECREF(classdict); - if (status < 0) - return -1; - } - - /* Recursively merge in the base types' (if any) dicts. */ - bases = PyObject_GetAttrString(aclass, "__bases__"); - if (bases == NULL) - PyErr_Clear(); - else { - /* We have no guarantee that bases is a real tuple */ - Py_ssize_t i, n; - n = PySequence_Size(bases); /* This better be right */ - if (n < 0) - PyErr_Clear(); - else { - for (i = 0; i < n; i++) { - int status; - PyObject *base = PySequence_GetItem(bases, i); - if (base == NULL) { - Py_DECREF(bases); - return -1; - } - status = merge_class_dict(dict, base); - Py_DECREF(base); - if (status < 0) { - Py_DECREF(bases); - return -1; - } - } - } - Py_DECREF(bases); - } - return 0; -} /* Helper for PyObject_Dir without arguments: returns the local scope. */ static PyObject * @@ -1269,133 +1209,34 @@ _dir_locals(void) return names; } -/* Helper for PyObject_Dir of type objects: returns __dict__ and __bases__. - We deliberately don't suck up its __class__, as methods belonging to the - metaclass would probably be more confusing than helpful. -*/ -static PyObject * -_specialized_dir_type(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = PyDict_New(); - - if (dict != NULL && merge_class_dict(dict, obj) == 0) - result = PyDict_Keys(dict); - - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir of module objects: returns the module's __dict__. */ -static PyObject * -_specialized_dir_module(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = PyObject_GetAttrString(obj, "__dict__"); - - if (dict != NULL) { - if (PyDict_Check(dict)) - result = PyDict_Keys(dict); - else { - const char *name = PyModule_GetName(obj); - if (name) - PyErr_Format(PyExc_TypeError, - "%.200s.__dict__ is not a dictionary", - name); - } - } - - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir of generic objects: returns __dict__, __class__, - and recursively up the __class__.__bases__ chain. -*/ -static PyObject * -_generic_dir(PyObject *obj) -{ - PyObject *result = NULL; - PyObject *dict = NULL; - PyObject *itsclass = NULL; - - /* Get __dict__ (which may or may not be a real dict...) */ - dict = PyObject_GetAttrString(obj, "__dict__"); - if (dict == NULL) { - PyErr_Clear(); - dict = PyDict_New(); - } - else if (!PyDict_Check(dict)) { - Py_DECREF(dict); - dict = PyDict_New(); - } - else { - /* Copy __dict__ to avoid mutating it. */ - PyObject *temp = PyDict_Copy(dict); - Py_DECREF(dict); - dict = temp; - } - - if (dict == NULL) - goto error; - - /* Merge in attrs reachable from its class. */ - itsclass = PyObject_GetAttrString(obj, "__class__"); - if (itsclass == NULL) - /* XXX(tomer): Perhaps fall back to obj->ob_type if no - __class__ exists? */ - PyErr_Clear(); - else { - if (merge_class_dict(dict, itsclass) != 0) - goto error; - } - - result = PyDict_Keys(dict); - /* fall through */ -error: - Py_XDECREF(itsclass); - Py_XDECREF(dict); - return result; -} - -/* Helper for PyObject_Dir: object introspection. - This calls one of the above specialized versions if no __dir__ method - exists. */ +/* Helper for PyObject_Dir: object introspection. */ static PyObject * _dir_object(PyObject *obj) { - PyObject *result = NULL; + PyObject *result; static PyObject *dir_str = NULL; PyObject *dirfunc = _PyObject_LookupSpecial(obj, "__dir__", &dir_str); assert(obj); if (dirfunc == NULL) { - if (PyErr_Occurred()) - return NULL; - /* use default implementation */ - if (PyModule_Check(obj)) - result = _specialized_dir_module(obj); - else if (PyType_Check(obj)) - result = _specialized_dir_type(obj); - else - result = _generic_dir(obj); + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "object does not provide __dir__"); + return NULL; } - else { - /* use __dir__ */ - result = PyObject_CallFunctionObjArgs(dirfunc, NULL); - Py_DECREF(dirfunc); - if (result == NULL) - return NULL; + /* use __dir__ */ + result = PyObject_CallFunctionObjArgs(dirfunc, NULL); + Py_DECREF(dirfunc); + if (result == NULL) + return NULL; - /* result must be a list */ - /* XXX(gbrandl): could also check if all items are strings */ - if (!PyList_Check(result)) { - PyErr_Format(PyExc_TypeError, - "__dir__() must return a list, not %.200s", - Py_TYPE(result)->tp_name); - Py_DECREF(result); - result = NULL; - } + /* result must be a list */ + /* XXX(gbrandl): could also check if all items are strings */ + if (!PyList_Check(result)) { + PyErr_Format(PyExc_TypeError, + "__dir__() must return a list, not %.200s", + Py_TYPE(result)->tp_name); + Py_DECREF(result); + result = NULL; } return result; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 434608f..128b9fe 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2572,6 +2572,82 @@ type_prepare(PyObject *self, PyObject *args, PyObject *kwds) return PyDict_New(); } +/* + Merge the __dict__ of aclass into dict, and recursively also all + the __dict__s of aclass's base classes. The order of merging isn't + defined, as it's expected that only the final set of dict keys is + interesting. + Return 0 on success, -1 on error. +*/ + +static int +merge_class_dict(PyObject *dict, PyObject *aclass) +{ + PyObject *classdict; + PyObject *bases; + + assert(PyDict_Check(dict)); + assert(aclass); + + /* Merge in the type's dict (if any). */ + classdict = PyObject_GetAttrString(aclass, "__dict__"); + if (classdict == NULL) + PyErr_Clear(); + else { + int status = PyDict_Update(dict, classdict); + Py_DECREF(classdict); + if (status < 0) + return -1; + } + + /* Recursively merge in the base types' (if any) dicts. */ + bases = PyObject_GetAttrString(aclass, "__bases__"); + if (bases == NULL) + PyErr_Clear(); + else { + /* We have no guarantee that bases is a real tuple */ + Py_ssize_t i, n; + n = PySequence_Size(bases); /* This better be right */ + if (n < 0) + PyErr_Clear(); + else { + for (i = 0; i < n; i++) { + int status; + PyObject *base = PySequence_GetItem(bases, i); + if (base == NULL) { + Py_DECREF(bases); + return -1; + } + status = merge_class_dict(dict, base); + Py_DECREF(base); + if (status < 0) { + Py_DECREF(bases); + return -1; + } + } + } + Py_DECREF(bases); + } + return 0; +} + +/* __dir__ for type objects: returns __dict__ and __bases__. + We deliberately don't suck up its __class__, as methods belonging to the + metaclass would probably be more confusing than helpful. +*/ +static PyObject * +type_dir(PyObject *self, PyObject *args) +{ + PyObject *result = NULL; + PyObject *dict = PyDict_New(); + + if (dict != NULL && merge_class_dict(dict, self) == 0) + result = PyDict_Keys(dict); + + Py_XDECREF(dict); + return result; +} + static PyMethodDef type_methods[] = { {"mro", (PyCFunction)mro_external, METH_NOARGS, PyDoc_STR("mro() -> list\nreturn a type's method resolution order")}, @@ -2585,6 +2661,8 @@ static PyMethodDef type_methods[] = { PyDoc_STR("__instancecheck__() -> check if an object is an instance")}, {"__subclasscheck__", type___subclasscheck__, METH_O, PyDoc_STR("__subclasscheck__() -> check if a class is a subclass")}, + {"__dir__", type_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> specialized __dir__ implementation for types")}, {0} }; @@ -3438,6 +3516,53 @@ object_sizeof(PyObject *self, PyObject *args) return PyLong_FromSsize_t(res); } +/* __dir__ for generic objects: returns __dict__, __class__, + and recursively up the __class__.__bases__ chain. +*/ +static PyObject * +object_dir(PyObject *self, PyObject *args) +{ + PyObject *result = NULL; + PyObject *dict = NULL; + PyObject *itsclass = NULL; + + /* Get __dict__ (which may or may not be a real dict...) */ + dict = PyObject_GetAttrString(self, "__dict__"); + if (dict == NULL) { + PyErr_Clear(); + dict = PyDict_New(); + } + else if (!PyDict_Check(dict)) { + Py_DECREF(dict); + dict = PyDict_New(); + } + else { + /* Copy __dict__ to avoid mutating it. */ + PyObject *temp = PyDict_Copy(dict); + Py_DECREF(dict); + dict = temp; + } + + if (dict == NULL) + goto error; + + /* Merge in attrs reachable from its class. */ + itsclass = PyObject_GetAttrString(self, "__class__"); + if (itsclass == NULL) + /* XXX(tomer): Perhaps fall back to obj->ob_type if no + __class__ exists? */ + PyErr_Clear(); + else if (merge_class_dict(dict, itsclass) != 0) + goto error; + + result = PyDict_Keys(dict); + /* fall through */ +error: + Py_XDECREF(itsclass); + Py_XDECREF(dict); + return result; +} + static PyMethodDef object_methods[] = { {"__reduce_ex__", object_reduce_ex, METH_VARARGS, PyDoc_STR("helper for pickle")}, @@ -3449,6 +3574,8 @@ static PyMethodDef object_methods[] = { PyDoc_STR("default object formatter")}, {"__sizeof__", object_sizeof, METH_NOARGS, PyDoc_STR("__sizeof__() -> size of object in memory, in bytes")}, + {"__dir__", object_dir, METH_NOARGS, + PyDoc_STR("__dir__() -> default dir() implementation")}, {0} }; -- cgit v0.12 From 7b54e7562d44826947d2e5167dcfdc7b91a7f181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Tue, 24 May 2011 18:23:15 +0200 Subject: Issue #5715: In socketserver, close the server socket in the child process. --- Lib/socketserver.py | 2 +- Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 3d32c3e..76ac50a 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -529,10 +529,10 @@ class ForkingMixIn: self.active_children = [] self.active_children.append(pid) self.close_request(request) - return else: # Child process. # This must never return, hence os._exit()! + self.socket.close() try: self.finish_request(request, client_address) os._exit(0) diff --git a/Misc/NEWS b/Misc/NEWS index 37bbbc1..17f933a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -75,6 +75,8 @@ Core and Builtins Library ------- +- Issue #5715: In socketserver, close the server socket in the child process. + - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore to be able to unload the module. -- cgit v0.12 From a7cdb0f218b991f7d8c8f45af0e4f975673ee351 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 12:06:31 -0500 Subject: generally, sockets should be closed after they're used --- Lib/socketserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 76ac50a..c53ac29 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -532,13 +532,14 @@ class ForkingMixIn: else: # Child process. # This must never return, hence os._exit()! - self.socket.close() try: self.finish_request(request, client_address) + self.socket.close() os._exit(0) except: try: self.handle_error(request, client_address) + self.socket.close() finally: os._exit(1) -- cgit v0.12 From fbe56bb8bdc40179e42ef14e59b2b24df902c52a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 12:42:51 -0500 Subject: use '->' to indicate return values --- Objects/typeobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 310a56c..5c20e0d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2515,9 +2515,9 @@ static PyMethodDef type_methods[] = { PyDoc_STR("__prepare__() -> dict\n" "used to create the namespace for the class statement")}, {"__instancecheck__", type___instancecheck__, METH_O, - PyDoc_STR("__instancecheck__() -> check if an object is an instance")}, + PyDoc_STR("__instancecheck__() -> bool\ncheck if an object is an instance")}, {"__subclasscheck__", type___subclasscheck__, METH_O, - PyDoc_STR("__subclasscheck__() -> check if a class is a subclass")}, + PyDoc_STR("__subclasscheck__() -> bool\ncheck if a class is a subclass")}, {0} }; @@ -3354,7 +3354,7 @@ static PyMethodDef object_methods[] = { {"__format__", object_format, METH_VARARGS, PyDoc_STR("default object formatter")}, {"__sizeof__", object_sizeof, METH_NOARGS, - PyDoc_STR("__sizeof__() -> size of object in memory, in bytes")}, + PyDoc_STR("__sizeof__() -> int\nsize of object in memory, in bytes")}, {0} }; -- cgit v0.12 From c7284122be9254a6e3b5471a933852b68692130b Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 12:46:15 -0500 Subject: indicate return value on __dir__ methods --- Objects/moduleobject.c | 2 +- Objects/typeobject.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d922249..3817ef3 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -437,7 +437,7 @@ module_dir(PyObject *self, PyObject *args) static PyMethodDef module_methods[] = { {"__dir__", module_dir, METH_NOARGS, - PyDoc_STR("__dir__() -> specialized dir() implementation")}, + PyDoc_STR("__dir__() -> list\nspecialized dir() implementation")}, {0} }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a597ba0..02f86ef 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2662,7 +2662,7 @@ static PyMethodDef type_methods[] = { {"__subclasscheck__", type___subclasscheck__, METH_O, PyDoc_STR("__subclasscheck__() -> bool\ncheck if a class is a subclass")}, {"__dir__", type_dir, METH_NOARGS, - PyDoc_STR("__dir__() -> specialized __dir__ implementation for types")}, + PyDoc_STR("__dir__() -> list\nspecialized __dir__ implementation for types")}, {0} }; @@ -3575,7 +3575,7 @@ static PyMethodDef object_methods[] = { {"__sizeof__", object_sizeof, METH_NOARGS, PyDoc_STR("__sizeof__() -> int\nsize of object in memory, in bytes")}, {"__dir__", object_dir, METH_NOARGS, - PyDoc_STR("__dir__() -> default dir() implementation")}, + PyDoc_STR("__dir__() -> list\ndefault dir() implementation")}, {0} }; -- cgit v0.12 From f8473933083cd045bdb0bf0dda8516e268846bd5 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 12:50:34 -0500 Subject: backout 8b384de4e780, so a proper fix can be considered (#5715) --- Lib/socketserver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index c53ac29..e7faf88 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -534,12 +534,10 @@ class ForkingMixIn: # This must never return, hence os._exit()! try: self.finish_request(request, client_address) - self.socket.close() os._exit(0) except: try: self.handle_error(request, client_address) - self.socket.close() finally: os._exit(1) -- cgit v0.12 From 19fb53c1195653e87b9197211015c624960c7b95 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 21:32:40 +0200 Subject: Issue #12049: improve RAND_bytes() and RAND_pseudo_bytes() documentation Add also a security warning in the module random pointing to ssl.RAND_bytes(). --- Doc/library/random.rst | 6 ++++++ Doc/library/ssl.rst | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index f0c4add..52419a1 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -43,6 +43,12 @@ The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. +.. warning:: + + The generators of the :mod:`random` module should not be used for security + purposes, they are not cryptographic. Use :func:`ssl.RAND_bytes` if you + require a cryptographically secure pseudorandom number generator. + Bookkeeping functions: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index a528a03..295d007 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -166,6 +166,11 @@ Random generation Returns *num* cryptographically strong pseudo-random bytes. + Read the Wikipedia article, `Cryptographically secure pseudorandom number + generator + `_, + to get the requirements of a cryptographically generator. + .. versionadded:: 3.3 .. function:: RAND_pseudo_bytes(num) @@ -174,6 +179,11 @@ Random generation is_cryptographic is True if the bytes generated are cryptographically strong. + Generated pseudo-random byte sequences will be unique if they are of + sufficient length, but are not necessarily unpredictable. They can be used + for non-cryptographic purposes and for certain purposes in cryptographic + protocols, but usually not for key generation etc. + .. versionadded:: 3.3 .. function:: RAND_status() -- cgit v0.12 From 6bcbef7da0127272aa97cdd43ec529bfe92c3251 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 22:17:55 +0200 Subject: Issue #12100: Don't reset incremental encoders of CJK codecs at each call to their encode() method anymore, but continue to call the reset() method if the final argument is True. --- Lib/test/test_multibytecodec.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ Modules/cjkcodecs/multibytecodec.c | 8 ++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index f3c8c61..069d090 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -257,6 +257,36 @@ class Test_ISO2022(unittest.TestCase): # Any ISO 2022 codec will cause the segfault myunichr(x).encode('iso_2022_jp', 'ignore') +class TestStateful(unittest.TestCase): + text = '\u4E16\u4E16' + encoding = 'iso-2022-jp' + expected = b'\x1b$B@$@$' + expected_reset = b'\x1b$B@$@$\x1b(B' + + def test_encode(self): + self.assertEqual(self.text.encode(self.encoding), self.expected_reset) + + def test_incrementalencoder(self): + encoder = codecs.getincrementalencoder(self.encoding)() + output = b''.join( + encoder.encode(char) + for char in self.text) + self.assertEqual(output, self.expected) + + def test_incrementalencoder_final(self): + encoder = codecs.getincrementalencoder(self.encoding)() + last_index = len(self.text) - 1 + output = b''.join( + encoder.encode(char, index == last_index) + for index, char in enumerate(self.text)) + self.assertEqual(output, self.expected_reset) + +class TestHZStateful(TestStateful): + text = '\u804a\u804a' + encoding = 'hz' + expected = b'~{ADAD' + expected_reset = b'~{ADAD~}' + def test_main(): support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS index 17f933a..7862e21 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -75,6 +75,10 @@ Core and Builtins Library ------- +- Issue #12100: Don't reset incremental encoders of CJK codecs at each call to + their encode() method anymore, but continue to call the reset() method if the + final argument is True. + - Issue #5715: In socketserver, close the server socket in the child process. - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index af7ea5b..7b04f020 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -479,7 +479,7 @@ multibytecodec_encode(MultibyteCodec *codec, MultibyteEncodeBuffer buf; Py_ssize_t finalsize, r = 0; - if (datalen == 0) + if (datalen == 0 && !(flags & MBENC_RESET)) return PyBytes_FromStringAndSize(NULL, 0); buf.excobj = NULL; @@ -514,7 +514,7 @@ multibytecodec_encode(MultibyteCodec *codec, break; } - if (codec->encreset != NULL) + if (codec->encreset != NULL && (flags & MBENC_RESET)) for (;;) { Py_ssize_t outleft; @@ -784,8 +784,8 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx, inbuf_end = inbuf + datalen; r = multibytecodec_encode(ctx->codec, &ctx->state, - (const Py_UNICODE **)&inbuf, - datalen, ctx->errors, final ? MBENC_FLUSH : 0); + (const Py_UNICODE **)&inbuf, datalen, + ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0); if (r == NULL) { /* recover the original pending buffer */ if (origpending > 0) -- cgit v0.12 From 1273b7cd9ccd19a168d3def5c7d1479c5fb6119a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 May 2011 23:37:07 +0200 Subject: Issue #12070: Fix the Makefile parser of the sysconfig module to handle correctly references to "bogus variable" (e.g. "prefix=$/opt/python"). --- Lib/sysconfig.py | 6 ++++-- Lib/test/test_sysconfig.py | 34 ++++++++++++++++++++++++---------- Misc/NEWS | 3 +++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 41bccf3..2279a51 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -294,7 +294,7 @@ def _parse_makefile(filename, vars=None): variables.remove(name) if name.startswith('PY_') \ - and name[3:] in renamed_variables: + and name[3:] in renamed_variables: name = name[3:] if name not in done: @@ -302,7 +302,9 @@ def _parse_makefile(filename, vars=None): else: - # bogus variable reference; just drop it since we can't deal + # bogus variable reference (e.g. "prefix=$/opt/python"); + # just drop it since we can't deal + done[name] = value variables.remove(name) # strip spurious spaces diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index a97b388..2ea8819 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -26,7 +26,6 @@ class TestSysConfig(unittest.TestCase): """Make a copy of sys.path""" super(TestSysConfig, self).setUp() self.sys_path = sys.path[:] - self.makefile = None # patching os.uname if hasattr(os, 'uname'): self.uname = os.uname @@ -49,8 +48,6 @@ class TestSysConfig(unittest.TestCase): def tearDown(self): """Restore sys.path""" sys.path[:] = self.sys_path - if self.makefile is not None: - os.unlink(self.makefile) self._cleanup_testfn() if self.uname is not None: os.uname = self.uname @@ -236,12 +233,6 @@ class TestSysConfig(unittest.TestCase): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) - @unittest.skipIf(sys.platform.startswith('win'), - 'Test is not Windows compatible') - def test_get_makefile_filename(self): - makefile = sysconfig.get_makefile_filename() - self.assertTrue(os.path.isfile(makefile), makefile) - def test_get_scheme_names(self): wanted = ('nt', 'nt_user', 'os2', 'os2_home', 'osx_framework_user', 'posix_home', 'posix_prefix', 'posix_user') @@ -339,10 +330,33 @@ class TestSysConfig(unittest.TestCase): self.assertEqual(my_platform, test_platform) +class MakefileTests(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith('win'), + 'Test is not Windows compatible') + def test_get_makefile_filename(self): + makefile = sysconfig.get_makefile_filename() + self.assertTrue(os.path.isfile(makefile), makefile) + + def test_parse_makefile(self): + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as makefile: + print("var1=a$(VAR2)", file=makefile) + print("VAR2=b$(var3)", file=makefile) + print("var3=42", file=makefile) + print("var4=$/invalid", file=makefile) + print("var5=dollar$$5", file=makefile) + vars = sysconfig._parse_makefile(TESTFN) + self.assertEqual(vars, { + 'var1': 'ab42', + 'VAR2': 'b42', + 'var3': 42, + 'var4': '$/invalid', + 'var5': 'dollar$5', + }) def test_main(): - run_unittest(TestSysConfig) + run_unittest(TestSysConfig, MakefileTests) if __name__ == "__main__": test_main() diff --git a/Misc/NEWS b/Misc/NEWS index bca1312..8d19f7a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #12070: Fix the Makefile parser of the sysconfig module to handle + correctly references to "bogus variable" (e.g. "prefix=$/opt/python"). + - Issue #12100: Don't reset incremental encoders of CJK codecs at each call to their encode() method anymore, but continue to call the reset() method if the final argument is True. -- cgit v0.12 From 1f4560c872f56e0d416dd091e9dbccbb93f716ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Tue, 24 May 2011 23:47:49 +0200 Subject: Issue #1441530: In imaplib, read the data in one chunk to speed up large reads and simplify code. --- Lib/imaplib.py | 10 +--------- Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 142e27b..1b54065 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -249,15 +249,7 @@ class IMAP4: def read(self, size): """Read 'size' bytes from remote.""" - chunks = [] - read = 0 - while read < size: - data = self.file.read(min(size-read, 4096)) - if not data: - break - read += len(data) - chunks.append(data) - return b''.join(chunks) + return self.file.read(size) def readline(self): diff --git a/Misc/NEWS b/Misc/NEWS index 917dfe3..68130a9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #1441530: In imaplib, read the data in one chunk to speed up large + reads and simplify code. + - Issue #12070: Fix the Makefile parser of the sysconfig module to handle correctly references to "bogus variable" (e.g. "prefix=$/opt/python"). -- cgit v0.12 From 8fdfc20fc43d7dbfa1770911bd8a183de4f225c5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 00:06:51 +0200 Subject: Issue #12057: Add tests for the HZ encoding --- Lib/test/cjkencodings/hz-utf8.txt | 2 ++ Lib/test/cjkencodings/hz.txt | 2 ++ Lib/test/test_codecencodings_cn.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 Lib/test/cjkencodings/hz-utf8.txt create mode 100644 Lib/test/cjkencodings/hz.txt diff --git a/Lib/test/cjkencodings/hz-utf8.txt b/Lib/test/cjkencodings/hz-utf8.txt new file mode 100644 index 0000000..7c11735 --- /dev/null +++ b/Lib/test/cjkencodings/hz-utf8.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.己所不欲,勿施於人。Bye. diff --git a/Lib/test/cjkencodings/hz.txt b/Lib/test/cjkencodings/hz.txt new file mode 100644 index 0000000..f882d46 --- /dev/null +++ b/Lib/test/cjkencodings/hz.txt @@ -0,0 +1,2 @@ +This sentence is in ASCII. +The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye. diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index 77bac13..9622b63 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -50,6 +50,35 @@ class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase): ) has_iso10646 = True +class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'hz' + tstring = test_multibytecodec_support.load_teststring('hz') + codectests = ( + # test '~\n' (3 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~{<:Ky2;S{#,~}~\n' + b'~{NpJ)l6HK!#~}Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # test '~\n' (4 lines) + (b'This sentence is in ASCII.\n' + b'The next sentence is in GB.~\n' + b'~{<:Ky2;S{#,NpJ)l6HK!#~}~\n' + b'Bye.\n', + 'strict', + 'This sentence is in ASCII.\n' + 'The next sentence is in GB.' + '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' + 'Bye.\n'), + # invalid bytes + (b'ab~cd', 'replace', 'ab\uFFFDd'), + (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), + (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + ) + def test_main(): support.run_unittest(__name__) -- cgit v0.12 From f1a1af21eb024d32261ffcd1b32ca287da543a5a Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Wed, 25 May 2011 00:32:08 +0200 Subject: Fix typo in bz2 module. --- Lib/bz2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/bz2.py b/Lib/bz2.py index 9506b4a..8ffeaac 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -155,7 +155,7 @@ class BZ2File(io.BufferedIOBase): if not self.seekable(): self._check_not_closed() raise io.UnsupportedOperation("Seeking is only supported " - "on files opening for reading") + "on files open for reading") # Fill the readahead buffer if it is empty. Returns False on EOF. def _fill_buffer(self): -- cgit v0.12 From 46f8264b8e976ca0a39485c8405acddc8aa8af7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 01:13:59 +0200 Subject: Issue #12057: Add cjkencodings directory to the Makefile and Tools/msi/msi.py --- Makefile.pre.in | 4 ++-- Tools/msi/msi.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index c2fc69a..270dfb4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -840,8 +840,8 @@ EXTRAPLATDIR= @EXTRAPLATDIR@ MACHDEPS= $(PLATDIR) $(EXTRAPLATDIR) XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ - tkinter/test/test_ttk site-packages test test/data \ - test/decimaltestdata \ + tkinter/test/test_ttk site-packages test test/data \ + test/cjkencodings test/decimaltestdata \ test/tracedmodules \ encodings \ email email/mime email/test email/test/data \ diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index b668c7a..78ff51b 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -1021,6 +1021,8 @@ def add_files(db): lib.add_file("zipdir.zip") if dir=='tests' and parent.physical=='distutils': lib.add_file("Setup.sample") + if dir=='cjkencodings': + lib.glob("*.txt") if dir=='decimaltestdata': lib.glob("*.decTest") if dir=='output': -- cgit v0.12 From 383c3fc6b41475884f717fea85611e9b423d6fa1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 01:35:05 +0200 Subject: Issue #10818: Remove the Tk GUI of the pydoc module (pydoc -g has been deprecated in Python 3.2). --- Doc/whatsnew/3.3.rst | 7 ++ Lib/pydoc.py | 194 +-------------------------------------------------- Misc/NEWS | 3 + 3 files changed, 11 insertions(+), 193 deletions(-) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 529665f..7b6327e 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -124,6 +124,13 @@ os (Patch submitted by Giampaolo Rodolà in :issue:`10784`.) +pydoc +----- + +The Tk GUI has been removed from the :mod:`ssl` module: ``pydoc -g`` has been +deprecated in Python 3.2. + + sys --- diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 8581d63..17ae29f 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -22,11 +22,6 @@ Run "pydoc -b" to start an HTTP server on an arbitrary unused port and open a Web browser to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. -For platforms without a command line, "pydoc -g" starts the HTTP server -and also pops up a little window for controlling it. This option is -deprecated, since the server can now be controlled directly from HTTP -clients. - Run "pydoc -w " to write out the HTML documentation for a module to a file named ".html". @@ -2140,187 +2135,6 @@ pydoc by Ka-Ping Yee <ping@lfw.org>''' finally: if completer: completer() -# ----------------------------------------------------- graphical interface - -def gui(): - """Graphical interface (starts Web server and pops up a control window).""" - - msg = ('the pydoc.gui() function and "pydoc -g" option are deprecated\n', - 'use "pydoc.browse() function and "pydoc -b" option instead.') - warnings.warn(msg, DeprecationWarning, stacklevel=2) - - class GUI: - def __init__(self, window, port=7464): - self.window = window - self.server = None - self.scanner = None - - import tkinter - self.server_frm = tkinter.Frame(window) - self.title_lbl = tkinter.Label(self.server_frm, - text='Starting server...\n ') - self.open_btn = tkinter.Button(self.server_frm, - text='open browser', command=self.open, state='disabled') - self.quit_btn = tkinter.Button(self.server_frm, - text='quit serving', command=self.quit, state='disabled') - - self.search_frm = tkinter.Frame(window) - self.search_lbl = tkinter.Label(self.search_frm, text='Search for') - self.search_ent = tkinter.Entry(self.search_frm) - self.search_ent.bind('', self.search) - self.stop_btn = tkinter.Button(self.search_frm, - text='stop', pady=0, command=self.stop, state='disabled') - if sys.platform == 'win32': - # Trying to hide and show this button crashes under Windows. - self.stop_btn.pack(side='right') - - self.window.title('pydoc') - self.window.protocol('WM_DELETE_WINDOW', self.quit) - self.title_lbl.pack(side='top', fill='x') - self.open_btn.pack(side='left', fill='x', expand=1) - self.quit_btn.pack(side='right', fill='x', expand=1) - self.server_frm.pack(side='top', fill='x') - - self.search_lbl.pack(side='left') - self.search_ent.pack(side='right', fill='x', expand=1) - self.search_frm.pack(side='top', fill='x') - self.search_ent.focus_set() - - font = ('helvetica', sys.platform == 'win32' and 8 or 10) - self.result_lst = tkinter.Listbox(window, font=font, height=6) - self.result_lst.bind('', self.select) - self.result_lst.bind('', self.goto) - self.result_scr = tkinter.Scrollbar(window, - orient='vertical', command=self.result_lst.yview) - self.result_lst.config(yscrollcommand=self.result_scr.set) - - self.result_frm = tkinter.Frame(window) - self.goto_btn = tkinter.Button(self.result_frm, - text='go to selected', command=self.goto) - self.hide_btn = tkinter.Button(self.result_frm, - text='hide results', command=self.hide) - self.goto_btn.pack(side='left', fill='x', expand=1) - self.hide_btn.pack(side='right', fill='x', expand=1) - - self.window.update() - self.minwidth = self.window.winfo_width() - self.minheight = self.window.winfo_height() - self.bigminheight = (self.server_frm.winfo_reqheight() + - self.search_frm.winfo_reqheight() + - self.result_lst.winfo_reqheight() + - self.result_frm.winfo_reqheight()) - self.bigwidth, self.bigheight = self.minwidth, self.bigminheight - self.expanded = 0 - self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight)) - self.window.wm_minsize(self.minwidth, self.minheight) - self.window.tk.willdispatch() - - import threading - threading.Thread( - target=serve, args=(port, self.ready, self.quit)).start() - - def ready(self, server): - self.server = server - self.title_lbl.config( - text='Python documentation server at\n' + server.url) - self.open_btn.config(state='normal') - self.quit_btn.config(state='normal') - - def open(self, event=None, url=None): - url = url or self.server.url - import webbrowser - webbrowser.open(url) - - def quit(self, event=None): - if self.server: - self.server.quit = 1 - self.window.quit() - - def search(self, event=None): - key = self.search_ent.get() - self.stop_btn.pack(side='right') - self.stop_btn.config(state='normal') - self.search_lbl.config(text='Searching for "%s"...' % key) - self.search_ent.forget() - self.search_lbl.pack(side='left') - self.result_lst.delete(0, 'end') - self.goto_btn.config(state='disabled') - self.expand() - - import threading - if self.scanner: - self.scanner.quit = 1 - self.scanner = ModuleScanner() - threading.Thread(target=self.scanner.run, - args=(self.update, key, self.done)).start() - - def update(self, path, modname, desc): - if modname[-9:] == '.__init__': - modname = modname[:-9] + ' (package)' - self.result_lst.insert('end', - modname + ' - ' + (desc or '(no description)')) - - def stop(self, event=None): - if self.scanner: - self.scanner.quit = 1 - self.scanner = None - - def done(self): - self.scanner = None - self.search_lbl.config(text='Search for') - self.search_lbl.pack(side='left') - self.search_ent.pack(side='right', fill='x', expand=1) - if sys.platform != 'win32': self.stop_btn.forget() - self.stop_btn.config(state='disabled') - - def select(self, event=None): - self.goto_btn.config(state='normal') - - def goto(self, event=None): - selection = self.result_lst.curselection() - if selection: - modname = self.result_lst.get(selection[0]).split()[0] - self.open(url=self.server.url + modname + '.html') - - def collapse(self): - if not self.expanded: return - self.result_frm.forget() - self.result_scr.forget() - self.result_lst.forget() - self.bigwidth = self.window.winfo_width() - self.bigheight = self.window.winfo_height() - self.window.wm_geometry('%dx%d' % (self.minwidth, self.minheight)) - self.window.wm_minsize(self.minwidth, self.minheight) - self.expanded = 0 - - def expand(self): - if self.expanded: return - self.result_frm.pack(side='bottom', fill='x') - self.result_scr.pack(side='right', fill='y') - self.result_lst.pack(side='top', fill='both', expand=1) - self.window.wm_geometry('%dx%d' % (self.bigwidth, self.bigheight)) - self.window.wm_minsize(self.minwidth, self.bigminheight) - self.expanded = 1 - - def hide(self, event=None): - self.stop() - self.collapse() - - import tkinter - try: - root = tkinter.Tk() - # Tk will crash if pythonw.exe has an XP .manifest - # file and the root has is not destroyed explicitly. - # If the problem is ever fixed in Tk, the explicit - # destroy can go. - try: - gui = GUI(root) - root.mainloop() - finally: - root.destroy() - except KeyboardInterrupt: - pass - # --------------------------------------- enhanced Web browser interface @@ -2778,15 +2592,12 @@ def cli(): sys.path.insert(0, '.') try: - opts, args = getopt.getopt(sys.argv[1:], 'bgk:p:w') + opts, args = getopt.getopt(sys.argv[1:], 'bk:p:w') writing = False start_server = False open_browser = False port = None for opt, val in opts: - if opt == '-g': - gui() - return if opt == '-b': start_server = True open_browser = True @@ -2847,9 +2658,6 @@ def cli(): to interactively browse documentation. The -p option can be used with the -b option to explicitly specify the server port. -{cmd} -g - Deprecated. - {cmd} -w ... Write out the HTML documentation for a module to a file in the current directory. If contains a '{sep}', it is treated as a filename; if diff --git a/Misc/NEWS b/Misc/NEWS index 68130a9..4b8d00a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #10818: Remove the Tk GUI of the pydoc module (pydoc -g has been + deprecated in Python 3.2). + - Issue #1441530: In imaplib, read the data in one chunk to speed up large reads and simplify code. -- cgit v0.12 From 6daa33c8acf47bcc41ea827e7b3dbcc5fae9f771 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 01:41:22 +0200 Subject: Issue #10818: Remove deprecated pydoc.serve() function The pydoc module has a new enhanced web server. --- Doc/whatsnew/3.3.rst | 5 ++-- Lib/pydoc.py | 85 ---------------------------------------------------- Misc/NEWS | 5 ++-- 3 files changed, 6 insertions(+), 89 deletions(-) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 7b6327e..2b9bd11 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -127,8 +127,9 @@ os pydoc ----- -The Tk GUI has been removed from the :mod:`ssl` module: ``pydoc -g`` has been -deprecated in Python 3.2. +The Tk GUI and the :func:`~pydoc.serve` function have been removed from the +:mod:`pydoc` module: ``pydoc -g`` and :func:`~pydoc.serve` have been deprecated +in Python 3.2. sys diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 17ae29f..548e71c 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2051,91 +2051,6 @@ def apropos(key): warnings.filterwarnings('ignore') # ignore problems during import ModuleScanner().run(callback, key, onerror=onerror) -# --------------------------------------------------- Web browser interface - -def serve(port, callback=None, completer=None): - import http.server, email.message, select - - msg = 'the pydoc.serve() function is deprecated' - warnings.warn(msg, DeprecationWarning, stacklevel=2) - - class DocHandler(http.server.BaseHTTPRequestHandler): - def send_document(self, title, contents): - try: - self.send_response(200) - self.send_header('Content-Type', 'text/html; charset=UTF-8') - self.end_headers() - self.wfile.write(html.page(title, contents).encode('utf-8')) - except IOError: pass - - def do_GET(self): - path = self.path - if path[-5:] == '.html': path = path[:-5] - if path[:1] == '/': path = path[1:] - if path and path != '.': - try: - obj = locate(path, forceload=1) - except ErrorDuringImport as value: - self.send_document(path, html.escape(str(value))) - return - if obj: - self.send_document(describe(obj), html.document(obj, path)) - else: - self.send_document(path, -'no Python documentation found for %s' % repr(path)) - else: - heading = html.heading( -'Python: Index of Modules', -'#ffffff', '#7799ee') - def bltinlink(name): - return '%s' % (name, name) - names = [x for x in sys.builtin_module_names if x != '__main__'] - contents = html.multicolumn(names, bltinlink) - indices = ['

' + html.bigsection( - 'Built-in Modules', '#ffffff', '#ee77aa', contents)] - - seen = {} - for dir in sys.path: - indices.append(html.index(dir, seen)) - contents = heading + ' '.join(indices) + '''

- -pydoc by Ka-Ping Yee <ping@lfw.org>''' - self.send_document('Index of Modules', contents) - - def log_message(self, *args): pass - - class DocServer(http.server.HTTPServer): - def __init__(self, port, callback): - host = 'localhost' - self.address = (host, port) - self.url = 'http://%s:%d/' % (host, port) - self.callback = callback - self.base.__init__(self, self.address, self.handler) - - def serve_until_quit(self): - import select - self.quit = False - while not self.quit: - rd, wr, ex = select.select([self.socket.fileno()], [], [], 1) - if rd: self.handle_request() - self.server_close() - - def server_activate(self): - self.base.server_activate(self) - if self.callback: self.callback(self) - - DocServer.base = http.server.HTTPServer - DocServer.handler = DocHandler - DocHandler.MessageClass = email.message.Message - try: - try: - DocServer(port, callback).serve_until_quit() - except (KeyboardInterrupt, select.error): - pass - finally: - if completer: completer() - - # --------------------------------------- enhanced Web browser interface def _start_server(urlhandler, port): diff --git a/Misc/NEWS b/Misc/NEWS index 4b8d00a..f602595 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,8 +161,9 @@ Core and Builtins Library ------- -- Issue #10818: Remove the Tk GUI of the pydoc module (pydoc -g has been - deprecated in Python 3.2). +- Issue #10818: Remove the Tk GUI and the serve() function of the pydoc module, + pydoc -g has been deprecated in Python 3.2 and it has a new enhanced web + server. - Issue #1441530: In imaplib, read the data in one chunk to speed up large reads and simplify code. -- cgit v0.12 From be6210363e1bd275df5a31c394f39b934dad1b14 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 02:01:55 +0200 Subject: Issue #8533: regrtest replaces also sys.stdout on Windows Replace sys.stdout to use backslashreplace. Use '\n' newline on all operating systems. --- Lib/test/regrtest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 04e87e3..b5e5127 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -794,17 +794,14 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" - if os.name == "nt": - # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 - return - import atexit stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, errors="backslashreplace", - closefd=False) + closefd=False, + newline='\n') def restore_stdout(): sys.stdout.close() -- cgit v0.12 From b45c7087aa967843f2e11116fbb4a7f5a32a1c3f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 24 May 2011 19:31:01 -0500 Subject: excellent place for assertRaises --- Lib/test/test_descr.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 31731d2..0fae2df 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1631,12 +1631,7 @@ order (MRO) for bases """ for attr, obj in env.items(): setattr(X, attr, obj) setattr(X, name, ErrDescr()) - try: - runner(X()) - except MyException: - pass - else: - self.fail("{0!r} didn't raise".format(name)) + self.assertRaises(MyException, runner, X()) def test_specials(self): # Testing special operators... -- cgit v0.12 From c13ef66649985025382c64f6af8e3b956411e05b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 02:35:58 +0200 Subject: Issue #8407: Fix the signal handler of the signal module: if it is called twice, it now writes the number of the second signal into the wakeup fd. --- Lib/test/test_signal.py | 36 ++++++++++++++++++++++++++++++++---- Modules/signalmodule.c | 9 +++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 3134031..cdd3f3e 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -226,10 +226,16 @@ class WakeupSignalTests(unittest.TestCase): TIMEOUT_FULL = 10 TIMEOUT_HALF = 5 - def check_signum(self, *signals): + def handler(self, signum, frame): + pass + + def check_signum(self, *signals, **kw): data = os.read(self.read, len(signals)+1) raised = struct.unpack('%uB' % len(data), data) - self.assertSequenceEqual(raised, signals) + if kw.get('unordered', False): + raised = set(raised) + signals = set(signals) + self.assertEqual(raised, signals) def test_wakeup_fd_early(self): import select @@ -259,16 +265,38 @@ class WakeupSignalTests(unittest.TestCase): self.check_signum(signal.SIGALRM) def test_signum(self): - old_handler = signal.signal(signal.SIGUSR1, lambda x,y:None) + old_handler = signal.signal(signal.SIGUSR1, self.handler) self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) os.kill(os.getpid(), signal.SIGUSR1) os.kill(os.getpid(), signal.SIGALRM) self.check_signum(signal.SIGUSR1, signal.SIGALRM) + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pending(self): + signum1 = signal.SIGUSR1 + signum2 = signal.SIGUSR2 + tid = threading.current_thread().ident + + old_handler = signal.signal(signum1, self.handler) + self.addCleanup(signal.signal, signum1, old_handler) + old_handler = signal.signal(signum2, self.handler) + self.addCleanup(signal.signal, signum2, old_handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, (signum1, signum2)) + signal.pthread_kill(tid, signum1) + signal.pthread_kill(tid, signum2) + # Unblocking the 2 signals calls the C signal handler twice + signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2)) + + self.check_signum(signum1, signum2, unordered=True) + def setUp(self): import fcntl - self.alrm = signal.signal(signal.SIGALRM, lambda x,y:None) + self.alrm = signal.signal(signal.SIGALRM, self.handler) self.read, self.write = os.pipe() flags = fcntl.fcntl(self.write, fcntl.F_GETFL, 0) flags = flags | os.O_NONBLOCK diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index ff65f04..6d27ab3 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -177,17 +177,18 @@ static void trip_signal(int sig_num) { unsigned char byte; + Handlers[sig_num].tripped = 1; + if (wakeup_fd != -1) { + byte = (unsigned char)sig_num; + write(wakeup_fd, &byte, 1); + } if (is_tripped) return; /* Set is_tripped after setting .tripped, as it gets cleared in PyErr_CheckSignals() before .tripped. */ is_tripped = 1; Py_AddPendingCall(checksignals_witharg, NULL); - if (wakeup_fd != -1) { - byte = (unsigned char)sig_num; - write(wakeup_fd, &byte, 1); - } } static void -- cgit v0.12 From 2e2baa9208763119c899d0026ec726dd4ccdc384 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 11:15:16 +0200 Subject: Issue #12049: test_ssl now checks also that RAND_bytes() raises an error if there is not enough entropy. --- Lib/test/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 8c21975..5193c15 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -109,6 +109,8 @@ class BasicSocketTests(unittest.TestCase): if v: data = ssl.RAND_bytes(16) self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) try: ssl.RAND_egd(1) -- cgit v0.12 From a675206366435432118e443090d3e08613db6679 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 11:27:40 +0200 Subject: Issue #12049: Document errors cases of ssl.RAND_bytes() and ssl.RAND_pseudo_bytes(). Add also links to RAND_status and RAND_add. --- Doc/library/ssl.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 295d007..ca71d20 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -164,10 +164,14 @@ Random generation .. function:: RAND_bytes(num) - Returns *num* cryptographically strong pseudo-random bytes. + Returns *num* cryptographically strong pseudo-random bytes. Raises an + :class:`SSLError` if the PRNG has not been seeded with enough data or if the + operation is not supported by the current RAND method. :func:`RAND_status` + can be used to check the status of the PRNG and :func:`RAND_add` can be used + to seed the PRNG. Read the Wikipedia article, `Cryptographically secure pseudorandom number - generator + generator (CSPRNG) `_, to get the requirements of a cryptographically generator. @@ -177,7 +181,8 @@ Random generation Returns (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, is_cryptographic is True if the bytes generated are cryptographically - strong. + strong. Raises an :class:`SSLError` if the operation is not supported by the + current RAND method. Generated pseudo-random byte sequences will be unique if they are of sufficient length, but are not necessarily unpredictable. They can be used -- cgit v0.12 From c58140c55798f77bd003ae48b1cf50daf1727f66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 13:13:55 +0200 Subject: Issue #12049: cleanup the warning in the random module doc --- Doc/library/random.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 52419a1..3040411 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -46,8 +46,8 @@ from sources provided by the operating system. .. warning:: The generators of the :mod:`random` module should not be used for security - purposes, they are not cryptographic. Use :func:`ssl.RAND_bytes` if you - require a cryptographically secure pseudorandom number generator. + purposes. Use :func:`ssl.RAND_bytes` if you require a cryptographically + secure pseudorandom number generator. Bookkeeping functions: -- cgit v0.12 From 81b21d77668520f34f288fd21b8dc47809316c77 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:21:46 -0500 Subject: test values and types of (True/False).(imag/real) --- Lib/test/test_bool.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index b296870..4bab28b 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -330,6 +330,16 @@ class BoolTest(unittest.TestCase): except (Exception) as e_len: self.assertEqual(str(e_bool), str(e_len)) + def test_real_and_imag(self): + self.assertEqual(True.real, 1) + self.assertEqual(True.imag, 0) + self.assertIs(type(True.real), int) + self.assertIs(type(True.imag), int) + self.assertEqual(False.real, 0) + self.assertEqual(False.imag, 0) + self.assertIs(type(False.real), int) + self.assertIs(type(False.imag), int) + def test_main(): support.run_unittest(BoolTest) -- cgit v0.12 From 8a021009ea5097dc7914d64b075f95133c15a58f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:27:43 -0500 Subject: wrap at 80 chars --- Misc/NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index f602595..6762731 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -823,7 +823,8 @@ Tests C-API ----- -- PY_PATCHLEVEL_REVISION has been removed, since it's meaningless with Mercurial. +- PY_PATCHLEVEL_REVISION has been removed, since it's meaningless with + Mercurial. Documentation ------------- -- cgit v0.12 From 04778a8150430edc2c441f2f86193ca690d60aa3 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:29:00 -0500 Subject: make PyImport_ImportModuleLevel's first arg const like similiar functions (closes #12173) --- Include/import.h | 2 +- Misc/NEWS | 3 +++ Python/import.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Include/import.h b/Include/import.h index 6331edf..45544111 100644 --- a/Include/import.h +++ b/Include/import.h @@ -44,7 +44,7 @@ PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock( const char *name /* UTF-8 encoded string */ ); PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel( - char *name, /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ PyObject *globals, PyObject *locals, PyObject *fromlist, diff --git a/Misc/NEWS b/Misc/NEWS index 6762731..36a3c9d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -826,6 +826,9 @@ C-API - PY_PATCHLEVEL_REVISION has been removed, since it's meaningless with Mercurial. +- Issue #12173: The first argument of PyImport_ImportModuleLevel is now `const + char *` instead of `char *1`. + Documentation ------------- diff --git a/Python/import.c b/Python/import.c index 5360d57..bfb976c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2818,7 +2818,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { PyObject *nameobj, *mod; -- cgit v0.12 From dac9acedfd9930d2368172c2cec13b6077153a70 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:33:37 -0500 Subject: test that object has a __dir__() implementation --- Lib/test/test_builtin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index b094a65..97619cf 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -385,6 +385,8 @@ class BuiltinTest(unittest.TestCase): except: self.assertEqual(len(dir(sys.exc_info()[2])), 4) + # test that object has a __dir__() + self.assertEqual(sorted([].__dir__()), dir([])) def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) -- cgit v0.12 From d4701f426261b93fa528712c0c44426bccdfed41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 25 May 2011 18:13:29 +0200 Subject: Fix two typos --- Doc/documenting/style.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/documenting/style.rst b/Doc/documenting/style.rst index ade845c..2548cb0 100644 --- a/Doc/documenting/style.rst +++ b/Doc/documenting/style.rst @@ -135,7 +135,7 @@ Good example (establishing confident knowledge in the effective use of the langu Economy of Expression --------------------- -More documentation is not necessarily better documentation. Error on the side +More documentation is not necessarily better documentation. Err on the side of being succinct. It is an unfortunate fact that making documentation longer can be an impediment @@ -197,7 +197,7 @@ Audience The tone of the tutorial (and all the docs) needs to be respectful of the reader's intelligence. Don't presume that the readers are stupid. Lay out the relevant information, show motivating use cases, provide glossary links, and do -our best to connect-the-dots, but don't talk down to them or waste their time. +your best to connect-the-dots, but don't talk down to them or waste their time. The tutorial is meant for newcomers, many of whom will be using the tutorial to evaluate the language as a whole. The experience needs to be positive and not -- cgit v0.12 From b46004c94f59af1adc081d2a89d9e3f73998ee59 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 25 May 2011 18:17:25 +0200 Subject: Issue #12045: Avoid duplicate execution of command in ctypes.util._get_soname(). Patch by Sijin Joseph. --- Lib/ctypes/util.py | 4 +--- Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index e4c204d..6815f94 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -137,9 +137,7 @@ elif os.name == "posix": rv = f.close() if rv == 10: raise OSError('objdump command not found') - with contextlib.closing(os.popen(cmd)) as f: - data = f.read() - res = re.search(r'\sSONAME\s+([^\s]+)', data) + res = re.search(r'\sSONAME\s+([^\s]+)', dump) if not res: return None return res.group(1) diff --git a/Misc/NEWS b/Misc/NEWS index 36a3c9d..cba2d50 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #12045: Avoid duplicate execution of command in ctypes.util._get_soname(). + Patch by Sijin Joseph. + - Issue #10818: Remove the Tk GUI and the serve() function of the pydoc module, pydoc -g has been deprecated in Python 3.2 and it has a new enhanced web server. -- cgit v0.12 From 559b5f1ca0a5dec9d485f01895b61e3d7da5075c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 25 May 2011 18:21:43 +0200 Subject: Minor cleanup in sysconfig. Also remove outdated and unhelpful docstrings in test_sysconfig. --- Lib/sysconfig.py | 5 ++--- Lib/test/test_sysconfig.py | 15 --------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 4c1fd1b..59073f4 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -656,11 +656,10 @@ def get_platform(): # to. This makes the compatibility story a bit more sane because the # machine is going to compile and link as if it were # MACOSX_DEPLOYMENT_TARGET. - # cfgvars = get_config_vars() macver = cfgvars.get('MACOSX_DEPLOYMENT_TARGET') - if 1: + if True: # Always calculate the release of the running machine, # needed to determine if we can build fat binaries or not. @@ -755,7 +754,7 @@ def _main(): print('Platform: "%s"' % get_platform()) print('Python version: "%s"' % get_python_version()) print('Current installation scheme: "%s"' % _get_default_scheme()) - print('') + print() _print_dict('Paths', get_paths()) print() _print_dict('Variables', get_config_vars()) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 214fc70..77c2364 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,9 +1,3 @@ -"""Tests for 'site'. - -Tests assume the initial paths in sys.path once the interpreter has begun -executing have not been removed. - -""" import unittest import sys import os @@ -23,7 +17,6 @@ from sysconfig import (get_paths, get_platform, get_config_vars, class TestSysConfig(unittest.TestCase): def setUp(self): - """Make a copy of sys.path""" super(TestSysConfig, self).setUp() self.sys_path = sys.path[:] self.makefile = None @@ -53,7 +46,6 @@ class TestSysConfig(unittest.TestCase): self._added_envvars.append(var) def tearDown(self): - """Restore sys.path""" sys.path[:] = self.sys_path if self.makefile is not None: os.unlink(self.makefile) @@ -145,8 +137,6 @@ class TestSysConfig(unittest.TestCase): ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' 'root:xnu-792.25.20~1/RELEASE_I386'), 'PowerPC')) - - get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3' get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g ' @@ -161,7 +151,6 @@ class TestSysConfig(unittest.TestCase): finally: sys.maxsize = maxint - self._set_uname(('Darwin', 'macziade', '8.11.1', ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' @@ -295,7 +284,6 @@ class TestSysConfig(unittest.TestCase): self.assertIn(ldflags, ldshared) - @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") def test_platform_in_subprocess(self): my_platform = sysconfig.get_platform() @@ -321,7 +309,6 @@ class TestSysConfig(unittest.TestCase): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) - # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and # using a value that is unlikely to be the default one. env = os.environ.copy() @@ -342,8 +329,6 @@ class TestSysConfig(unittest.TestCase): self.assertEqual(my_platform, test_platform) - - def test_main(): run_unittest(TestSysConfig) -- cgit v0.12 From 5e826e8a1b4727f42cbbb4bd279741ed96404809 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 26 May 2011 00:22:59 +0800 Subject: Fix closes issue #11109 - socketserver.ForkingMixIn leaves zombies, also fails to reap all zombies in one pass. A new method called service_action is made available in BaseServer, called by serve_forever loop. This useful in cases where Mixins can use it for cleanup action. ForkingMixin class uses service_action to collect the zombie child processes. Initial Patch by Justin Wark. --- Doc/library/socketserver.rst | 15 ++++++++++++++- Lib/socketserver.py | 22 ++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index ed547f5e..a4c9b63 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -153,8 +153,21 @@ Server Objects .. method:: BaseServer.serve_forever(poll_interval=0.5) Handle requests until an explicit :meth:`shutdown` request. Polls for - shutdown every *poll_interval* seconds. + shutdown every *poll_interval* seconds. It also calls + :meth:`service_actions` which may be used by a subclass or Mixin to provide + various cleanup actions. For e.g. ForkingMixin class uses + :meth:`service_actions` to cleanup the zombie child processes. + .. versionchanged:: 3.3 + Added service_actions call to the serve_forever method. + + +.. method:: BaseServer.service_actions() + + This is called by the serve_forever loop. This method is can be overridden + by Mixin's to add cleanup or service specific actions. + + .. versionadded:: 3.3 .. method:: BaseServer.shutdown() diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 7389608..a2f0f39 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -82,7 +82,7 @@ On the other hand, if you are building e.g. an HTTP server, where all data is stored externally (e.g. in the file system), a synchronous class will essentially render the service "deaf" while one request is being handled -- which may be for a very long time if a client is slow -to reqd all the data it has requested. Here a threading or forking +to recv all the data it has requested. Here a threading or forking server is appropriate. In some cases, it may be appropriate to process part of a request @@ -170,6 +170,7 @@ class BaseServer: - process_request(request, client_address) - shutdown_request(request) - close_request(request) + - service_actions() - handle_error() Methods for derived classes: @@ -225,6 +226,8 @@ class BaseServer: r, w, e = select.select([self], [], [], poll_interval) if self in r: self._handle_request_noblock() + + self.service_actions() finally: self.__shutdown_request = False self.__is_shut_down.set() @@ -239,6 +242,14 @@ class BaseServer: self.__shutdown_request = True self.__is_shut_down.wait() + def service_actions(self): + """Called by the serve_forever() loop. + + May be overridden by a subclass / Mixin to implement any code that + needs to be run during the loop. + """ + pass + # The distinction between handling, getting, processing and # finishing a request is fairly arbitrary. Remember: # @@ -539,9 +550,15 @@ class ForkingMixIn: """ self.collect_children() + def service_actions(self): + """Collect the zombie child processes regularly in the ForkingMixin. + + service_actions is called in the BaseServer's serve_forver loop. + """ + self.collect_children() + def process_request(self, request, client_address): """Fork a new subprocess to process the request.""" - self.collect_children() pid = os.fork() if pid: # Parent process @@ -549,6 +566,7 @@ class ForkingMixIn: self.active_children = [] self.active_children.append(pid) self.close_request(request) + return else: # Child process. # This must never return, hence os._exit()! -- cgit v0.12 From 656ce500a641a30d436da7a9c29816cedbc0e093 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 26 May 2011 00:24:38 +0800 Subject: News entry for issue11109. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 36a3c9d..9ecd4a7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #11109 - New service_action method for BaseServer, used by + ForkingMixin class for cleanup. Initial Patch by Justin Wark. + - Issue #10818: Remove the Tk GUI and the serve() function of the pydoc module, pydoc -g has been deprecated in Python 3.2 and it has a new enhanced web server. -- cgit v0.12 From fd8211372dc61374dd0280385285b64f190df28b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 22:01:33 +0200 Subject: Fast path for IncrementalNewlineDecoder.decode() in io.TextIOWrapper.read(-1) Copy/paste code from textiowrapper_read_chunk(). --- Modules/_io/textio.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 35bd922..70d062b 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1513,8 +1513,13 @@ textiowrapper_read(textio *self, PyObject *args) PyObject *decoded; if (bytes == NULL) goto fail; - decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode, - bytes, Py_True, NULL); + + if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) + decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, + bytes, 1); + else + decoded = PyObject_CallMethodObjArgs( + self->decoder, _PyIO_str_decode, bytes, Py_True, NULL); Py_DECREF(bytes); if (decoded == NULL) goto fail; -- cgit v0.12 From b79f28ccbd0cde0580a8d7198ac62e97e7cfb4c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 22:09:03 +0200 Subject: Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError if the file is closed. --- Lib/test/test_io.py | 2 ++ Misc/NEWS | 3 +++ Modules/_io/fileio.c | 2 ++ 3 files changed, 7 insertions(+) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 1ec6f93..109c82d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2467,6 +2467,8 @@ class MiscIOTest(unittest.TestCase): self.assertRaises(ValueError, f.read) if hasattr(f, "read1"): self.assertRaises(ValueError, f.read1, 1024) + if hasattr(f, "readall"): + self.assertRaises(ValueError, f.readall) if hasattr(f, "readinto"): self.assertRaises(ValueError, f.readinto, bytearray(1024)) self.assertRaises(ValueError, f.readline) diff --git a/Misc/NEWS b/Misc/NEWS index 7862e21..fd8f160 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -75,6 +75,9 @@ Core and Builtins Library ------- +- Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError + if the file is closed. + - Issue #12100: Don't reset incremental encoders of CJK codecs at each call to their encode() method anymore, but continue to call the reset() method if the final argument is True. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 473919b..0fce1a3 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -536,6 +536,8 @@ fileio_readall(fileio *self) Py_ssize_t total = 0; int n; + if (self->fd < 0) + return err_closed(); if (!_PyVerify_fd(self->fd)) return PyErr_SetFromErrno(PyExc_IOError); -- cgit v0.12 From a80987f20d0c73532127e1c3f69f7983c5c443d2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 25 May 2011 22:47:16 +0200 Subject: Issue #12175: RawIOBase.readall() now returns None if read() returns None. --- Lib/_pyio.py | 6 +++++- Lib/test/test_io.py | 7 +++++-- Misc/NEWS | 2 ++ Modules/_io/iobase.c | 8 ++++++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index fa00eb4..5474a4e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -556,7 +556,11 @@ class RawIOBase(IOBase): if not data: break res += data - return bytes(res) + if res: + return bytes(res) + else: + # b'' or None + return data def readinto(self, b): """Read up to len(b) bytes into b. diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 109c82d..5333bb6 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -790,14 +790,17 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): # Inject some None's in there to simulate EWOULDBLOCK rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) bufio = self.tp(rawio) - self.assertEqual(b"abcd", bufio.read(6)) self.assertEqual(b"e", bufio.read(1)) self.assertEqual(b"fg", bufio.read()) self.assertEqual(b"", bufio.peek(1)) - self.assertTrue(None is bufio.read()) + self.assertIsNone(bufio.read()) self.assertEqual(b"", bufio.read()) + rawio = self.MockRawIO((b"a", None, None)) + self.assertEqual(b"a", rawio.readall()) + self.assertIsNone(rawio.readall()) + def test_read_past_eof(self): rawio = self.MockRawIO((b"abc", b"d", b"efg")) bufio = self.tp(rawio) diff --git a/Misc/NEWS b/Misc/NEWS index fd8f160..015018c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -75,6 +75,8 @@ Core and Builtins Library ------- +- Issue #12175: RawIOBase.readall() now returns None if read() returns None. + - Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError if the file is closed. diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 84b560a..316321a 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -814,6 +814,14 @@ rawiobase_readall(PyObject *self, PyObject *args) Py_DECREF(chunks); return NULL; } + if (data == Py_None) { + if (PyList_GET_SIZE(chunks) == 0) { + Py_DECREF(chunks); + return data; + } + Py_DECREF(data); + break; + } if (!PyBytes_Check(data)) { Py_DECREF(chunks); Py_DECREF(data); -- cgit v0.12 From 5eb555990a5f507d417d0b37d9e4a88a3c161800 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Wed, 25 May 2011 23:46:09 +0200 Subject: Issue #12180: Fixed a few remaining errors in test_packaging when no threading. --- Lib/packaging/tests/support.py | 9 +++++++++ Lib/packaging/tests/test_install.py | 5 +++-- Lib/packaging/tests/test_pypi_dist.py | 5 +++-- Lib/packaging/tests/test_pypi_simple.py | 24 +++++++++++++++++++++--- Misc/NEWS | 3 +++ 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py index 6d60b9e..66b5583 100644 --- a/Lib/packaging/tests/support.py +++ b/Lib/packaging/tests/support.py @@ -253,6 +253,15 @@ def create_distribution(configfiles=()): return d +def fake_dec(*args, **kw): + """Fake decorator""" + def _wrap(func): + def __wrap(*args, **kw): + return func(*args, **kw) + return __wrap + return _wrap + + try: from test.support import skip_unless_symlink except ImportError: diff --git a/Lib/packaging/tests/test_install.py b/Lib/packaging/tests/test_install.py index 01c3dcf..c0924bf 100644 --- a/Lib/packaging/tests/test_install.py +++ b/Lib/packaging/tests/test_install.py @@ -6,13 +6,14 @@ from packaging import install from packaging.pypi.xmlrpc import Client from packaging.metadata import Metadata -from packaging.tests.support import LoggingCatcher, TempdirManager, unittest +from packaging.tests.support import (LoggingCatcher, TempdirManager, unittest, + fake_dec) try: import threading from packaging.tests.pypi_server import use_xmlrpc_server except ImportError: threading = None - use_xmlrpc_server = None + use_xmlrpc_server = fake_dec class InstalledDist: diff --git a/Lib/packaging/tests/test_pypi_dist.py b/Lib/packaging/tests/test_pypi_dist.py index b7f4e98..0c88c9b 100644 --- a/Lib/packaging/tests/test_pypi_dist.py +++ b/Lib/packaging/tests/test_pypi_dist.py @@ -7,12 +7,13 @@ from packaging.pypi.dist import (ReleaseInfo, ReleasesList, DistInfo, from packaging.pypi.errors import HashDoesNotMatch, UnsupportedHashName from packaging.tests import unittest -from packaging.tests.support import TempdirManager, requires_zlib +from packaging.tests.support import TempdirManager, requires_zlib, fake_dec try: import threading from packaging.tests.pypi_server import use_pypi_server except ImportError: - threading = use_pypi_server = None + threading = None + use_pypi_server = fake_dec def Dist(*args, **kwargs): diff --git a/Lib/packaging/tests/test_pypi_simple.py b/Lib/packaging/tests/test_pypi_simple.py index d50e3f4..bd50d01 100644 --- a/Lib/packaging/tests/test_pypi_simple.py +++ b/Lib/packaging/tests/test_pypi_simple.py @@ -10,9 +10,19 @@ import urllib.request from packaging.pypi.simple import Crawler from packaging.tests import unittest -from packaging.tests.support import TempdirManager, LoggingCatcher -from packaging.tests.pypi_server import (use_pypi_server, PyPIServer, - PYPI_DEFAULT_STATIC_PATH) +from packaging.tests.support import (TempdirManager, LoggingCatcher, + fake_dec) + +try: + import _thread + from packaging.tests.pypi_server import (use_pypi_server, PyPIServer, + PYPI_DEFAULT_STATIC_PATH) +except ImportError: + _thread = None + use_pypi_server = fake_dec + PYPI_DEFAULT_STATIC_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'pypiserver') + class SimpleCrawlerTestCase(TempdirManager, @@ -28,6 +38,7 @@ class SimpleCrawlerTestCase(TempdirManager, return Crawler(server.full_address + base_url, *args, **kwargs) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server() def test_bad_urls(self, server): crawler = Crawler() @@ -84,6 +95,7 @@ class SimpleCrawlerTestCase(TempdirManager, 'http://www.famfamfam.com/">') crawler._process_url(url, page) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server("test_found_links") def test_found_links(self, server): # Browse the index, asking for a specified release version @@ -139,6 +151,7 @@ class SimpleCrawlerTestCase(TempdirManager, self.assertTrue( crawler._is_browsable("http://pypi.example.org/a/path")) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server("with_externals") def test_follow_externals(self, server): # Include external pages @@ -149,6 +162,7 @@ class SimpleCrawlerTestCase(TempdirManager, self.assertIn(server.full_address + "/external/external.html", crawler._processed_urls) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server("with_real_externals") def test_restrict_hosts(self, server): # Only use a list of allowed hosts is possible @@ -159,6 +173,7 @@ class SimpleCrawlerTestCase(TempdirManager, self.assertNotIn(server.full_address + "/external/external.html", crawler._processed_urls) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server(static_filesystem_paths=["with_externals"], static_uri_paths=["simple", "external"]) def test_links_priority(self, server): @@ -192,6 +207,7 @@ class SimpleCrawlerTestCase(TempdirManager, releases[0].dists['sdist'].url['hashval']) self.assertEqual('md5', releases[0].dists['sdist'].url['hashname']) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server(static_filesystem_paths=["with_norel_links"], static_uri_paths=["simple", "external"]) def test_not_scan_all_links(self, server): @@ -217,6 +233,7 @@ class SimpleCrawlerTestCase(TempdirManager, self.assertIn("%s/foobar-2.0.tar.gz" % server.full_address, crawler._processed_urls) # linked from external homepage (rel) + @unittest.skipIf(_thread is None, 'needs threads') def test_uses_mirrors(self): # When the main repository seems down, try using the given mirrors""" server = PyPIServer("foo_bar_baz") @@ -314,6 +331,7 @@ class SimpleCrawlerTestCase(TempdirManager, self.assertIn('http://example.org/some/simpleurl', found_links) self.assertIn('http://example.org/some/download', found_links) + @unittest.skipIf(_thread is None, 'needs threads') @use_pypi_server("project_list") def test_search_projects(self, server): # we can search the index for some projects, on their names diff --git a/Misc/NEWS b/Misc/NEWS index 60209bf..b1d26dd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #12180: Fixed a few remaining errors in test_packaging when no + threading. + - Issue #12175: RawIOBase.readall() now returns None if read() returns None. - Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError -- cgit v0.12 From e9d44ccb2279a49a69277d38e956731675f1b556 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 00:16:44 +0200 Subject: Issue #12175: FileIO.readall() now only reads the file position and size once. --- Misc/NEWS | 5 ++++- Modules/_io/fileio.c | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index b1d26dd..d1db0ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,7 +161,10 @@ Core and Builtins Library ------- -- Issue #12180: Fixed a few remaining errors in test_packaging when no +- Issue #12175: FileIO.readall() now only reads the file position and size + once. + +- Issue #12180: Fixed a few remaining errors in test_packaging when no threading. - Issue #12175: RawIOBase.readall() now returns None if read() returns None. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 141b6de..b3b9e44 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -547,14 +547,14 @@ fileio_readinto(fileio *self, PyObject *args) } static size_t -new_buffersize(fileio *self, size_t currentsize) +new_buffersize(fileio *self, size_t currentsize +#ifdef HAVE_FSTAT + , off_t pos, off_t end +#endif + ) { #ifdef HAVE_FSTAT - off_t pos, end; - struct stat st; - if (fstat(self->fd, &st) == 0) { - end = st.st_size; - pos = lseek(self->fd, 0L, SEEK_CUR); + if (end != (off_t)-1) { /* Files claiming a size smaller than SMALLCHUNK may actually be streaming pseudo-files. In this case, we apply the more aggressive algorithm below. @@ -579,9 +579,14 @@ new_buffersize(fileio *self, size_t currentsize) static PyObject * fileio_readall(fileio *self) { +#ifdef HAVE_FSTAT + struct stat st; + off_t pos, end; +#endif PyObject *result; Py_ssize_t total = 0; int n; + size_t newsize; if (self->fd < 0) return err_closed(); @@ -592,8 +597,23 @@ fileio_readall(fileio *self) if (result == NULL) return NULL; +#ifdef HAVE_FSTAT +#if defined(MS_WIN64) || defined(MS_WINDOWS) + pos = _lseeki64(self->fd, 0L, SEEK_CUR); +#else + pos = lseek(self->fd, 0L, SEEK_CUR); +#endif + if (fstat(self->fd, &st) == 0) + end = st.st_size; + else + end = (off_t)-1; +#endif while (1) { - size_t newsize = new_buffersize(self, total); +#ifdef HAVE_FSTAT + newsize = new_buffersize(self, total, pos, end); +#else + newsize = new_buffersize(self, total); +#endif if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { PyErr_SetString(PyExc_OverflowError, "unbounded read returned more bytes " @@ -632,6 +652,9 @@ fileio_readall(fileio *self) return NULL; } total += n; +#ifdef HAVE_FSTAT + pos += n; +#endif } if (PyBytes_GET_SIZE(result) > total) { -- cgit v0.12 From b57f108b03df332d276a224c92e32cee13a042f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 00:19:38 +0200 Subject: Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. --- Lib/_pyio.py | 6 ++++++ Misc/NEWS | 2 ++ Modules/_io/_iomodule.c | 3 +++ Modules/_io/_iomodule.h | 1 + Modules/_io/bufferedio.c | 47 ++++++++++++++++++++++++++++++++++++----------- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 74047bf..265edab 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -944,6 +944,12 @@ class BufferedReader(_BufferedIOMixin): # Special case for when the number of bytes to read is unspecified. if n is None or n == -1: self._reset_read_buf() + if hasattr(self.raw, 'readall'): + chunk = self.raw.readall() + if chunk is None: + return buf[pos:] or None + else: + return buf[pos:] + chunk chunks = [buf[pos:]] # Strip the consumed bytes. current_size = 0 while True: diff --git a/Misc/NEWS b/Misc/NEWS index d1db0ab..b7f3907 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,8 @@ Core and Builtins Library ------- +- Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. + - Issue #12175: FileIO.readall() now only reads the file position and size once. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 44bdac6..6f5bd48 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -36,6 +36,7 @@ PyObject *_PyIO_str_nl; PyObject *_PyIO_str_read; PyObject *_PyIO_str_read1; PyObject *_PyIO_str_readable; +PyObject *_PyIO_str_readall; PyObject *_PyIO_str_readinto; PyObject *_PyIO_str_readline; PyObject *_PyIO_str_reset; @@ -767,6 +768,8 @@ PyInit__io(void) goto fail; if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable"))) goto fail; + if (!(_PyIO_str_readall = PyUnicode_InternFromString("readall"))) + goto fail; if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto"))) goto fail; if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline"))) diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 925e4f2..9174bdd 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -155,6 +155,7 @@ extern PyObject *_PyIO_str_nl; extern PyObject *_PyIO_str_read; extern PyObject *_PyIO_str_read1; extern PyObject *_PyIO_str_readable; +extern PyObject *_PyIO_str_readall; extern PyObject *_PyIO_str_readinto; extern PyObject *_PyIO_str_readline; extern PyObject *_PyIO_str_reset; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 3b8b7e9..63ae1cb 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1407,32 +1407,57 @@ static PyObject * _bufferedreader_read_all(buffered *self) { Py_ssize_t current_size; - PyObject *res, *data = NULL; - PyObject *chunks = PyList_New(0); - - if (chunks == NULL) - return NULL; + PyObject *res, *data = NULL, *chunk, *chunks; /* First copy what we have in the current buffer. */ current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (current_size) { data = PyBytes_FromStringAndSize( self->buffer + self->pos, current_size); - if (data == NULL) { - Py_DECREF(chunks); + if (data == NULL) return NULL; - } } _bufferedreader_reset_buf(self); /* We're going past the buffer's bounds, flush it */ if (self->writable) { res = _bufferedwriter_flush_unlocked(self, 1); - if (res == NULL) { - Py_DECREF(chunks); + if (res == NULL) return NULL; - } Py_CLEAR(res); } + + if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) { + chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL); + if (chunk == NULL) + return NULL; + if (chunk != Py_None && !PyBytes_Check(chunk)) { + Py_XDECREF(data); + Py_DECREF(chunk); + PyErr_SetString(PyExc_TypeError, "readall() should return bytes"); + return NULL; + } + if (chunk == Py_None) { + if (current_size == 0) + return chunk; + else { + Py_DECREF(chunk); + return data; + } + } + else if (current_size) { + PyBytes_Concat(&data, chunk); + Py_DECREF(chunk); + if (data == NULL) + return NULL; + return data; + } else + return chunk; + } + + chunks = PyList_New(0); + if (chunks == NULL) + return NULL; + while (1) { if (data) { if (PyList_Append(chunks, data) < 0) { -- cgit v0.12 From f2c6db5faca7ff04bb4d269780b1dd625bc54606 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Thu, 26 May 2011 00:37:45 +0200 Subject: Issue #12090: backport 79fcd71d0356 --- Modules/_testembed.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Modules/_testembed.c b/Modules/_testembed.c index 0df5ede..51b439f 100644 --- a/Modules/_testembed.c +++ b/Modules/_testembed.c @@ -17,7 +17,9 @@ void print_subinterp(void) int main(int argc, char *argv[]) { PyThreadState *mainstate, *substate; +#ifdef WITH_THREAD PyGILState_STATE gilstate; +#endif int i, j; for (i=0; i<3; i++) { @@ -28,10 +30,12 @@ int main(int argc, char *argv[]) Py_Initialize(); mainstate = PyThreadState_Get(); +#ifdef WITH_THREAD PyEval_InitThreads(); PyEval_ReleaseThread(mainstate); gilstate = PyGILState_Ensure(); +#endif print_subinterp(); PyThreadState_Swap(NULL); @@ -43,7 +47,9 @@ int main(int argc, char *argv[]) PyThreadState_Swap(mainstate); print_subinterp(); +#ifdef WITH_THREAD PyGILState_Release(gilstate); +#endif PyEval_RestoreThread(mainstate); Py_Finalize(); -- cgit v0.12 From 25095b2be655cbe88a05e99f6956aa29cf6207d9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:47:08 +0200 Subject: Remove useless assignments Warnings found by the the Clang Static Analyzer. --- Modules/faulthandler.c | 9 ++++----- Python/import.c | 4 ---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 83c47ce..46c2c42 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1005,9 +1005,10 @@ static int faulthandler_env_options(void) { PyObject *xoptions, *key, *module, *res; - int enable; if (!Py_GETENV("PYTHONFAULTHANDLER")) { + int has_key; + xoptions = PySys_GetXOptions(); if (xoptions == NULL) return -1; @@ -1016,13 +1017,11 @@ faulthandler_env_options(void) if (key == NULL) return -1; - enable = PyDict_Contains(xoptions, key); + has_key = PyDict_Contains(xoptions, key); Py_DECREF(key); - if (!enable) + if (!has_key) return 0; } - else - enable = 1; module = PyImport_ImportModule("faulthandler"); if (module == NULL) { diff --git a/Python/import.c b/Python/import.c index bfb976c..1f28d22 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1733,7 +1733,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_UNICODE buf[MAXPATHLEN+1]; Py_ssize_t buflen = MAXPATHLEN+1; PyObject *path_unicode, *filename; - const Py_UNICODE *base; Py_ssize_t len; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; @@ -1751,7 +1750,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - base = PyUnicode_AS_UNICODE(path_unicode); len = PyUnicode_GET_SIZE(path_unicode); if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { Py_DECREF(path_unicode); @@ -2275,12 +2273,10 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) static int find_init_module(PyObject *directory) { - size_t len; struct stat statbuf; PyObject *filename; int match; - len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; -- cgit v0.12 From 97e561ef24b6270d0000e21be3afb994caefcd8f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:53:47 +0200 Subject: Avoid useless "++" at the end of functions Warnings found by the Clang Static Analyzer. --- Objects/setobject.c | 4 ++-- Objects/unicodeobject.c | 2 +- Python/compile.c | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 48edad8..22243ea 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -612,9 +612,9 @@ set_repr(PySetObject *so) *u++ = '{'; /* Omit the brackets from the listrepr */ Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, - PyUnicode_GET_SIZE(listrepr)-2); + newsize-2); u += newsize-2; - *u++ = '}'; + *u = '}'; } Py_DECREF(listrepr); if (Py_TYPE(so) != &PySet_Type) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4361908..309159c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6474,7 +6474,7 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, } } /* 0-terminate the output string */ - *output++ = '\0'; + *output = '\0'; Py_XDECREF(exc); Py_XDECREF(errorHandler); return 0; diff --git a/Python/compile.c b/Python/compile.c index 53f5a12..d195967 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3747,11 +3747,11 @@ assemble_lnotab(struct assembler *a, struct instr *i) a->a_lnotab_off += 2; if (d_bytecode) { *lnotab++ = d_bytecode; - *lnotab++ = d_lineno; + *lnotab = d_lineno; } else { /* First line of a block; def stmt, etc. */ *lnotab++ = 0; - *lnotab++ = d_lineno; + *lnotab = d_lineno; } a->a_lineno = i->i_lineno; a->a_lineno_off = a->a_offset; @@ -3796,7 +3796,7 @@ assemble_emit(struct assembler *a, struct instr *i) if (i->i_hasarg) { assert(size == 3 || size == 6); *code++ = arg & 0xff; - *code++ = arg >> 8; + *code = arg >> 8; } return 1; } -- cgit v0.12 From 9a2261a3729805ca5e5062c37767df80ced9e5df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:59:41 +0200 Subject: zipimport: initialize fullpath to NULL In some cases, fullpath value is used whereas fullpath was not always initialized. Warning found by the Clang Static Analyzer. --- Modules/zipimport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/zipimport.c b/Modules/zipimport.c index de89a76..a83bf8b 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1196,7 +1196,7 @@ get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath) { PyObject *code = NULL, *toc_entry, *subname; - PyObject *path, *fullpath; + PyObject *path, *fullpath = NULL; struct st_zip_searchorder *zso; subname = get_subname(fullname); -- cgit v0.12 From 77af1729143a72fe96ba7c85b34ae74b104e0119 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 14:05:59 +0200 Subject: socket.sendto(): exit directly after setting the error Dummy change to avoid a false positive with the Clang Static Analyzer. --- Modules/socketmodule.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8107b98..d18b5b1 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2782,6 +2782,7 @@ sock_sendto(PySocketSockObject *s, PyObject *args) PyErr_Format(PyExc_TypeError, "sendto() takes 2 or 3 arguments (%d given)", arglen); + return NULL; } if (PyErr_Occurred()) return NULL; @@ -3144,7 +3145,7 @@ socket_gethostname(PyObject *self, PyObject *unused) } return PyErr_SetExcFromWindowsErr(PyExc_WindowsError, GetLastError()); } - return PyUnicode_FromUnicode(buf, size); + return PyUnicode_FromUnicode(buf, size); #else char buf[1024]; int res; @@ -4038,7 +4039,7 @@ socket_inet_ntop(PyObject *self, PyObject *args) static PyObject * socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) { - static char* kwnames[] = {"host", "port", "family", "type", "proto", + static char* kwnames[] = {"host", "port", "family", "type", "proto", "flags", 0}; struct addrinfo hints, *res; struct addrinfo *res0 = NULL; @@ -4053,7 +4054,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) family = socktype = protocol = flags = 0; family = AF_UNSPEC; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", kwnames, &hobj, &pobj, &family, &socktype, &protocol, &flags)) { return NULL; @@ -4289,7 +4290,7 @@ socket_if_nameindex(PyObject *self, PyObject *arg) PyObject *list; int i; struct if_nameindex *ni; - + ni = if_nameindex(); if (ni == NULL) { PyErr_SetFromErrno(socket_error); -- cgit v0.12 From 5572ba7e15b462098aac2def8e85e59b055689c3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 14:10:08 +0200 Subject: _posixsubprocess.c: don't redefine _GNU_SOURCE if it's already defined --- Modules/_posixsubprocess.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index bf10cbb..af0d971 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1,7 +1,7 @@ /* Authors: Gregory P. Smith & Jeffrey Yasskin */ #include "Python.h" -#ifdef HAVE_PIPE2 -#define _GNU_SOURCE +#if defined(HAVE_PIPE2) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE #endif #include #include -- cgit v0.12 From a1a807b6efdbdaa0af7da4aff46a4a205d87e8f9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 14:24:30 +0200 Subject: set_repr(): handle correctly PyUnicode_FromUnicode() error (MemoryError) Bug found by the Clang Static Analyzer. --- Objects/setobject.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 22243ea..ebfddb3 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -607,16 +607,18 @@ set_repr(PySetObject *so) goto done; newsize = PyUnicode_GET_SIZE(listrepr); result = PyUnicode_FromUnicode(NULL, newsize); - if (result) { - u = PyUnicode_AS_UNICODE(result); - *u++ = '{'; - /* Omit the brackets from the listrepr */ - Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, - newsize-2); - u += newsize-2; - *u = '}'; - } + if (result == NULL) + goto done; + + u = PyUnicode_AS_UNICODE(result); + *u++ = '{'; + /* Omit the brackets from the listrepr */ + Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, + newsize-2); + u += newsize-2; + *u = '}'; Py_DECREF(listrepr); + if (Py_TYPE(so) != &PySet_Type) { PyObject *tmp = PyUnicode_FromFormat("%s(%U)", Py_TYPE(so)->tp_name, -- cgit v0.12 From 13b21bd749aef0f6b18cec0e7e5d1d3e643f106a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 14:25:13 +0200 Subject: print_exception(): handle correctly PyObject_GetAttrString() failure Bug found by the Clang Static Analyzer. --- Python/pythonrun.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b55dc5b..232d7be 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1593,7 +1593,7 @@ print_exception(PyObject *f, PyObject *value) moduleName = PyObject_GetAttrString(type, "__module__"); if (moduleName == NULL || !PyUnicode_Check(moduleName)) { - Py_DECREF(moduleName); + Py_XDECREF(moduleName); err = PyFile_WriteString("", f); } else { -- cgit v0.12 From 92236e5651ade38b31281ed92e2c2846f8ccd0d8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 14:25:54 +0200 Subject: SystemExit_init(): avoid an useless test Make silent a false positive of the Clang Static Analyzer. --- Objects/exceptions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 0106ba3..fb7864f 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -499,7 +499,7 @@ SystemExit_init(PySystemExitObject *self, PyObject *args, PyObject *kwds) Py_CLEAR(self->code); if (size == 1) self->code = PyTuple_GET_ITEM(args, 0); - else if (size > 1) + else /* size > 1 */ self->code = args; Py_INCREF(self->code); return 0; -- cgit v0.12 From 42fc33a5b6c90e073596e16b32d49e394129d355 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 26 May 2011 09:59:17 -0500 Subject: add ack from 2.7 --- Misc/ACKS | 1 + 1 file changed, 1 insertion(+) diff --git a/Misc/ACKS b/Misc/ACKS index 506e5c9..68e7eef 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -417,6 +417,7 @@ Eric Huss Jeremy Hylton Gerhard Häring Fredrik Håård +Catalin Iacob Mihai Ibanescu Lars Immisch Bobby Impollonia -- cgit v0.12 From 08ae8bb9374b762e53f8e569e293bc24bb081fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Thu, 26 May 2011 17:06:47 +0200 Subject: Move test_packaging news entries to the tests section --- Misc/NEWS | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index b7f3907..8735965 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -166,9 +166,6 @@ Library - Issue #12175: FileIO.readall() now only reads the file position and size once. -- Issue #12180: Fixed a few remaining errors in test_packaging when no - threading. - - Issue #12175: RawIOBase.readall() now returns None if read() returns None. - Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError @@ -213,9 +210,6 @@ Library - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore to be able to unload the module. -- Issue #12120, #12119: skip a test in packaging and distutils - if sys.dont_write_bytecode is set to True. - - Issue #12065: connect_ex() on an SSL socket now returns the original errno when the socket's timeout expires (it used to return None). @@ -749,6 +743,12 @@ Extension Modules Tests ----- +- Issue #12180: Fixed a few remaining errors in test_packaging when no + threading. + +- Issue #12120, #12119: skip a test in packaging and distutils + if sys.dont_write_bytecode is set to True. + - Issue #12096: Fix a race condition in test_threading.test_waitfor(). Patch written by Charles-François Natali. -- cgit v0.12 From c556e10b94541d7bf20e908f8eca78e7f63fc28c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 26 May 2011 17:49:57 -0500 Subject: fix typo --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8735965..f3143ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -846,7 +846,7 @@ C-API Mercurial. - Issue #12173: The first argument of PyImport_ImportModuleLevel is now `const - char *` instead of `char *1`. + char *` instead of `char *`. Documentation ------------- -- cgit v0.12 From 98fe1a0c3bacdc51071d960d8d76b3b9f5b0d8c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 27 May 2011 01:51:18 +0200 Subject: Issue #8796: codecs.open() calls the builtin open() function instead of using StreamReaderWriter. Deprecate StreamReader, StreamWriter, StreamReaderWriter, StreamRecoder and EncodedFile() of the codec module. Use the builtin open() function or io.TextIOWrapper instead. --- Doc/library/codecs.rst | 25 ++++++++ Lib/codecs.py | 25 ++++---- Lib/test/test_codecs.py | 152 +++++++++++++++++++++++++++++++++--------------- Misc/NEWS | 5 ++ 4 files changed, 148 insertions(+), 59 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4d5058e..b58e410 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -85,6 +85,9 @@ It defines the following functions: In case a search function cannot find a given encoding, it should return ``None``. + .. deprecated:: 3.3 + *streamreader* and *streamwriter* attributes are now deprecated. + .. function:: lookup(encoding) @@ -139,6 +142,8 @@ functions which use :func:`lookup` for the codec lookup: Raises a :exc:`LookupError` in case the encoding cannot be found. + .. deprecated:: 3.3 + .. function:: getwriter(encoding) @@ -147,6 +152,8 @@ functions which use :func:`lookup` for the codec lookup: Raises a :exc:`LookupError` in case the encoding cannot be found. + .. deprecated:: 3.3 + .. function:: register_error(name, error_handler) @@ -217,6 +224,11 @@ utility functions: .. note:: + This function is kept for backward compatibility with Python 2, the + builtin :func:`open` function should be used instead. + + .. note:: + The wrapped version's methods will accept and return strings only. Bytes arguments will be rejected. @@ -251,6 +263,8 @@ utility functions: ``'strict'``, which causes :exc:`ValueError` to be raised in case an encoding error occurs. + .. deprecated:: 3.3 + .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) @@ -563,6 +577,9 @@ The :class:`StreamWriter` class is a subclass of :class:`Codec` and defines the following methods which every stream writer must define in order to be compatible with the Python codec registry. +.. deprecated:: 3.3 + Use the builtin the :class:`io.TextIOWrapper` class. + .. class:: StreamWriter(stream[, errors]) @@ -628,6 +645,9 @@ The :class:`StreamReader` class is a subclass of :class:`Codec` and defines the following methods which every stream reader must define in order to be compatible with the Python codec registry. +.. deprecated:: 3.3 + Use the builtin the :class:`io.TextIOWrapper` class. + .. class:: StreamReader(stream[, errors]) @@ -728,6 +748,9 @@ and write modes. The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. +.. deprecated:: 3.3 + Use the :class:`io.TextIOWrapper` class. + .. class:: StreamReaderWriter(stream, Reader, Writer, errors) @@ -752,6 +775,8 @@ which is sometimes useful when dealing with different encoding environments. The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. +.. deprecated:: 3.3 + .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors) diff --git a/Lib/codecs.py b/Lib/codecs.py index b150d64..ec7879f 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -345,6 +345,8 @@ class StreamWriter(Codec): The set of allowed parameter values can be extended via register_error. """ + import warnings + warnings.warn('use io.TextIOWrapper', DeprecationWarning, stacklevel=2) self.stream = stream self.errors = errors @@ -416,6 +418,8 @@ class StreamReader(Codec): The set of allowed parameter values can be extended via register_error. """ + import warnings + warnings.warn('use io.TextIOWrapper', DeprecationWarning, stacklevel=2) self.stream = stream self.errors = errors self.bytebuffer = b"" @@ -846,7 +850,7 @@ class StreamRecoder: ### Shortcuts -def open(filename, mode='rb', encoding=None, errors='strict', buffering=1): +def open(filename, mode='r', encoding=None, errors=None, buffering=1): """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -877,18 +881,13 @@ def open(filename, mode='rb', encoding=None, errors='strict', buffering=1): parameter. """ - if encoding is not None and \ - 'b' not in mode: - # Force opening of the file in binary mode - mode = mode + 'b' - file = builtins.open(filename, mode, buffering) - if encoding is None: - return file - info = lookup(encoding) - srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) - # Add attributes to simplify introspection - srw.encoding = encoding - return srw + if encoding is not None: + return builtins.open(filename, mode, buffering, + encoding, errors, newline='') + else: + if 'b' not in mode: + mode = mode + 'b' + return builtins.open(filename, mode, buffering, encoding, errors) def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c0450e7..1f46560 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,7 +1,10 @@ from test import support -import unittest +import _testcapi import codecs -import sys, _testcapi, io +import io +import sys +import unittest +import warnings class Queue(object): """ @@ -63,7 +66,9 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): # the StreamReader and check that the results equal the appropriate # entries from partialresults. q = Queue(b"") - r = codecs.getreader(self.encoding)(q) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + r = codecs.getreader(self.encoding)(q) result = "" for (c, partialresult) in zip(input.encode(self.encoding), partialresults): q.write(bytes([c])) @@ -106,7 +111,9 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): return codecs.getreader(self.encoding)(stream) def readalllines(input, keepends=True, size=None): - reader = getreader(input) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = getreader(input) lines = [] while True: line = reader.readline(size=size, keepends=keepends) @@ -215,14 +222,18 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): ' \r\n', ] stream = io.BytesIO("".join(s).encode(self.encoding)) - reader = codecs.getreader(self.encoding)(stream) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(self.encoding)(stream) for (i, line) in enumerate(reader): self.assertEqual(line, s[i]) def test_readlinequeue(self): q = Queue(b"") - writer = codecs.getwriter(self.encoding)(q) - reader = codecs.getreader(self.encoding)(q) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + writer = codecs.getwriter(self.encoding)(q) + reader = codecs.getreader(self.encoding)(q) # No lineends writer.write("foo\r") @@ -253,7 +264,9 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): s = (s1+s2+s3).encode(self.encoding) stream = io.BytesIO(s) - reader = codecs.getreader(self.encoding)(stream) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(self.encoding)(stream) self.assertEqual(reader.readline(), s1) self.assertEqual(reader.readline(), s2) self.assertEqual(reader.readline(), s3) @@ -268,7 +281,9 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): s = (s1+s2+s3+s4+s5).encode(self.encoding) stream = io.BytesIO(s) - reader = codecs.getreader(self.encoding)(stream) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(self.encoding)(stream) self.assertEqual(reader.readline(), s1) self.assertEqual(reader.readline(), s2) self.assertEqual(reader.readline(), s3) @@ -290,7 +305,9 @@ class UTF32Test(ReadTest): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream s = io.BytesIO() - f = writer(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = writer(s) f.write("spam") f.write("spam") d = s.getvalue() @@ -298,16 +315,22 @@ class UTF32Test(ReadTest): self.assertTrue(d == self.spamle or d == self.spambe) # try to read it back s = io.BytesIO(d) - f = reader(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = reader(s) self.assertEqual(f.read(), "spamspam") def test_badbom(self): s = io.BytesIO(4*b"\xff") - f = codecs.getreader(self.encoding)(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) s = io.BytesIO(8*b"\xff") - f = codecs.getreader(self.encoding)(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) def test_partial(self): @@ -454,7 +477,9 @@ class UTF16Test(ReadTest): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream s = io.BytesIO() - f = writer(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = writer(s) f.write("spam") f.write("spam") d = s.getvalue() @@ -462,16 +487,22 @@ class UTF16Test(ReadTest): self.assertTrue(d == self.spamle or d == self.spambe) # try to read it back s = io.BytesIO(d) - f = reader(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = reader(s) self.assertEqual(f.read(), "spamspam") def test_badbom(self): s = io.BytesIO(b"\xff\xff") - f = codecs.getreader(self.encoding)(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") - f = codecs.getreader(self.encoding)(s) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) def test_partial(self): @@ -517,7 +548,8 @@ class UTF16Test(ReadTest): self.addCleanup(support.unlink, support.TESTFN) with open(support.TESTFN, 'wb') as fp: fp.write(s) - with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader: + with codecs.open(support.TESTFN, 'U', + encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) class UTF16LETest(ReadTest): @@ -705,7 +737,9 @@ class UTF8SigTest(ReadTest): reader = codecs.getreader("utf-8-sig") for sizehint in [None] + list(range(1, 11)) + \ [64, 128, 256, 512, 1024]: - istream = reader(io.BytesIO(bytestring)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + istream = reader(io.BytesIO(bytestring)) ostream = io.StringIO() while 1: if sizehint is not None: @@ -727,7 +761,9 @@ class UTF8SigTest(ReadTest): reader = codecs.getreader("utf-8-sig") for sizehint in [None] + list(range(1, 11)) + \ [64, 128, 256, 512, 1024]: - istream = reader(io.BytesIO(bytestring)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + istream = reader(io.BytesIO(bytestring)) ostream = io.StringIO() while 1: if sizehint is not None: @@ -749,7 +785,9 @@ class EscapeDecodeTest(unittest.TestCase): class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() - f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8") f2.write("a") f2.close() # Python used to crash on this at exit because of a refcount @@ -1126,7 +1164,9 @@ class IDNACodecTest(unittest.TestCase): self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") def test_stream(self): - r = codecs.getreader("idna")(io.BytesIO(b"abc")) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + r = codecs.getreader("idna")(io.BytesIO(b"abc")) r.read(3) self.assertEqual(r.read(), "") @@ -1233,18 +1273,24 @@ class CodecsModuleTest(unittest.TestCase): class StreamReaderTest(unittest.TestCase): def setUp(self): - self.reader = codecs.getreader('utf-8') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.reader = codecs.getreader('utf-8') self.stream = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80') def test_readlines(self): - f = self.reader(self.stream) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) class EncodedFileTest(unittest.TestCase): def test_basic(self): f = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80') - ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8') self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae') f = io.BytesIO() @@ -1388,7 +1434,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): if encoding not in broken_unicode_with_streams: # check stream reader/writer q = Queue(b"") - writer = codecs.getwriter(encoding)(q) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + writer = codecs.getwriter(encoding)(q) encodedresult = b"" for c in s: writer.write(c) @@ -1396,7 +1444,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.assertTrue(type(chunk) is bytes, type(chunk)) encodedresult += chunk q = Queue(b"") - reader = codecs.getreader(encoding)(q) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(encoding)(q) decodedresult = "" for c in encodedresult: q.write(bytes([c])) @@ -1470,7 +1520,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): continue if encoding in broken_unicode_with_streams: continue - reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) for t in range(5): # Test that calling seek resets the internal codec state and buffers reader.seek(0, 0) @@ -1539,15 +1591,19 @@ class CharmapTest(unittest.TestCase): class WithStmtTest(unittest.TestCase): def test_encodedfile(self): f = io.BytesIO(b"\xc3\xbc") - with codecs.EncodedFile(f, "latin-1", "utf-8") as ef: - self.assertEqual(ef.read(), b"\xfc") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with codecs.EncodedFile(f, "latin-1", "utf-8") as ef: + self.assertEqual(ef.read(), b"\xfc") def test_streamreaderwriter(self): f = io.BytesIO(b"\xc3\xbc") info = codecs.lookup("utf-8") - with codecs.StreamReaderWriter(f, info.streamreader, - info.streamwriter, 'strict') as srw: - self.assertEqual(srw.read(), "\xfc") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with codecs.StreamReaderWriter(f, info.streamreader, + info.streamwriter, 'strict') as srw: + self.assertEqual(srw.read(), "\xfc") class TypesTest(unittest.TestCase): def test_decode_unicode(self): @@ -1644,15 +1700,15 @@ class BomTest(unittest.TestCase): # (StreamWriter) Check that the BOM is written after a seek(0) with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: - f.writer.write(data[0]) - self.assertNotEqual(f.writer.tell(), 0) - f.writer.seek(0) - f.writer.write(data) + f.write(data[0]) + self.assertNotEqual(f.tell(), 0) + f.seek(0) + f.write(data) f.seek(0) self.assertEqual(f.read(), data) - # Check that the BOM is not written after a seek() at a position - # different than the start + # Check that the BOM is not written after a seek() at a + # position different than the start with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.seek(f.tell()) @@ -1660,12 +1716,12 @@ class BomTest(unittest.TestCase): f.seek(0) self.assertEqual(f.read(), data * 2) - # (StreamWriter) Check that the BOM is not written after a seek() - # at a position different than the start + # (StreamWriter) Check that the BOM is not written after a + # seek() at a position different than the start with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: - f.writer.write(data) - f.writer.seek(f.writer.tell()) - f.writer.write(data) + f.write(data) + f.seek(f.tell()) + f.write(data) f.seek(0) self.assertEqual(f.read(), data * 2) @@ -1704,7 +1760,9 @@ class TransformCodecTest(unittest.TestCase): def test_read(self): for encoding in bytes_transform_encodings: sin = codecs.encode(b"\x80", encoding) - reader = codecs.getreader(encoding)(io.BytesIO(sin)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.read() self.assertEqual(sout, b"\x80") @@ -1713,7 +1771,9 @@ class TransformCodecTest(unittest.TestCase): if encoding in ['uu_codec', 'zlib_codec']: continue sin = codecs.encode(b"\x80", encoding) - reader = codecs.getreader(encoding)(io.BytesIO(sin)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.readline() self.assertEqual(sout, b"\x80") diff --git a/Misc/NEWS b/Misc/NEWS index f3143ab..715539d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,11 @@ Core and Builtins Library ------- +- Issue #8796: codecs.open() calls the builtin open() function instead of using + StreamReaderWriter. Deprecate StreamReader, StreamWriter, StreamReaderWriter, + StreamRecoder and EncodedFile() of the codec module. Use the builtin open() + function or io.TextIOWrapper instead. + - Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. - Issue #12175: FileIO.readall() now only reads the file position and size -- cgit v0.12 From 55b4338874ede31619a097c0b4a271b90e980472 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Fri, 27 May 2011 01:52:15 +0200 Subject: Issue #1625: BZ2File and bz2.decompress() now support multi-stream files. Initial patch by Nir Aides. --- Lib/bz2.py | 47 +++++++++++++------ Lib/test/test_bz2.py | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++- Misc/NEWS | 3 ++ 3 files changed, 161 insertions(+), 15 deletions(-) diff --git a/Lib/bz2.py b/Lib/bz2.py index 8ffeaac..4b25f5d 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -76,6 +76,10 @@ class BZ2File(io.BufferedIOBase): mode = "wb" mode_code = _MODE_WRITE self._compressor = BZ2Compressor() + elif mode in ("a", "ab"): + mode = "ab" + mode_code = _MODE_WRITE + self._compressor = BZ2Compressor() else: raise ValueError("Invalid mode: {!r}".format(mode)) @@ -161,14 +165,25 @@ class BZ2File(io.BufferedIOBase): def _fill_buffer(self): if self._buffer: return True - if self._decompressor.eof: - self._mode = _MODE_READ_EOF - self._size = self._pos - return False - rawblock = self._fp.read(_BUFFER_SIZE) + + if self._decompressor.unused_data: + rawblock = self._decompressor.unused_data + else: + rawblock = self._fp.read(_BUFFER_SIZE) + if not rawblock: - raise EOFError("Compressed file ended before the " - "end-of-stream marker was reached") + if self._decompressor.eof: + self._mode = _MODE_READ_EOF + self._size = self._pos + return False + else: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + + # Continue to next stream. + if self._decompressor.eof: + self._decompressor = BZ2Decompressor() + self._buffer = self._decompressor.decompress(rawblock) return True @@ -384,9 +399,15 @@ def decompress(data): """ if len(data) == 0: return b"" - decomp = BZ2Decompressor() - result = decomp.decompress(data) - if not decomp.eof: - raise ValueError("Compressed data ended before the " - "end-of-stream marker was reached") - return result + + result = b"" + while True: + decomp = BZ2Decompressor() + result += decomp.decompress(data) + if not decomp.eof: + raise ValueError("Compressed data ended before the " + "end-of-stream marker was reached") + if not decomp.unused_data: + return result + # There is unused data left over. Proceed to next stream. + data = decomp.unused_data diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 3567b36..4d66840 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -84,9 +84,9 @@ class BZ2FileTest(BaseTest): else: return self.DATA - def createTempFile(self, crlf=False): + def createTempFile(self, crlf=False, streams=1): with open(self.filename, "wb") as f: - f.write(self.getData(crlf)) + f.write(self.getData(crlf) * streams) def testRead(self): # "Test BZ2File.read()" @@ -95,6 +95,26 @@ class BZ2FileTest(BaseTest): self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT) + def testReadMultiStream(self): + # "Test BZ2File.read() with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + + def testReadMonkeyMultiStream(self): + # "Test BZ2File.read() with a multi stream archive in which stream" + # "end is alined with internal buffer size" + buffer_size = bz2._BUFFER_SIZE + bz2._BUFFER_SIZE = len(self.DATA) + try: + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + finally: + bz2._BUFFER_SIZE = buffer_size + def testRead0(self): # "Test BBZ2File.read(0)" self.createTempFile() @@ -114,6 +134,18 @@ class BZ2FileTest(BaseTest): text += str self.assertEqual(text, self.TEXT) + def testReadChunk10MultiStream(self): + # "Test BZ2File.read() in chunks of 10 bytes with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + text = b'' + while 1: + str = bz2f.read(10) + if not str: + break + text += str + self.assertEqual(text, self.TEXT * 5) + def testRead100(self): # "Test BZ2File.read(100)" self.createTempFile() @@ -151,6 +183,15 @@ class BZ2FileTest(BaseTest): for line in sio.readlines(): self.assertEqual(bz2f.readline(), line) + def testReadLineMultiStream(self): + # "Test BZ2File.readline() with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readline, None) + sio = BytesIO(self.TEXT * 5) + for line in sio.readlines(): + self.assertEqual(bz2f.readline(), line) + def testReadLines(self): # "Test BZ2File.readlines()" self.createTempFile() @@ -159,6 +200,14 @@ class BZ2FileTest(BaseTest): sio = BytesIO(self.TEXT) self.assertEqual(bz2f.readlines(), sio.readlines()) + def testReadLinesMultiStream(self): + # "Test BZ2File.readlines() with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readlines, None) + sio = BytesIO(self.TEXT * 5) + self.assertEqual(bz2f.readlines(), sio.readlines()) + def testIterator(self): # "Test iter(BZ2File)" self.createTempFile() @@ -166,6 +215,13 @@ class BZ2FileTest(BaseTest): sio = BytesIO(self.TEXT) self.assertEqual(list(iter(bz2f)), sio.readlines()) + def testIteratorMultiStream(self): + # "Test iter(BZ2File) with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + sio = BytesIO(self.TEXT * 5) + self.assertEqual(list(iter(bz2f)), sio.readlines()) + def testClosedIteratorDeadlock(self): # "Test that iteration on a closed bz2file releases the lock." # http://bugs.python.org/issue3309 @@ -217,6 +273,17 @@ class BZ2FileTest(BaseTest): self.assertRaises(IOError, bz2f.write, b"a") self.assertRaises(IOError, bz2f.writelines, [b"a"]) + def testAppend(self): + # "Test BZ2File.write()" + with BZ2File(self.filename, "w") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with BZ2File(self.filename, "a") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with open(self.filename, 'rb') as f: + self.assertEqual(self.decompress(f.read()), self.TEXT * 2) + def testSeekForward(self): # "Test BZ2File.seek(150, 0)" self.createTempFile() @@ -225,6 +292,14 @@ class BZ2FileTest(BaseTest): bz2f.seek(150) self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekForwardMultiStream(self): + # "Test BZ2File.seek(150, 0) across stream boundaries" + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.seek) + bz2f.seek(len(self.TEXT) + 150) + self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekBackwards(self): # "Test BZ2File.seek(-150, 1)" self.createTempFile() @@ -233,6 +308,16 @@ class BZ2FileTest(BaseTest): bz2f.seek(-150, 1) self.assertEqual(bz2f.read(), self.TEXT[500-150:]) + def testSeekBackwardsMultiStream(self): + # "Test BZ2File.seek(-150, 1) across stream boundaries" + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + readto = len(self.TEXT) + 100 + while readto > 0: + readto -= len(bz2f.read(readto)) + bz2f.seek(-150, 1) + self.assertEqual(bz2f.read(), self.TEXT[100-150:] + self.TEXT) + def testSeekBackwardsFromEnd(self): # "Test BZ2File.seek(-150, 2)" self.createTempFile() @@ -240,6 +325,13 @@ class BZ2FileTest(BaseTest): bz2f.seek(-150, 2) self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) + def testSeekBackwardsFromEndMultiStream(self): + # "Test BZ2File.seek(-1000, 2) across stream boundaries" + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-1000, 2) + self.assertEqual(bz2f.read(), (self.TEXT * 2)[-1000:]) + def testSeekPostEnd(self): # "Test BZ2File.seek(150000)" self.createTempFile() @@ -248,6 +340,14 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndMultiStream(self): + # "Test BZ2File.seek(150000)" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwice(self): # "Test BZ2File.seek(150000) twice" self.createTempFile() @@ -257,6 +357,15 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwiceMultiStream(self): + # "Test BZ2File.seek(150000) twice with a multi stream archive" + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPreStart(self): # "Test BZ2File.seek(-150, 0)" self.createTempFile() @@ -265,6 +374,14 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.tell(), 0) self.assertEqual(bz2f.read(), self.TEXT) + def testSeekPreStartMultiStream(self): + # "Test BZ2File.seek(-150, 0) with a multi stream archive" + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-150) + self.assertEqual(bz2f.tell(), 0) + self.assertEqual(bz2f.read(), self.TEXT * 2) + def testFileno(self): # "Test BZ2File.fileno()" self.createTempFile() @@ -510,6 +627,11 @@ class FuncTest(BaseTest): # "Test decompress() function with incomplete data" self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) + def testDecompressMultiStream(self): + # "Test decompress() function for data with multiple streams" + text = bz2.decompress(self.DATA * 5) + self.assertEqual(text, self.TEXT * 5) + def test_main(): support.run_unittest( BZ2FileTest, diff --git a/Misc/NEWS b/Misc/NEWS index f3143ab..503661a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -161,6 +161,9 @@ Core and Builtins Library ------- +- Issue #1625: BZ2File and bz2.decompress() now support multi-stream files. + Initial patch by Nir Aides. + - Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. - Issue #12175: FileIO.readall() now only reads the file position and size -- cgit v0.12 From 200e00a90441f0ee40a71306256774feb0beca7b Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Fri, 27 May 2011 01:52:16 +0200 Subject: Update bz2 docs following issue #1625. --- Doc/library/bz2.rst | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 2ccdb51..87f2cf3 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -37,14 +37,18 @@ All of the classes in this module may safely be accessed from multiple threads. *fileobj*), or operate directly on a named file (named by *filename*). Exactly one of these two parameters should be provided. - The *mode* argument can be either ``'r'`` for reading (default), or ``'w'`` - for writing. + The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for + overwriting, or ``'a'`` for appending. If *fileobj* is provided, a mode of + ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. The *buffering* argument is ignored. Its use is deprecated. - If *mode* is ``'w'``, *compresslevel* can be a number between ``1`` and - ``9`` specifying the level of compression: ``1`` produces the least - compression, and ``9`` (default) produces the most compression. + If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between + ``1`` and ``9`` specifying the level of compression: ``1`` produces the + least compression, and ``9`` (default) produces the most compression. + + If *mode* is ``'r'``, the input file may be the concatenation of multiple + compressed streams. :class:`BZ2File` provides all of the members specified by the :class:`io.BufferedIOBase`, except for :meth:`detach` and :meth:`truncate`. @@ -70,6 +74,10 @@ All of the classes in this module may safely be accessed from multiple threads. .. versionchanged:: 3.3 The *fileobj* argument to the constructor was added. + .. versionchanged:: 3.3 + The ``'a'`` (append) mode was added, along with support for reading + multi-stream files. + Incremental (de)compression --------------------------- @@ -106,14 +114,20 @@ Incremental (de)compression incrementally. For one-shot compression, use the :func:`decompress` function instead. + .. note:: + This class does not transparently handle inputs containing multiple + compressed streams, unlike :func:`decompress` and :class:`BZ2File`. If + you need to decompress a multi-stream input with :class:`BZ2Decompressor`, + you must use a new decompressor for each stream. + .. method:: decompress(data) Provide data to the decompressor object. Returns a chunk of decompressed data if possible, or an empty byte string otherwise. - Attempting to decompress data after the end of stream is reached raises - an :exc:`EOFError`. If any data is found after the end of the stream, it - is ignored and saved in the :attr:`unused_data` attribute. + Attempting to decompress data after the end of the current stream is + reached raises an :exc:`EOFError`. If any data is found after the end of + the stream, it is ignored and saved in the :attr:`unused_data` attribute. .. attribute:: eof @@ -127,6 +141,9 @@ Incremental (de)compression Data found after the end of the compressed stream. + If this attribute is accessed before the end of the stream has been + reached, its value will be ``b''``. + One-shot (de)compression ------------------------ @@ -145,5 +162,11 @@ One-shot (de)compression Decompress *data*. + If *data* is the concatenation of multiple compressed streams, decompress + all of the streams. + For incremental decompression, use a :class:`BZ2Decompressor` instead. + .. versionchanged:: 3.3 + Support for multi-stream inputs was added. + -- cgit v0.12 From e0854f9dbff33c941fc7897c6f1c44cadec9d2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Fri, 27 May 2011 04:36:52 +0200 Subject: Add missing closing paren --- Doc/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 7431545..6a4daa5 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -242,7 +242,7 @@ Glossary processing, remembering the location execution state (including local variables and pending try-statements). When the generator resumes, it picks-up where it left-off (in contrast to functions which start fresh on - every invocation. + every invocation). .. index:: single: generator expression -- cgit v0.12 From f33de715147371c39713a1421fc8bdb01676902f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Fri, 27 May 2011 04:42:47 +0200 Subject: Minor doc addition for clarity --- Doc/library/functions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 1547f6d..42f2bc9 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -527,7 +527,7 @@ are always available. They are listed here in alphabetical order. Two objects with non-overlapping lifetimes may have the same :func:`id` value. - .. impl-detail:: This is the address of the object. + .. impl-detail:: This is the address of the object in memory. .. function:: input([prompt]) -- cgit v0.12 From d408503b2c74b53f734adab6cd35866be460ce5e Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 07:53:28 -0500 Subject: remove unused string WILFE attribute --- Python/marshal.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Python/marshal.c b/Python/marshal.c index 73d4f37..f66b765 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -60,7 +60,6 @@ typedef struct { PyObject *str; char *ptr; char *end; - PyObject *strings; /* dict on marshal, list on unmarshal */ int version; } WFILE; @@ -444,7 +443,6 @@ PyMarshal_WriteLongToFile(long x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; - wf.strings = NULL; wf.version = version; w_long(x, &wf); } @@ -456,10 +454,8 @@ PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version) wf.fp = fp; wf.error = WFERR_OK; wf.depth = 0; - wf.strings = (version > 0) ? PyDict_New() : NULL; wf.version = version; w_object(x, &wf); - Py_XDECREF(wf.strings); } typedef WFILE RFILE; /* Same struct with different invariants */ @@ -1041,7 +1037,6 @@ PyMarshal_ReadShortFromFile(FILE *fp) RFILE rf; assert(fp); rf.fp = fp; - rf.strings = NULL; rf.end = rf.ptr = NULL; return r_short(&rf); } @@ -1051,7 +1046,6 @@ PyMarshal_ReadLongFromFile(FILE *fp) { RFILE rf; rf.fp = fp; - rf.strings = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); } @@ -1112,11 +1106,9 @@ PyMarshal_ReadObjectFromFile(FILE *fp) RFILE rf; PyObject *result; rf.fp = fp; - rf.strings = PyList_New(0); rf.depth = 0; rf.ptr = rf.end = NULL; result = r_object(&rf); - Py_DECREF(rf.strings); return result; } @@ -1128,10 +1120,8 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) rf.fp = NULL; rf.ptr = str; rf.end = str + len; - rf.strings = PyList_New(0); rf.depth = 0; result = r_object(&rf); - Py_DECREF(rf.strings); return result; } @@ -1150,9 +1140,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version) wf.error = WFERR_OK; wf.depth = 0; wf.version = version; - wf.strings = (version > 0) ? PyDict_New() : NULL; w_object(x, &wf); - Py_XDECREF(wf.strings); if (wf.str != NULL) { char *base = PyBytes_AS_STRING((PyBytesObject *)wf.str); if (wf.ptr - base > PY_SSIZE_T_MAX) { @@ -1242,10 +1230,8 @@ marshal_load(PyObject *self, PyObject *f) Py_DECREF(data); return NULL; } - rf.strings = PyList_New(0); rf.depth = 0; result = read_object(&rf); - Py_DECREF(rf.strings); Py_DECREF(data); return result; } @@ -1298,10 +1284,8 @@ marshal_loads(PyObject *self, PyObject *args) rf.fp = NULL; rf.ptr = s; rf.end = s + n; - rf.strings = PyList_New(0); rf.depth = 0; result = read_object(&rf); - Py_DECREF(rf.strings); PyBuffer_Release(&p); return result; } -- cgit v0.12 From 43b068648e81d951f78d9f1cf1cd8f42731e164e Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 09:08:01 -0500 Subject: try to use the same str object for all code filenames when compiling or unmarshalling (#12190) This should reduce memory usage. --- Lib/test/test_compile.py | 9 +++++++++ Lib/test/test_marshal.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ Python/compile.c | 27 +++++++++------------------ Python/marshal.c | 19 +++++++++++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 58ef297..c5f9189 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import unittest import sys import _ast +import types from test import support class TestSpecifics(unittest.TestCase): @@ -433,6 +434,14 @@ if 1: ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '', 'exec') + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + c = compile(s, "myfile", "exec") + for obj in c.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(obj.co_filename, c.co_filename) + def test_main(): support.run_unittest(TestSpecifics) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 81cf598..8a5590a 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -5,6 +5,7 @@ import marshal import sys import unittest import os +import types class HelperMixin: def helper(self, sample, *extra): @@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase): codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) + def test_different_filenames(self): + co1 = compile("x", "f1", "exec") + co2 = compile("y", "f2", "exec") + co1, co2 = marshal.loads(marshal.dumps((co1, co2))) + self.assertEqual(co1.co_filename, "f1") + self.assertEqual(co2.co_filename, "f2") + + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + co = compile(s, "myfile", "exec") + co = marshal.loads(marshal.dumps(co)) + for obj in co.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(co.co_filename, obj.co_filename) + class ContainerTestCase(unittest.TestCase, HelperMixin): d = {'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, diff --git a/Misc/NEWS b/Misc/NEWS index 0514c96..df8bca8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12190: Try to use the same filename object when compiling unmarshalling + a code objects in the same file. + - Issue #12166: Move implementations of dir() specialized for various types into the __dir__() methods of those types. diff --git a/Python/compile.c b/Python/compile.c index d195967..96d01cd 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -135,6 +135,7 @@ managed by compiler_enter_scope() and compiler_exit_scope(). struct compiler { const char *c_filename; + PyObject *c_filename_obj; struct symtable *c_st; PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; @@ -272,6 +273,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, if (!compiler_init(&c)) return NULL; c.c_filename = filename; + c.c_filename_obj = PyUnicode_DecodeFSDefault(filename); + if (!c.c_filename_obj) + goto finally; c.c_arena = arena; c.c_future = PyFuture_FromAST(mod, filename); if (c.c_future == NULL) @@ -324,6 +328,8 @@ compiler_free(struct compiler *c) PySymtable_Free(c->c_st); if (c->c_future) PyObject_Free(c->c_future); + if (c->c_filename_obj) + Py_DECREF(c->c_filename_obj); Py_DECREF(c->c_stack); } @@ -3361,7 +3367,7 @@ compiler_in_loop(struct compiler *c) { static int compiler_error(struct compiler *c, const char *errstr) { - PyObject *loc, *filename; + PyObject *loc; PyObject *u = NULL, *v = NULL; loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno); @@ -3369,16 +3375,7 @@ compiler_error(struct compiler *c, const char *errstr) Py_INCREF(Py_None); loc = Py_None; } - if (c->c_filename != NULL) { - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto exit; - } - else { - Py_INCREF(Py_None); - filename = Py_None; - } - u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno, + u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno, c->u->u_col_offset, loc); if (!u) goto exit; @@ -3927,7 +3924,6 @@ makecode(struct compiler *c, struct assembler *a) PyObject *consts = NULL; PyObject *names = NULL; PyObject *varnames = NULL; - PyObject *filename = NULL; PyObject *name = NULL; PyObject *freevars = NULL; PyObject *cellvars = NULL; @@ -3951,10 +3947,6 @@ makecode(struct compiler *c, struct assembler *a) freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars)); if (!freevars) goto error; - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto error; - nlocals = PyDict_Size(c->u->u_varnames); flags = compute_code_flags(c); if (flags < 0) @@ -3974,14 +3966,13 @@ makecode(struct compiler *c, struct assembler *a) nlocals, stackdepth(c), flags, bytecode, consts, names, varnames, freevars, cellvars, - filename, c->u->u_name, + c->c_filename_obj, c->u->u_name, c->u->u_firstlineno, a->a_lnotab); error: Py_XDECREF(consts); Py_XDECREF(names); Py_XDECREF(varnames); - Py_XDECREF(filename); Py_XDECREF(name); Py_XDECREF(freevars); Py_XDECREF(cellvars); diff --git a/Python/marshal.c b/Python/marshal.c index f66b765..7b327ad 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -58,6 +58,7 @@ typedef struct { int depth; /* If fp == NULL, the following are valid: */ PyObject *str; + PyObject *current_filename; char *ptr; char *end; int version; @@ -976,6 +977,18 @@ r_object(RFILE *p) filename = r_object(p); if (filename == NULL) goto code_error; + if (PyUnicode_CheckExact(filename)) { + if (p->current_filename != NULL) { + if (!PyUnicode_Compare(filename, p->current_filename)) { + Py_DECREF(filename); + Py_INCREF(p->current_filename); + filename = p->current_filename; + } + } + else { + p->current_filename = filename; + } + } name = r_object(p); if (name == NULL) goto code_error; @@ -1037,6 +1050,7 @@ PyMarshal_ReadShortFromFile(FILE *fp) RFILE rf; assert(fp); rf.fp = fp; + rf.current_filename = NULL; rf.end = rf.ptr = NULL; return r_short(&rf); } @@ -1046,6 +1060,7 @@ PyMarshal_ReadLongFromFile(FILE *fp) { RFILE rf; rf.fp = fp; + rf.current_filename = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); } @@ -1106,6 +1121,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) RFILE rf; PyObject *result; rf.fp = fp; + rf.current_filename = NULL; rf.depth = 0; rf.ptr = rf.end = NULL; result = r_object(&rf); @@ -1118,6 +1134,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) RFILE rf; PyObject *result; rf.fp = NULL; + rf.current_filename = NULL; rf.ptr = str; rf.end = str + len; rf.depth = 0; @@ -1214,6 +1231,7 @@ marshal_load(PyObject *self, PyObject *f) if (data == NULL) return NULL; rf.fp = NULL; + rf.current_filename = NULL; if (PyBytes_Check(data)) { rf.ptr = PyBytes_AS_STRING(data); rf.end = rf.ptr + PyBytes_GET_SIZE(data); @@ -1282,6 +1300,7 @@ marshal_loads(PyObject *self, PyObject *args) s = p.buf; n = p.len; rf.fp = NULL; + rf.current_filename = NULL; rf.ptr = s; rf.end = s + n; rf.depth = 0; -- cgit v0.12 From 4f2dab5c337f202b8fe3058108efc40cb1a0a724 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 27 May 2011 16:46:51 +0200 Subject: Revert my commit 7ba176c2f558: "Avoid useless "++" at the end of functions Warnings found by the Clang Static Analyzer." Most people prefer ++ at the end of functions. --- Objects/setobject.c | 2 +- Objects/unicodeobject.c | 2 +- Python/compile.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index ebfddb3..2e62518 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -616,7 +616,7 @@ set_repr(PySetObject *so) Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1, newsize-2); u += newsize-2; - *u = '}'; + *u++ = '}'; Py_DECREF(listrepr); if (Py_TYPE(so) != &PySet_Type) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 309159c..4361908 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6474,7 +6474,7 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, } } /* 0-terminate the output string */ - *output = '\0'; + *output++ = '\0'; Py_XDECREF(exc); Py_XDECREF(errorHandler); return 0; diff --git a/Python/compile.c b/Python/compile.c index 96d01cd..34e7603 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3744,11 +3744,11 @@ assemble_lnotab(struct assembler *a, struct instr *i) a->a_lnotab_off += 2; if (d_bytecode) { *lnotab++ = d_bytecode; - *lnotab = d_lineno; + *lnotab++ = d_lineno; } else { /* First line of a block; def stmt, etc. */ *lnotab++ = 0; - *lnotab = d_lineno; + *lnotab++ = d_lineno; } a->a_lineno = i->i_lineno; a->a_lineno_off = a->a_offset; @@ -3793,7 +3793,7 @@ assemble_emit(struct assembler *a, struct instr *i) if (i->i_hasarg) { assert(size == 3 || size == 6); *code++ = arg & 0xff; - *code = arg >> 8; + *code++ = arg >> 8; } return 1; } -- cgit v0.12 From 05010706697ce9c18e7f8a8e571753b0bcfd6548 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 27 May 2011 16:50:40 +0200 Subject: Revert my commit 3555cf6f9c98: "Issue #8796: codecs.open() calls the builtin open() function instead of using StreamReaderWriter. Deprecate StreamReader, StreamWriter, StreamReaderWriter, StreamRecoder and EncodedFile() of the codec module. Use the builtin open() function or io.TextIOWrapper instead." "It has not been approved !" wrote Marc-Andre Lemburg. --- Doc/library/codecs.rst | 25 -------- Lib/codecs.py | 25 ++++---- Lib/test/test_codecs.py | 152 +++++++++++++++--------------------------------- Misc/NEWS | 5 -- 4 files changed, 59 insertions(+), 148 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index b58e410..4d5058e 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -85,9 +85,6 @@ It defines the following functions: In case a search function cannot find a given encoding, it should return ``None``. - .. deprecated:: 3.3 - *streamreader* and *streamwriter* attributes are now deprecated. - .. function:: lookup(encoding) @@ -142,8 +139,6 @@ functions which use :func:`lookup` for the codec lookup: Raises a :exc:`LookupError` in case the encoding cannot be found. - .. deprecated:: 3.3 - .. function:: getwriter(encoding) @@ -152,8 +147,6 @@ functions which use :func:`lookup` for the codec lookup: Raises a :exc:`LookupError` in case the encoding cannot be found. - .. deprecated:: 3.3 - .. function:: register_error(name, error_handler) @@ -224,11 +217,6 @@ utility functions: .. note:: - This function is kept for backward compatibility with Python 2, the - builtin :func:`open` function should be used instead. - - .. note:: - The wrapped version's methods will accept and return strings only. Bytes arguments will be rejected. @@ -263,8 +251,6 @@ utility functions: ``'strict'``, which causes :exc:`ValueError` to be raised in case an encoding error occurs. - .. deprecated:: 3.3 - .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) @@ -577,9 +563,6 @@ The :class:`StreamWriter` class is a subclass of :class:`Codec` and defines the following methods which every stream writer must define in order to be compatible with the Python codec registry. -.. deprecated:: 3.3 - Use the builtin the :class:`io.TextIOWrapper` class. - .. class:: StreamWriter(stream[, errors]) @@ -645,9 +628,6 @@ The :class:`StreamReader` class is a subclass of :class:`Codec` and defines the following methods which every stream reader must define in order to be compatible with the Python codec registry. -.. deprecated:: 3.3 - Use the builtin the :class:`io.TextIOWrapper` class. - .. class:: StreamReader(stream[, errors]) @@ -748,9 +728,6 @@ and write modes. The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. -.. deprecated:: 3.3 - Use the :class:`io.TextIOWrapper` class. - .. class:: StreamReaderWriter(stream, Reader, Writer, errors) @@ -775,8 +752,6 @@ which is sometimes useful when dealing with different encoding environments. The design is such that one can use the factory functions returned by the :func:`lookup` function to construct the instance. -.. deprecated:: 3.3 - .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors) diff --git a/Lib/codecs.py b/Lib/codecs.py index ec7879f..b150d64 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -345,8 +345,6 @@ class StreamWriter(Codec): The set of allowed parameter values can be extended via register_error. """ - import warnings - warnings.warn('use io.TextIOWrapper', DeprecationWarning, stacklevel=2) self.stream = stream self.errors = errors @@ -418,8 +416,6 @@ class StreamReader(Codec): The set of allowed parameter values can be extended via register_error. """ - import warnings - warnings.warn('use io.TextIOWrapper', DeprecationWarning, stacklevel=2) self.stream = stream self.errors = errors self.bytebuffer = b"" @@ -850,7 +846,7 @@ class StreamRecoder: ### Shortcuts -def open(filename, mode='r', encoding=None, errors=None, buffering=1): +def open(filename, mode='rb', encoding=None, errors='strict', buffering=1): """ Open an encoded file using the given mode and return a wrapped version providing transparent encoding/decoding. @@ -881,13 +877,18 @@ def open(filename, mode='r', encoding=None, errors=None, buffering=1): parameter. """ - if encoding is not None: - return builtins.open(filename, mode, buffering, - encoding, errors, newline='') - else: - if 'b' not in mode: - mode = mode + 'b' - return builtins.open(filename, mode, buffering, encoding, errors) + if encoding is not None and \ + 'b' not in mode: + # Force opening of the file in binary mode + mode = mode + 'b' + file = builtins.open(filename, mode, buffering) + if encoding is None: + return file + info = lookup(encoding) + srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) + # Add attributes to simplify introspection + srw.encoding = encoding + return srw def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'): diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 1f46560..c0450e7 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,10 +1,7 @@ from test import support -import _testcapi -import codecs -import io -import sys import unittest -import warnings +import codecs +import sys, _testcapi, io class Queue(object): """ @@ -66,9 +63,7 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): # the StreamReader and check that the results equal the appropriate # entries from partialresults. q = Queue(b"") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - r = codecs.getreader(self.encoding)(q) + r = codecs.getreader(self.encoding)(q) result = "" for (c, partialresult) in zip(input.encode(self.encoding), partialresults): q.write(bytes([c])) @@ -111,9 +106,7 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): return codecs.getreader(self.encoding)(stream) def readalllines(input, keepends=True, size=None): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = getreader(input) + reader = getreader(input) lines = [] while True: line = reader.readline(size=size, keepends=keepends) @@ -222,18 +215,14 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): ' \r\n', ] stream = io.BytesIO("".join(s).encode(self.encoding)) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(self.encoding)(stream) + reader = codecs.getreader(self.encoding)(stream) for (i, line) in enumerate(reader): self.assertEqual(line, s[i]) def test_readlinequeue(self): q = Queue(b"") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - writer = codecs.getwriter(self.encoding)(q) - reader = codecs.getreader(self.encoding)(q) + writer = codecs.getwriter(self.encoding)(q) + reader = codecs.getreader(self.encoding)(q) # No lineends writer.write("foo\r") @@ -264,9 +253,7 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): s = (s1+s2+s3).encode(self.encoding) stream = io.BytesIO(s) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(self.encoding)(stream) + reader = codecs.getreader(self.encoding)(stream) self.assertEqual(reader.readline(), s1) self.assertEqual(reader.readline(), s2) self.assertEqual(reader.readline(), s3) @@ -281,9 +268,7 @@ class ReadTest(unittest.TestCase, MixInCheckStateHandling): s = (s1+s2+s3+s4+s5).encode(self.encoding) stream = io.BytesIO(s) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(self.encoding)(stream) + reader = codecs.getreader(self.encoding)(stream) self.assertEqual(reader.readline(), s1) self.assertEqual(reader.readline(), s2) self.assertEqual(reader.readline(), s3) @@ -305,9 +290,7 @@ class UTF32Test(ReadTest): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream s = io.BytesIO() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = writer(s) + f = writer(s) f.write("spam") f.write("spam") d = s.getvalue() @@ -315,22 +298,16 @@ class UTF32Test(ReadTest): self.assertTrue(d == self.spamle or d == self.spambe) # try to read it back s = io.BytesIO(d) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = reader(s) + f = reader(s) self.assertEqual(f.read(), "spamspam") def test_badbom(self): s = io.BytesIO(4*b"\xff") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = codecs.getreader(self.encoding)(s) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) s = io.BytesIO(8*b"\xff") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = codecs.getreader(self.encoding)(s) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) def test_partial(self): @@ -477,9 +454,7 @@ class UTF16Test(ReadTest): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream s = io.BytesIO() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = writer(s) + f = writer(s) f.write("spam") f.write("spam") d = s.getvalue() @@ -487,22 +462,16 @@ class UTF16Test(ReadTest): self.assertTrue(d == self.spamle or d == self.spambe) # try to read it back s = io.BytesIO(d) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = reader(s) + f = reader(s) self.assertEqual(f.read(), "spamspam") def test_badbom(self): s = io.BytesIO(b"\xff\xff") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = codecs.getreader(self.encoding)(s) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = codecs.getreader(self.encoding)(s) + f = codecs.getreader(self.encoding)(s) self.assertRaises(UnicodeError, f.read) def test_partial(self): @@ -548,8 +517,7 @@ class UTF16Test(ReadTest): self.addCleanup(support.unlink, support.TESTFN) with open(support.TESTFN, 'wb') as fp: fp.write(s) - with codecs.open(support.TESTFN, 'U', - encoding=self.encoding) as reader: + with codecs.open(support.TESTFN, 'U', encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) class UTF16LETest(ReadTest): @@ -737,9 +705,7 @@ class UTF8SigTest(ReadTest): reader = codecs.getreader("utf-8-sig") for sizehint in [None] + list(range(1, 11)) + \ [64, 128, 256, 512, 1024]: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - istream = reader(io.BytesIO(bytestring)) + istream = reader(io.BytesIO(bytestring)) ostream = io.StringIO() while 1: if sizehint is not None: @@ -761,9 +727,7 @@ class UTF8SigTest(ReadTest): reader = codecs.getreader("utf-8-sig") for sizehint in [None] + list(range(1, 11)) + \ [64, 128, 256, 512, 1024]: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - istream = reader(io.BytesIO(bytestring)) + istream = reader(io.BytesIO(bytestring)) ostream = io.StringIO() while 1: if sizehint is not None: @@ -785,9 +749,7 @@ class EscapeDecodeTest(unittest.TestCase): class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8") + f2 = codecs.EncodedFile(f, "unicode_internal", "utf-8") f2.write("a") f2.close() # Python used to crash on this at exit because of a refcount @@ -1164,9 +1126,7 @@ class IDNACodecTest(unittest.TestCase): self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") def test_stream(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - r = codecs.getreader("idna")(io.BytesIO(b"abc")) + r = codecs.getreader("idna")(io.BytesIO(b"abc")) r.read(3) self.assertEqual(r.read(), "") @@ -1273,24 +1233,18 @@ class CodecsModuleTest(unittest.TestCase): class StreamReaderTest(unittest.TestCase): def setUp(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.reader = codecs.getreader('utf-8') + self.reader = codecs.getreader('utf-8') self.stream = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80') def test_readlines(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - f = self.reader(self.stream) + f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) class EncodedFileTest(unittest.TestCase): def test_basic(self): f = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80') - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8') + ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8') self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae') f = io.BytesIO() @@ -1434,9 +1388,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): if encoding not in broken_unicode_with_streams: # check stream reader/writer q = Queue(b"") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - writer = codecs.getwriter(encoding)(q) + writer = codecs.getwriter(encoding)(q) encodedresult = b"" for c in s: writer.write(c) @@ -1444,9 +1396,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.assertTrue(type(chunk) is bytes, type(chunk)) encodedresult += chunk q = Queue(b"") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(encoding)(q) + reader = codecs.getreader(encoding)(q) decodedresult = "" for c in encodedresult: q.write(bytes([c])) @@ -1520,9 +1470,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): continue if encoding in broken_unicode_with_streams: continue - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) + reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding))) for t in range(5): # Test that calling seek resets the internal codec state and buffers reader.seek(0, 0) @@ -1591,19 +1539,15 @@ class CharmapTest(unittest.TestCase): class WithStmtTest(unittest.TestCase): def test_encodedfile(self): f = io.BytesIO(b"\xc3\xbc") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - with codecs.EncodedFile(f, "latin-1", "utf-8") as ef: - self.assertEqual(ef.read(), b"\xfc") + with codecs.EncodedFile(f, "latin-1", "utf-8") as ef: + self.assertEqual(ef.read(), b"\xfc") def test_streamreaderwriter(self): f = io.BytesIO(b"\xc3\xbc") info = codecs.lookup("utf-8") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - with codecs.StreamReaderWriter(f, info.streamreader, - info.streamwriter, 'strict') as srw: - self.assertEqual(srw.read(), "\xfc") + with codecs.StreamReaderWriter(f, info.streamreader, + info.streamwriter, 'strict') as srw: + self.assertEqual(srw.read(), "\xfc") class TypesTest(unittest.TestCase): def test_decode_unicode(self): @@ -1700,15 +1644,15 @@ class BomTest(unittest.TestCase): # (StreamWriter) Check that the BOM is written after a seek(0) with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: - f.write(data[0]) - self.assertNotEqual(f.tell(), 0) - f.seek(0) - f.write(data) + f.writer.write(data[0]) + self.assertNotEqual(f.writer.tell(), 0) + f.writer.seek(0) + f.writer.write(data) f.seek(0) self.assertEqual(f.read(), data) - # Check that the BOM is not written after a seek() at a - # position different than the start + # Check that the BOM is not written after a seek() at a position + # different than the start with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: f.write(data) f.seek(f.tell()) @@ -1716,12 +1660,12 @@ class BomTest(unittest.TestCase): f.seek(0) self.assertEqual(f.read(), data * 2) - # (StreamWriter) Check that the BOM is not written after a - # seek() at a position different than the start + # (StreamWriter) Check that the BOM is not written after a seek() + # at a position different than the start with codecs.open(support.TESTFN, 'w+', encoding=encoding) as f: - f.write(data) - f.seek(f.tell()) - f.write(data) + f.writer.write(data) + f.writer.seek(f.writer.tell()) + f.writer.write(data) f.seek(0) self.assertEqual(f.read(), data * 2) @@ -1760,9 +1704,7 @@ class TransformCodecTest(unittest.TestCase): def test_read(self): for encoding in bytes_transform_encodings: sin = codecs.encode(b"\x80", encoding) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(encoding)(io.BytesIO(sin)) + reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.read() self.assertEqual(sout, b"\x80") @@ -1771,9 +1713,7 @@ class TransformCodecTest(unittest.TestCase): if encoding in ['uu_codec', 'zlib_codec']: continue sin = codecs.encode(b"\x80", encoding) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - reader = codecs.getreader(encoding)(io.BytesIO(sin)) + reader = codecs.getreader(encoding)(io.BytesIO(sin)) sout = reader.readline() self.assertEqual(sout, b"\x80") diff --git a/Misc/NEWS b/Misc/NEWS index df8bca8..2a916fd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -167,11 +167,6 @@ Library - Issue #1625: BZ2File and bz2.decompress() now support multi-stream files. Initial patch by Nir Aides. -- Issue #8796: codecs.open() calls the builtin open() function instead of using - StreamReaderWriter. Deprecate StreamReader, StreamWriter, StreamReaderWriter, - StreamRecoder and EncodedFile() of the codec module. Use the builtin open() - function or io.TextIOWrapper instead. - - Issue #12175: BufferedReader.read(-1) now calls raw.readall() if available. - Issue #12175: FileIO.readall() now only reads the file position and size -- cgit v0.12 From bf1bbc145299964a37cfae5bc5565177192f68ad Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 13:58:08 -0500 Subject: reflect with statements with multiple items in the AST (closes #12106) --- Include/Python-ast.h | 19 ++++-- Misc/NEWS | 3 + Parser/Python.asdl | 4 +- Python/Python-ast.c | 161 ++++++++++++++++++++++++++++++++++++++------------- Python/ast.c | 43 +++++--------- Python/compile.c | 21 ++++--- Python/symtable.c | 16 +++-- 7 files changed, 182 insertions(+), 85 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 0ad788b..74c5264 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -36,6 +36,8 @@ typedef struct _keyword *keyword_ty; typedef struct _alias *alias_ty; +typedef struct _withitem *withitem_ty; + enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, Suite_kind=4}; @@ -128,8 +130,7 @@ struct _stmt { } If; struct { - expr_ty context_expr; - expr_ty optional_vars; + asdl_seq *items; asdl_seq *body; } With; @@ -383,6 +384,11 @@ struct _alias { identifier asname; }; +struct _withitem { + expr_ty context_expr; + expr_ty optional_vars; +}; + #define Module(a0, a1) _Py_Module(a0, a1) mod_ty _Py_Module(asdl_seq * body, PyArena *arena); @@ -421,9 +427,9 @@ stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, #define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5) stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, PyArena *arena); -#define With(a0, a1, a2, a3, a4, a5) _Py_With(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, - int lineno, int col_offset, PyArena *arena); +#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4) +stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, + PyArena *arena); #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4) stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena); @@ -547,6 +553,9 @@ arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena); keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); +#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) +withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena + *arena); PyObject* PyAST_mod2obj(mod_ty t); mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); diff --git a/Misc/NEWS b/Misc/NEWS index 2a916fd..9c00590 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12106: The use of the multiple-with shorthand syntax is now reflected + in the AST. + - Issue #12190: Try to use the same filename object when compiling unmarshalling a code objects in the same file. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 8e2e1ac..de48643 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -28,7 +28,7 @@ module Python | For(expr target, expr iter, stmt* body, stmt* orelse) | While(expr test, stmt* body, stmt* orelse) | If(expr test, stmt* body, stmt* orelse) - | With(expr context_expr, expr? optional_vars, stmt* body) + | With(withitem* items, stmt* body) | Raise(expr? exc, expr? cause) | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) @@ -115,5 +115,7 @@ module Python -- import name with optional 'as' alias. alias = (identifier name, identifier? asname) + + withitem = (expr context_expr, expr? optional_vars) } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 6b1ea3c..2364db3 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -95,8 +95,7 @@ static char *If_fields[]={ }; static PyTypeObject *With_type; static char *With_fields[]={ - "context_expr", - "optional_vars", + "items", "body", }; static PyTypeObject *Raise_type; @@ -392,6 +391,12 @@ static char *alias_fields[]={ "name", "asname", }; +static PyTypeObject *withitem_type; +static PyObject* ast2obj_withitem(void*); +static char *withitem_fields[]={ + "context_expr", + "optional_vars", +}; static int @@ -680,7 +685,7 @@ static int init_types(void) if (!While_type) return 0; If_type = make_type("If", stmt_type, If_fields, 3); if (!If_type) return 0; - With_type = make_type("With", stmt_type, With_fields, 3); + With_type = make_type("With", stmt_type, With_fields, 2); if (!With_type) return 0; Raise_type = make_type("Raise", stmt_type, Raise_fields, 2); if (!Raise_type) return 0; @@ -938,6 +943,8 @@ static int init_types(void) if (!keyword_type) return 0; alias_type = make_type("alias", &AST_type, alias_fields, 2); if (!alias_type) return 0; + withitem_type = make_type("withitem", &AST_type, withitem_fields, 2); + if (!withitem_type) return 0; initialized = 1; return 1; } @@ -960,6 +967,7 @@ static int obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena); static int obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena); static int obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena); static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena); +static int obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena); mod_ty Module(asdl_seq * body, PyArena *arena) @@ -1225,21 +1233,15 @@ If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int } stmt_ty -With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, int lineno, - int col_offset, PyArena *arena) +With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, PyArena + *arena) { stmt_ty p; - if (!context_expr) { - PyErr_SetString(PyExc_ValueError, - "field context_expr is required for With"); - return NULL; - } p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; p->kind = With_kind; - p->v.With.context_expr = context_expr; - p->v.With.optional_vars = optional_vars; + p->v.With.items = items; p->v.With.body = body; p->lineno = lineno; p->col_offset = col_offset; @@ -2135,6 +2137,23 @@ alias(identifier name, identifier asname, PyArena *arena) return p; } +withitem_ty +withitem(expr_ty context_expr, expr_ty optional_vars, PyArena *arena) +{ + withitem_ty p; + if (!context_expr) { + PyErr_SetString(PyExc_ValueError, + "field context_expr is required for withitem"); + return NULL; + } + p = (withitem_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->context_expr = context_expr; + p->optional_vars = optional_vars; + return p; +} + PyObject* ast2obj_mod(void* _o) @@ -2390,15 +2409,9 @@ ast2obj_stmt(void* _o) case With_kind: result = PyType_GenericNew(With_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_expr(o->v.With.context_expr); + value = ast2obj_list(o->v.With.items, ast2obj_withitem); if (!value) goto failed; - if (PyObject_SetAttrString(result, "context_expr", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_expr(o->v.With.optional_vars); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "optional_vars", value) == - -1) + if (PyObject_SetAttrString(result, "items", value) == -1) goto failed; Py_DECREF(value); value = ast2obj_list(o->v.With.body, ast2obj_stmt); @@ -3370,6 +3383,35 @@ failed: return NULL; } +PyObject* +ast2obj_withitem(void* _o) +{ + withitem_ty o = (withitem_ty)_o; + PyObject *result = NULL, *value = NULL; + if (!o) { + Py_INCREF(Py_None); + return Py_None; + } + + result = PyType_GenericNew(withitem_type, NULL, NULL); + if (!result) return NULL; + value = ast2obj_expr(o->context_expr); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "context_expr", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->optional_vars); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "optional_vars", value) == -1) + goto failed; + Py_DECREF(value); + return result; +failed: + Py_XDECREF(value); + Py_XDECREF(result); + return NULL; +} + int obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) @@ -4210,33 +4252,34 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) return 1; } if (isinstance) { - expr_ty context_expr; - expr_ty optional_vars; + asdl_seq* items; asdl_seq* body; - if (PyObject_HasAttrString(obj, "context_expr")) { + if (PyObject_HasAttrString(obj, "items")) { int res; - tmp = PyObject_GetAttrString(obj, "context_expr"); + Py_ssize_t len; + Py_ssize_t i; + tmp = PyObject_GetAttrString(obj, "items"); if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &context_expr, arena); - if (res != 0) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + items = asdl_seq_new(len, arena); + if (items == NULL) goto failed; + for (i = 0; i < len; i++) { + withitem_ty value; + res = obj2ast_withitem(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(items, i, value); + } Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from With"); + PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from With"); return 1; } - if (PyObject_HasAttrString(obj, "optional_vars")) { - int res; - tmp = PyObject_GetAttrString(obj, "optional_vars"); - if (tmp == NULL) goto failed; - res = obj2ast_expr(tmp, &optional_vars, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - optional_vars = NULL; - } if (PyObject_HasAttrString(obj, "body")) { int res; Py_ssize_t len; @@ -4262,8 +4305,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from With"); return 1; } - *out = With(context_expr, optional_vars, body, lineno, - col_offset, arena); + *out = With(items, body, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6723,6 +6765,43 @@ failed: return 1; } +int +obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena) +{ + PyObject* tmp = NULL; + expr_ty context_expr; + expr_ty optional_vars; + + if (PyObject_HasAttrString(obj, "context_expr")) { + int res; + tmp = PyObject_GetAttrString(obj, "context_expr"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &context_expr, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); + return 1; + } + if (PyObject_HasAttrString(obj, "optional_vars")) { + int res; + tmp = PyObject_GetAttrString(obj, "optional_vars"); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &optional_vars, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + optional_vars = NULL; + } + *out = withitem(context_expr, optional_vars, arena); + return 0; +failed: + Py_XDECREF(tmp); + return 1; +} + static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, "_ast" @@ -6940,6 +7019,8 @@ PyInit__ast(void) return NULL; if (PyDict_SetItemString(d, "alias", (PyObject*)alias_type) < 0) return NULL; + if (PyDict_SetItemString(d, "withitem", (PyObject*)withitem_type) < 0) + return NULL; return m; } diff --git a/Python/ast.c b/Python/ast.c index 5b12da8..882452b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2967,8 +2967,8 @@ ast_for_try_stmt(struct compiling *c, const node *n) } /* with_item: test ['as' expr] */ -static stmt_ty -ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) +static withitem_ty +ast_for_with_item(struct compiling *c, const node *n) { expr_ty context_expr, optional_vars = NULL; @@ -2987,43 +2987,32 @@ ast_for_with_item(struct compiling *c, const node *n, asdl_seq *content) } } - return With(context_expr, optional_vars, content, LINENO(n), - n->n_col_offset, c->c_arena); + return withitem(context_expr, optional_vars, c->c_arena); } /* with_stmt: 'with' with_item (',' with_item)* ':' suite */ static stmt_ty ast_for_with_stmt(struct compiling *c, const node *n) { - int i; - stmt_ty ret; - asdl_seq *inner; + int i, n_items; + asdl_seq *items, *body; REQ(n, with_stmt); - /* process the with items inside-out */ - i = NCH(n) - 1; - /* the suite of the innermost with item is the suite of the with stmt */ - inner = ast_for_suite(c, CHILD(n, i)); - if (!inner) - return NULL; - - for (;;) { - i -= 2; - ret = ast_for_with_item(c, CHILD(n, i), inner); - if (!ret) + n_items = (NCH(n) - 2) / 2; + items = asdl_seq_new(n_items, c->c_arena); + for (i = 1; i < NCH(n) - 2; i += 2) { + withitem_ty item = ast_for_with_item(c, CHILD(n, i)); + if (!item) return NULL; - /* was this the last item? */ - if (i == 1) - break; - /* if not, wrap the result so far in a new sequence */ - inner = asdl_seq_new(1, c->c_arena); - if (!inner) - return NULL; - asdl_seq_SET(inner, 0, ret); + asdl_seq_SET(items, (i - 1) / 2, item); } - return ret; + body = ast_for_suite(c, CHILD(n, NCH(n) - 1)); + if (!body) + return NULL; + + return With(items, body, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty diff --git a/Python/compile.c b/Python/compile.c index 34e7603..09983d2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -179,7 +179,7 @@ static int compiler_in_loop(struct compiler *); static int inplace_binop(struct compiler *, operator_ty); static int expr_constant(struct compiler *, expr_ty); -static int compiler_with(struct compiler *, stmt_ty); +static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords, @@ -2341,7 +2341,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Continue_kind: return compiler_continue(c); case With_kind: - return compiler_with(c, s); + return compiler_with(c, s, 0); } return 1; } @@ -3068,9 +3068,10 @@ expr_constant(struct compiler *c, expr_ty e) exit(*exc) */ static int -compiler_with(struct compiler *c, stmt_ty s) +compiler_with(struct compiler *c, stmt_ty s, int pos) { basicblock *block, *finally; + withitem_ty item = asdl_seq_GET(s->v.With.items, pos); assert(s->kind == With_kind); @@ -3080,7 +3081,7 @@ compiler_with(struct compiler *c, stmt_ty s) return 0; /* Evaluate EXPR */ - VISIT(c, expr, s->v.With.context_expr); + VISIT(c, expr, item->context_expr); ADDOP_JREL(c, SETUP_WITH, finally); /* SETUP_WITH pushes a finally block. */ @@ -3089,16 +3090,20 @@ compiler_with(struct compiler *c, stmt_ty s) return 0; } - if (s->v.With.optional_vars) { - VISIT(c, expr, s->v.With.optional_vars); + if (item->optional_vars) { + VISIT(c, expr, item->optional_vars); } else { /* Discard result from context.__enter__() */ ADDOP(c, POP_TOP); } - /* BLOCK code */ - VISIT_SEQ(c, stmt, s->v.With.body); + pos++; + if (pos == asdl_seq_LEN(s->v.With.items)) + /* BLOCK code */ + VISIT_SEQ(c, stmt, s->v.With.body) + else if (!compiler_with(c, s, pos)) + return 0; /* End of try block; start the finally block */ ADDOP(c, POP_BLOCK); diff --git a/Python/symtable.c b/Python/symtable.c index 8040665..d276254 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -185,6 +185,7 @@ static int symtable_visit_params(struct symtable *st, asdl_seq *args); static int symtable_visit_argannotations(struct symtable *st, asdl_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); static int symtable_visit_annotations(struct symtable *st, stmt_ty s); +static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static identifier top = NULL, lambda = NULL, genexpr = NULL, @@ -1305,10 +1306,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) /* nothing to do here */ break; case With_kind: - VISIT(st, expr, s->v.With.context_expr); - if (s->v.With.optional_vars) { - VISIT(st, expr, s->v.With.optional_vars); - } + VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; } @@ -1540,6 +1538,16 @@ symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh) return 1; } +static int +symtable_visit_withitem(struct symtable *st, withitem_ty item) +{ + VISIT(st, expr, item->context_expr); + if (item->optional_vars) { + VISIT(st, expr, item->optional_vars); + } + return 1; +} + static int symtable_visit_alias(struct symtable *st, alias_ty a) -- cgit v0.12 From 7f670e5dad6837d1684a0228eeb6449dea5fe713 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 14:01:01 -0500 Subject: bump ast version --- Python/Python-ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 2364db3..f076c7e 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2,7 +2,7 @@ /* - __version__ 0daa6ba25d9b. + __version__ 9b11cc4e2918. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -6818,7 +6818,7 @@ PyInit__ast(void) NULL; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return NULL; - if (PyModule_AddStringConstant(m, "__version__", "0daa6ba25d9b") < 0) + if (PyModule_AddStringConstant(m, "__version__", "9b11cc4e2918") < 0) return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; -- cgit v0.12 From 74897ba46fe7a0be6d700f647c514298a0de06ba Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 14:10:24 -0500 Subject: fix indentation --- Python/compile.c | 100 +++++++++++++++++++++++++++---------------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 19b2add..04c65b4 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1977,7 +1977,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_use_next_block(c, except); for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.TryExcept.handlers, i); + s->v.TryExcept.handlers, i); if (!handler->v.ExceptHandler.type && i < n-1) return compiler_error(c, "default 'except:' must be last"); c->u->u_lineno_set = 0; @@ -1993,70 +1993,70 @@ compiler_try_except(struct compiler *c, stmt_ty s) } ADDOP(c, POP_TOP); if (handler->v.ExceptHandler.name) { - basicblock *cleanup_end, *cleanup_body; + basicblock *cleanup_end, *cleanup_body; - cleanup_end = compiler_new_block(c); - cleanup_body = compiler_new_block(c); - if(!(cleanup_end || cleanup_body)) - return 0; + cleanup_end = compiler_new_block(c); + cleanup_body = compiler_new_block(c); + if(!(cleanup_end || cleanup_body)) + return 0; - compiler_nameop(c, handler->v.ExceptHandler.name, Store); - ADDOP(c, POP_TOP); + compiler_nameop(c, handler->v.ExceptHandler.name, Store); + ADDOP(c, POP_TOP); - /* - try: - # body - except type as name: - try: - # body - finally: - name = None - del name - */ + /* + try: + # body + except type as name: + try: + # body + finally: + name = None + del name + */ - /* second try: */ - ADDOP_JREL(c, SETUP_FINALLY, cleanup_end); - compiler_use_next_block(c, cleanup_body); - if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) - return 0; + /* second try: */ + ADDOP_JREL(c, SETUP_FINALLY, cleanup_end); + compiler_use_next_block(c, cleanup_body); + if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) + return 0; - /* second # body */ - VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); - ADDOP(c, POP_BLOCK); - ADDOP(c, POP_EXCEPT); - compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); + /* second # body */ + VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); + ADDOP(c, POP_BLOCK); + ADDOP(c, POP_EXCEPT); + compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); - /* finally: */ - ADDOP_O(c, LOAD_CONST, Py_None, consts); - compiler_use_next_block(c, cleanup_end); - if (!compiler_push_fblock(c, FINALLY_END, cleanup_end)) - return 0; + /* finally: */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + compiler_use_next_block(c, cleanup_end); + if (!compiler_push_fblock(c, FINALLY_END, cleanup_end)) + return 0; - /* name = None */ - ADDOP_O(c, LOAD_CONST, Py_None, consts); - compiler_nameop(c, handler->v.ExceptHandler.name, Store); + /* name = None */ + ADDOP_O(c, LOAD_CONST, Py_None, consts); + compiler_nameop(c, handler->v.ExceptHandler.name, Store); - /* del name */ - compiler_nameop(c, handler->v.ExceptHandler.name, Del); + /* del name */ + compiler_nameop(c, handler->v.ExceptHandler.name, Del); - ADDOP(c, END_FINALLY); - compiler_pop_fblock(c, FINALLY_END, cleanup_end); + ADDOP(c, END_FINALLY); + compiler_pop_fblock(c, FINALLY_END, cleanup_end); } else { - basicblock *cleanup_body; + basicblock *cleanup_body; - cleanup_body = compiler_new_block(c); - if(!cleanup_body) - return 0; + cleanup_body = compiler_new_block(c); + if(!cleanup_body) + return 0; ADDOP(c, POP_TOP); - ADDOP(c, POP_TOP); - compiler_use_next_block(c, cleanup_body); - if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) - return 0; + ADDOP(c, POP_TOP); + compiler_use_next_block(c, cleanup_body); + if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body)) + return 0; VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body); - ADDOP(c, POP_EXCEPT); - compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); + ADDOP(c, POP_EXCEPT); + compiler_pop_fblock(c, FINALLY_TRY, cleanup_body); } ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, except); -- cgit v0.12 From 0a5dad9ef1e3203f3ab7a15ffe051299081ef672 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 14:17:04 -0500 Subject: fix spacing --- Python/compile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 04c65b4..0a85bf8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1997,7 +1997,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) cleanup_end = compiler_new_block(c); cleanup_body = compiler_new_block(c); - if(!(cleanup_end || cleanup_body)) + if (!(cleanup_end || cleanup_body)) return 0; compiler_nameop(c, handler->v.ExceptHandler.name, Store); @@ -2046,7 +2046,7 @@ compiler_try_except(struct compiler *c, stmt_ty s) basicblock *cleanup_body; cleanup_body = compiler_new_block(c); - if(!cleanup_body) + if (!cleanup_body) return 0; ADDOP(c, POP_TOP); -- cgit v0.12 From aeabd5f29c6ed47986fcdfc88417e8f386824f52 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 15:02:03 -0500 Subject: add some with syntax examples --- Lib/test/test_ast.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index da07060..fde2cfe 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -38,6 +38,9 @@ exec_tests = [ "while v:pass", # If "if v:pass", + # With + "with x as y: pass", + "with x as y, z as q: pass", # Raise "raise Exception('string')", # TryExcept @@ -341,6 +344,8 @@ exec_results = [ ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), ('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), ('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), -- cgit v0.12 From 9a7c524dc613595e0d88b3e22b0e49b69467ffc4 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 28 May 2011 00:19:56 -0700 Subject: Issue #9670: Increase the default stack size for secondary threads on Mac OS X and FreeBSD to reduce the chances of a crash instead of a "maximum recursion depth" RuntimeError exception. (patch by Ronald Oussoren) --- Lib/test/test_threading.py | 31 +++++++++++++++++++++++++++++++ Misc/NEWS | 5 +++++ Python/thread_pthread.h | 12 ++++++++++++ 3 files changed, 48 insertions(+) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f977a7f..63ef7b9 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -673,6 +673,37 @@ class SemaphoreTests(lock_tests.SemaphoreTests): class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): semtype = staticmethod(threading.BoundedSemaphore) + @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RuntimeError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error") + self.assertEqual(data, expected_output) def test_main(): test.support.run_unittest(LockTests, RLockTests, EventTests, diff --git a/Misc/NEWS b/Misc/NEWS index 015018c..dada5c2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ What's New in Python 3.1.4? Core and Builtins ----------------- +- Issue #9670: Increase the default stack size for secondary threads on + Mac OS X and FreeBSD to reduce the chances of a crash instead of a + "maximum recursion depth" RuntimeError exception. + (patch by Ronald Oussoren) + - Correct lookup of __dir__ on objects. Among other things, this causes errors besides AttributeError found on lookup to be propagated. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 32fd2d0..244ebac 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -18,6 +18,18 @@ #ifndef THREAD_STACK_SIZE #define THREAD_STACK_SIZE 0 /* use default stack size */ #endif + +#if (defined(__APPLE__) || defined(__FreeBSD__)) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 + /* The default stack size for new threads on OSX is small enough that + * we'll get hard crashes instead of 'maximum recursion depth exceeded' + * exceptions. + * + * The default stack size below is the minimal stack size where a + * simple recursive function doesn't cause a hard crash. + */ +#undef THREAD_STACK_SIZE +#define THREAD_STACK_SIZE 0x400000 +#endif /* for safety, ensure a viable minimum stacksize */ #define THREAD_STACK_MIN 0x8000 /* 32kB */ #else /* !_POSIX_THREAD_ATTR_STACKSIZE */ -- cgit v0.12 From b8e59f77e65ba4caeda8d910bd66df01a468cbea Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 28 May 2011 02:19:19 -0700 Subject: Issue #985064: Make plistlib more resilient to faulty input plists. Patch by Mher Movsisyan. --- Lib/plistlib.py | 59 +++++++++++++++++++++++++++++------------------ Lib/test/test_plistlib.py | 26 +++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/Lib/plistlib.py b/Lib/plistlib.py index fbba791..fe622ad 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -68,13 +68,15 @@ def readPlist(pathOrFile): usually is a dictionary). """ didOpen = False - if isinstance(pathOrFile, str): - pathOrFile = open(pathOrFile, 'rb') - didOpen = True - p = PlistParser() - rootObject = p.parse(pathOrFile) - if didOpen: - pathOrFile.close() + try: + if isinstance(pathOrFile, str): + pathOrFile = open(pathOrFile, 'rb') + didOpen = True + p = PlistParser() + rootObject = p.parse(pathOrFile) + finally: + if didOpen: + pathOrFile.close() return rootObject @@ -83,15 +85,17 @@ def writePlist(rootObject, pathOrFile): file name or a (writable) file object. """ didOpen = False - if isinstance(pathOrFile, str): - pathOrFile = open(pathOrFile, 'wb') - didOpen = True - writer = PlistWriter(pathOrFile) - writer.writeln("") - writer.writeValue(rootObject) - writer.writeln("") - if didOpen: - pathOrFile.close() + try: + if isinstance(pathOrFile, str): + pathOrFile = open(pathOrFile, 'wb') + didOpen = True + writer = PlistWriter(pathOrFile) + writer.writeln("") + writer.writeValue(rootObject) + writer.writeln("") + finally: + if didOpen: + pathOrFile.close() def readPlistFromBytes(data): @@ -352,7 +356,6 @@ class Data: def __repr__(self): return "%s(%s)" % (self.__class__.__name__, repr(self.data)) - class PlistParser: def __init__(self): @@ -362,11 +365,11 @@ class PlistParser: def parse(self, fileobj): from xml.parsers.expat import ParserCreate - parser = ParserCreate() - parser.StartElementHandler = self.handleBeginElement - parser.EndElementHandler = self.handleEndElement - parser.CharacterDataHandler = self.handleData - parser.ParseFile(fileobj) + self.parser = ParserCreate() + self.parser.StartElementHandler = self.handleBeginElement + self.parser.EndElementHandler = self.handleEndElement + self.parser.CharacterDataHandler = self.handleData + self.parser.ParseFile(fileobj) return self.root def handleBeginElement(self, element, attrs): @@ -385,12 +388,18 @@ class PlistParser: def addObject(self, value): if self.currentKey is not None: + if not isinstance(self.stack[-1], type({})): + raise ValueError("unexpected element at line %d" % + self.parser.CurrentLineNumber) self.stack[-1][self.currentKey] = value self.currentKey = None elif not self.stack: # this is the root object self.root = value else: + if not isinstance(self.stack[-1], type([])): + raise ValueError("unexpected element at line %d" % + self.parser.CurrentLineNumber) self.stack[-1].append(value) def getData(self): @@ -405,9 +414,15 @@ class PlistParser: self.addObject(d) self.stack.append(d) def end_dict(self): + if self.currentKey: + raise ValueError("missing value for key '%s' at line %d" % + (self.currentKey,self.parser.CurrentLineNumber)) self.stack.pop() def end_key(self): + if self.currentKey or not isinstance(self.stack[-1], type({})): + raise ValueError("unexpected key at line %d" % + self.parser.CurrentLineNumber) self.currentKey = self.getData() def begin_array(self, attrs): diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index b9a46b7..ccda920 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -175,6 +175,32 @@ class TestPlistlib(unittest.TestCase): self.assertEqual(test1, result1) self.assertEqual(test2, result2) + def test_invalidarray(self): + for i in ["key inside an array", + "key inside an array23", + "key inside an array3"]: + self.assertRaises(ValueError, plistlib.readPlistFromBytes, + ("%s"%i).encode()) + + def test_invaliddict(self): + for i in ["kcompound key", + "single key", + "missing key", + "k1v15.3" + "k1k2double key"]: + self.assertRaises(ValueError, plistlib.readPlistFromBytes, + ("%s"%i).encode()) + self.assertRaises(ValueError, plistlib.readPlistFromBytes, + ("%s"%i).encode()) + + def test_invalidinteger(self): + self.assertRaises(ValueError, plistlib.readPlistFromBytes, + b"not integer") + + def test_invalidreal(self): + self.assertRaises(ValueError, plistlib.readPlistFromBytes, + b"not real") + def test_main(): support.run_unittest(TestPlistlib) diff --git a/Misc/ACKS b/Misc/ACKS index 80fda10..c698d73 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -563,6 +563,7 @@ Skip Montanaro Paul Moore Derek Morr James A Morrison +Mher Movsisyan Sjoerd Mullender Sape Mullender Michael Muller diff --git a/Misc/NEWS b/Misc/NEWS index dada5c2..ec1bbf2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -80,6 +80,9 @@ Core and Builtins Library ------- +- Issue #985064: Make plistlib more resilient to faulty input plists. + Patch by Mher Movsisyan. + - Issue #12175: RawIOBase.readall() now returns None if read() returns None. - Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError -- cgit v0.12 From 6e5fd04ce21b421ff975c93611ff6c81547ce183 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sat, 28 May 2011 05:59:55 -0700 Subject: Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds, ensure "make install" creates symlinks in --prefix bin for the "-32" files in the framework bin directory like the installer does. --- Mac/Makefile.in | 13 +++++++++++++ Misc/NEWS | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 5e57b57..8a62a90 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -76,6 +76,13 @@ installunixtools: do \ ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\ done +ifneq ($(LIPO_32BIT_FLAGS),) + for fn in python3-32 pythonw3-32 \ + python$(VERSION)-32 pythonw$(VERSION)-32 ;\ + do \ + ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\ + done +endif # @@ -90,6 +97,12 @@ altinstallunixtools: do \ ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\ done +ifneq ($(LIPO_32BIT_FLAGS),) + for fn in python$(VERSION)-32 pythonw$(VERSION)-32 ;\ + do \ + ln -fs "$(prefix)/bin/$${fn}" "$(DESTDIR)$(FRAMEWORKUNIXTOOLSPREFIX)/bin/$${fn}" ;\ + done +endif pythonw: $(srcdir)/Tools/pythonw.c Makefile $(CC) $(LDFLAGS) -DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' -o $@ $(srcdir)/Tools/pythonw.c -I.. -I$(srcdir)/../Include ../$(PYTHONFRAMEWORK).framework/Versions/$(VERSION)/$(PYTHONFRAMEWORK) diff --git a/Misc/NEWS b/Misc/NEWS index df6933a..4962721 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -44,6 +44,12 @@ Library - Issue #12065: connect_ex() on an SSL socket now returns the original errno when the socket's timeout expires (it used to return None). +Build +----- + +- Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds, + ensure "make install" creates symlinks in --prefix bin for the "-32" + files in the framework bin directory like the installer does. What's New in Python 3.2.1 release candidate 1? -- cgit v0.12 From c80504fb6954c8aabc97e79116fd93bb8d55a03c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 May 2011 09:06:02 -0700 Subject: Fix ProcessTestCasePOSIXPurePython to test the module from import when _posixsubprocess doesn't exist rather than simply stubbing it out after the fact. This adds coverage for the RuntimeWarning as well as using the pure python _create_pipe instead of using _posixsubprocess.cloexec_pipe unintentionally with the pure python code. Ironically: I don't think any platform should ever actually _use_ the pure Python subprocess code on POSIX platforms anymore. This at least tests it properly in this stable branch. The pure python code for this is likely to be removed in 3.3. --- Lib/test/test_subprocess.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 776e143..08d2653 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1498,15 +1498,23 @@ class ProcessTestCaseNoPoll(ProcessTestCase): @unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), "_posixsubprocess extension module not found.") class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): - def setUp(self): - subprocess._posixsubprocess = None - ProcessTestCase.setUp(self) - POSIXProcessTestCase.setUp(self) - - def tearDown(self): - subprocess._posixsubprocess = sys.modules['_posixsubprocess'] - POSIXProcessTestCase.tearDown(self) - ProcessTestCase.tearDown(self) + @classmethod + def setUpClass(cls): + global subprocess + assert subprocess._posixsubprocess + # Reimport subprocess while forcing _posixsubprocess to not exist. + with support.check_warnings(('.*_posixsubprocess .* not being used.*', + RuntimeWarning)): + subprocess = support.import_fresh_module( + 'subprocess', blocked=['_posixsubprocess']) + assert not subprocess._posixsubprocess + + @classmethod + def tearDownClass(cls): + global subprocess + # Reimport subprocess as it should be, restoring order to the universe. + subprocess = support.import_fresh_module('subprocess') + assert subprocess._posixsubprocess class HelperFunctionTests(unittest.TestCase): -- cgit v0.12 From 7439e7b7cec2727f5f891dc5ea38acf8b239fb8e Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 May 2011 09:06:02 -0700 Subject: Fix ProcessTestCasePOSIXPurePython to test the module from import when _posixsubprocess doesn't exist rather than simply stubbing it out after the fact. This adds coverage for the RuntimeWarning as well as using the pure python _create_pipe instead of using _posixsubprocess.cloexec_pipe unintentionally with the pure python code. Ironically: I don't think any platform should ever actually _use_ the pure Python subprocess code on POSIX platforms anymore. This at least tests it properly in this stable branch. The pure python code for this is likely to be removed in 3.3. --- Lib/test/test_subprocess.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 176ff10..ad89864 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1399,15 +1399,23 @@ class ProcessTestCaseNoPoll(ProcessTestCase): @unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), "_posixsubprocess extension module not found.") class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): - def setUp(self): - subprocess._posixsubprocess = None - ProcessTestCase.setUp(self) - POSIXProcessTestCase.setUp(self) - - def tearDown(self): - subprocess._posixsubprocess = sys.modules['_posixsubprocess'] - POSIXProcessTestCase.tearDown(self) - ProcessTestCase.tearDown(self) + @classmethod + def setUpClass(cls): + global subprocess + assert subprocess._posixsubprocess + # Reimport subprocess while forcing _posixsubprocess to not exist. + with support.check_warnings(('.*_posixsubprocess .* not being used.*', + RuntimeWarning)): + subprocess = support.import_fresh_module( + 'subprocess', blocked=['_posixsubprocess']) + assert not subprocess._posixsubprocess + + @classmethod + def tearDownClass(cls): + global subprocess + # Reimport subprocess as it should be, restoring order to the universe. + subprocess = support.import_fresh_module('subprocess') + assert subprocess._posixsubprocess class HelperFunctionTests(unittest.TestCase): -- cgit v0.12 From 59fd1bfcc1dc04b346141da6eb15de9c64d5d4ea Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 May 2011 09:32:39 -0700 Subject: The _posixsubprocess module is now required on POSIX. Remove the pure Python POSIX subprocess implementation. If non-CPython VMs (are there any for 3.x yet?) were somehow depending on this, they already have the exact same set of problems with Python code being executed after os.fork() that _posixsubprocess was written to deal with. They should implement an equivalent outside of Python. --- Lib/subprocess.py | 188 +++++++------------------------------------- Lib/test/test_subprocess.py | 22 ------ 2 files changed, 28 insertions(+), 182 deletions(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index c5128d8..6e0cf06 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -397,39 +397,14 @@ if mswindows: else: import select _has_poll = hasattr(select, 'poll') - import fcntl - import pickle - - try: - import _posixsubprocess - except ImportError: - _posixsubprocess = None - warnings.warn("The _posixsubprocess module is not being used. " - "Child process reliability may suffer if your " - "program uses threads.", RuntimeWarning) + import _posixsubprocess + _create_pipe = _posixsubprocess.cloexec_pipe # When select or poll has indicated that the file is writable, # we can write up to _PIPE_BUF bytes without risk of blocking. # POSIX defines PIPE_BUF as >= 512. _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) - _FD_CLOEXEC = getattr(fcntl, 'FD_CLOEXEC', 1) - - def _set_cloexec(fd, cloexec): - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if cloexec: - fcntl.fcntl(fd, fcntl.F_SETFD, old | _FD_CLOEXEC) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~_FD_CLOEXEC) - - if _posixsubprocess: - _create_pipe = _posixsubprocess.cloexec_pipe - else: - def _create_pipe(): - fds = os.pipe() - _set_cloexec(fds[0], True) - _set_cloexec(fds[1], True) - return fds __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput", "getoutput", "check_output", "CalledProcessError", "DEVNULL"] @@ -1267,140 +1242,33 @@ class Popen(object): errpipe_read, errpipe_write = _create_pipe() try: try: - - if _posixsubprocess: - # We must avoid complex work that could involve - # malloc or free in the child process to avoid - # potential deadlocks, thus we do all this here. - # and pass it to fork_exec() - - if env: - env_list = [os.fsencode(k) + b'=' + os.fsencode(v) - for k, v in env.items()] - else: - env_list = None # Use execv instead of execve. - executable = os.fsencode(executable) - if os.path.dirname(executable): - executable_list = (executable,) - else: - # This matches the behavior of os._execvpe(). - executable_list = tuple( - os.path.join(os.fsencode(dir), executable) - for dir in os.get_exec_path(env)) - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self.pid = _posixsubprocess.fork_exec( - args, executable_list, - close_fds, sorted(fds_to_keep), cwd, env_list, - p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite, - errpipe_read, errpipe_write, - restore_signals, start_new_session, preexec_fn) + # We must avoid complex work that could involve + # malloc or free in the child process to avoid + # potential deadlocks, thus we do all this here. + # and pass it to fork_exec() + + if env: + env_list = [os.fsencode(k) + b'=' + os.fsencode(v) + for k, v in env.items()] else: - # Pure Python implementation: It is not thread safe. - # This implementation may deadlock in the child if your - # parent process has any other threads running. - - gc_was_enabled = gc.isenabled() - # Disable gc to avoid bug where gc -> file_dealloc -> - # write to stderr -> hang. See issue1336 - gc.disable() - try: - self.pid = os.fork() - except: - if gc_was_enabled: - gc.enable() - raise - self._child_created = True - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite != -1: - os.close(p2cwrite) - if c2pread != -1: - os.close(c2pread) - if errread != -1: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - def _dup2(a, b): - # dup2() removes the CLOEXEC flag but - # we must do it ourselves if dup2() - # would be a no-op (issue #10806). - if a == b: - _set_cloexec(a, False) - elif a != -1: - os.dup2(a, b) - _dup2(p2cread, 0) - _dup2(c2pwrite, 1) - _dup2(errwrite, 2) - - # Close pipe fds. Make sure we don't close the - # same fd more than once, or standard fds. - closed = set() - for fd in [p2cread, c2pwrite, errwrite]: - if fd > 2 and fd not in closed: - os.close(fd) - closed.add(fd) - - # Close all other fds, if asked for - if close_fds: - fds_to_keep = set(pass_fds) - fds_to_keep.add(errpipe_write) - self._close_fds(fds_to_keep) - - - if cwd is not None: - os.chdir(cwd) - - # This is a copy of Python/pythonrun.c - # _Py_RestoreSignals(). If that were exposed - # as a sys._py_restoresignals func it would be - # better.. but this pure python implementation - # isn't likely to be used much anymore. - if restore_signals: - signals = ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') - for sig in signals: - if hasattr(signal, sig): - signal.signal(getattr(signal, sig), - signal.SIG_DFL) - - if start_new_session and hasattr(os, 'setsid'): - os.setsid() - - if preexec_fn: - preexec_fn() - - if env is None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - try: - exc_type, exc_value = sys.exc_info()[:2] - if isinstance(exc_value, OSError): - errno_num = exc_value.errno - else: - errno_num = 0 - message = '%s:%x:%s' % (exc_type.__name__, - errno_num, exc_value) - message = message.encode(errors="surrogatepass") - os.write(errpipe_write, message) - except Exception: - # We MUST not allow anything odd happening - # above to prevent us from exiting below. - pass - - # This exitcode won't be reported to applications - # so it really doesn't matter what we return. - os._exit(255) - - # Parent - if gc_was_enabled: - gc.enable() + env_list = None # Use execv instead of execve. + executable = os.fsencode(executable) + if os.path.dirname(executable): + executable_list = (executable,) + else: + # This matches the behavior of os._execvpe(). + executable_list = tuple( + os.path.join(os.fsencode(dir), executable) + for dir in os.get_exec_path(env)) + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self.pid = _posixsubprocess.fork_exec( + args, executable_list, + close_fds, sorted(fds_to_keep), cwd, env_list, + p2cread, p2cwrite, c2pread, c2pwrite, + errread, errwrite, + errpipe_read, errpipe_write, + restore_signals, start_new_session, preexec_fn) finally: # be sure the FD is closed no matter what os.close(errpipe_write) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 08d2653..99d7a92 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1495,28 +1495,6 @@ class ProcessTestCaseNoPoll(ProcessTestCase): ProcessTestCase.tearDown(self) -@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), - "_posixsubprocess extension module not found.") -class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): - @classmethod - def setUpClass(cls): - global subprocess - assert subprocess._posixsubprocess - # Reimport subprocess while forcing _posixsubprocess to not exist. - with support.check_warnings(('.*_posixsubprocess .* not being used.*', - RuntimeWarning)): - subprocess = support.import_fresh_module( - 'subprocess', blocked=['_posixsubprocess']) - assert not subprocess._posixsubprocess - - @classmethod - def tearDownClass(cls): - global subprocess - # Reimport subprocess as it should be, restoring order to the universe. - subprocess = support.import_fresh_module('subprocess') - assert subprocess._posixsubprocess - - class HelperFunctionTests(unittest.TestCase): @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") def test_eintr_retry_call(self): -- cgit v0.12 From 8281ba8bcb6ba9313074769337eb2a9f875c0ed9 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 28 May 2011 10:00:14 -0700 Subject: follow on to the last commit, remove the final POSIXPurePythonTestCase reference. (the difference between running test_subprocess.py directly rather than via regrtest.py) --- Lib/test/test_subprocess.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 99d7a92..2eeff72 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1603,7 +1603,6 @@ def test_main(): unit_tests = (ProcessTestCase, POSIXProcessTestCase, Win32ProcessTestCase, - ProcessTestCasePOSIXPurePython, CommandTests, ProcessTestCaseNoPoll, HelperFunctionTests, -- cgit v0.12 From 489c8ff26b673553f568266c732182346ee74acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sat, 28 May 2011 23:32:50 +0200 Subject: Fix test_build_py when sys.dont_write_bytecode is true (#9831). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests now pass all combinations of -O/-OO and -B. See also #7071 and #6292 for previous variations on the same theme. test_versionpredicate needs a skip when sys.flags.optimize is true, but I don’t know how to make that work with a DocTestSuite. --- Lib/distutils/tests/test_build_py.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/distutils/tests/test_build_py.py b/Lib/distutils/tests/test_build_py.py index da3232c..4e46339 100644 --- a/Lib/distutils/tests/test_build_py.py +++ b/Lib/distutils/tests/test_build_py.py @@ -57,11 +57,15 @@ class BuildPyTestCase(support.TempdirManager, self.assertEqual(len(cmd.get_outputs()), 3) pkgdest = os.path.join(destination, "pkg") files = os.listdir(pkgdest) - self.assertTrue("__init__.py" in files) - self.assertTrue("__init__.pyc" in files) - self.assertTrue("README.txt" in files) - - def test_empty_package_dir (self): + self.assertIn("__init__.py", files) + self.assertIn("README.txt", files) + # XXX even with -O, distutils writes pyc, not pyo; bug? + if sys.dont_write_bytecode: + self.assertNotIn("__init__.pyc", files) + else: + self.assertIn("__init__.pyc", files) + + def test_empty_package_dir(self): # See SF 1668596/1720897. cwd = os.getcwd() @@ -109,7 +113,7 @@ class BuildPyTestCase(support.TempdirManager, finally: sys.dont_write_bytecode = old_dont_write_bytecode - self.assertTrue('byte-compiling is disabled' in self.logs[0][1]) + self.assertIn('byte-compiling is disabled', self.logs[0][1]) def test_suite(): return unittest.makeSuite(BuildPyTestCase) -- cgit v0.12 From 000893fab2c03c8947c9c34422e44086d88c9120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 00:14:45 +0200 Subject: Minor touch-ups in distutils.cmd.Command doc --- Doc/distutils/apiref.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 81de1ad..74446bc 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -21,7 +21,7 @@ setup script). Indirectly provides the :class:`distutils.dist.Distribution` and .. function:: setup(arguments) The basic do-everything function that does most everything you could ever ask - for from a Distutils method. See XXXXX + for from a Distutils method. The setup function takes a large number of arguments. These are laid out in the following table. @@ -1759,7 +1759,7 @@ Subclasses of :class:`Command` must define the following methods. predicate)``, with *command_name* a string and *predicate* a function, a string or ``None``. *predicate* is a method of the parent command that determines whether the corresponding command is applicable in the current - situation. (E.g. we ``install_headers`` is only applicable if we have any C + situation. (E.g. ``install_headers`` is only applicable if we have any C header files to install.) If *predicate* is ``None``, that command is always applicable. -- cgit v0.12 From 456ae893d7a0b40a5346cbba7771127aca2113e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 00:22:06 +0200 Subject: Re-add missing doc stub for the distutils check command --- Doc/distutils/apiref.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 74446bc..712fffb 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -2006,3 +2006,17 @@ The ``register`` command registers the package with the Python Package Index. This is described in more detail in :pep:`301`. .. % todo + + +:mod:`distutils.command.check` --- Check the meta-data of a package +=================================================================== + +.. module:: distutils.command.check + :synopsis: Check the metadata of a package + + +The ``check`` command performs some tests on the meta-data of a package. +For example, it verifies that all required meta-data are provided as +the arguments passed to the :func:`setup` function. + +.. % todo -- cgit v0.12 From 7f9b37be4510e701184896dd22994412455e6e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 02:59:52 +0200 Subject: =?UTF-8?q?Port=20r86353=20to=20packaging=20(#10359:=20=E2=80=9C;?= =?UTF-8?q?=E2=80=9D=20after=20function=20definition=20is=20invalid=20in?= =?UTF-8?q?=20ISO=20C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/packaging/tests/test_command_build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/packaging/tests/test_command_build_ext.py b/Lib/packaging/tests/test_command_build_ext.py index fba27c7..a7856d2 100644 --- a/Lib/packaging/tests/test_command_build_ext.py +++ b/Lib/packaging/tests/test_command_build_ext.py @@ -265,7 +265,7 @@ class BuildExtTestCase(support.TempdirManager, def test_get_outputs(self): tmp_dir = self.mkdtemp() c_file = os.path.join(tmp_dir, 'foo.c') - self.write_file(c_file, 'void PyInit_foo(void) {};\n') + self.write_file(c_file, 'void PyInit_foo(void) {}\n') ext = Extension('foo', [c_file], optional=False) dist = Distribution({'name': 'xx', 'ext_modules': [ext]}) -- cgit v0.12 From bc57789dafaadd3652b7f02a2b17684b6c74055e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 03:24:45 +0200 Subject: Improve example for crypt module. No string exceptions.. --- Doc/library/crypt.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index ec5a4b0..27236c1 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -131,18 +131,20 @@ Examples A simple example illustrating typical use:: - import crypt, getpass, pwd + import pwd + import crypt + import getpass def login(): - username = input('Python login:') + username = input('Python login: ') cryptedpasswd = pwd.getpwnam(username)[1] if cryptedpasswd: if cryptedpasswd == 'x' or cryptedpasswd == '*': - raise "Sorry, currently no support for shadow passwords" + raise ValueError('no support for shadow passwords') cleartext = getpass.getpass() return crypt.crypt(cleartext, cryptedpasswd) == cryptedpasswd else: - return 1 + return True To generate a hash of a password using the strongest available method and check it against the original:: @@ -151,4 +153,4 @@ check it against the original:: hashed = crypt.crypt(plaintext) if hashed != crypt.crypt(plaintext, hashed): - raise "Hashed version doesn't validate against original" + raise ValueError("hashed version doesn't validate against original") -- cgit v0.12 From f21563160e66069bac9c82186e17cfba7058ec52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 03:27:48 +0200 Subject: Put version* directives at the right scope --- Doc/library/crypt.rst | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 27236c1..1ba2ed3 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -29,6 +29,8 @@ this module. Hashing Methods --------------- +.. versionadded:: 3.3 + The :mod:`crypt` module defines the list of hashing methods (not all methods are available on all platforms): @@ -37,33 +39,26 @@ are available on all platforms): A Modular Crypt Format method with 16 character salt and 86 character hash. This is the strongest method. - .. versionadded:: 3.3 - .. data:: METHOD_SHA256 Another Modular Crypt Format method with 16 character salt and 43 character hash. - .. versionadded:: 3.3 - .. data:: METHOD_MD5 Another Modular Crypt Format method with 8 character salt and 22 character hash. - .. versionadded:: 3.3 - .. data:: METHOD_CRYPT The traditional method with a 2 character salt and 13 characters of hash. This is the weakest method. - .. versionadded:: 3.3 - Module Attributes ----------------- +.. versionadded:: 3.3 .. attribute:: methods @@ -71,8 +66,6 @@ Module Attributes ``crypt.METHOD_*`` objects. This list is sorted from strongest to weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``. - .. versionadded:: 3.3 - Module Functions ---------------- @@ -108,9 +101,8 @@ The :mod:`crypt` module defines the following functions: different sizes in the *salt*, it is recommended to use the full crypted password as salt when checking for a password. -.. versionchanged:: 3.3 - Before version 3.3, *salt* must be specified as a string and cannot - accept ``crypt.METHOD_*`` values (which don't exist anyway). + .. versionchanged:: 3.3 + Accept ``crypt.METHOD_*`` values in addition to strings for *salt*. .. function:: mksalt(method=None) @@ -124,7 +116,7 @@ The :mod:`crypt` module defines the following functions: 16 random characters from the set ``[./a-zA-Z0-9]``, suitable for passing as the *salt* argument to :func:`crypt`. -.. versionadded:: 3.3 + .. versionadded:: 3.3 Examples -------- -- cgit v0.12 From f0ab5d6e7f46ec8d06d2a62db0e316cd7443238f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 03:43:59 +0200 Subject: Fix typos in Misc/NEWS --- Misc/NEWS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index ce6c964..b561e9b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -285,7 +285,7 @@ Library - Issue #7311: fix html.parser to accept non-ASCII attribute values. - Issue #11605: email.parser.BytesFeedParser was incorrectly converting - multipart subpararts with an 8bit CTE into unicode instead of preserving the + multipart subparts with an 8-bit CTE into unicode instead of preserving the bytes. - Issue #10963: Ensure that subprocess.communicate() never raises EPIPE. @@ -482,7 +482,7 @@ Tests - Issue #11577: improve test coverage of binhex.py. Patch by Arkady Koplyarov. -- Issue #11578: added test for the timeit module. Patch Michael Henry. +- Issue #11578: added test for the timeit module. Patch by Michael Henry. - Issue #11503: improve test coverage of posixpath.py. Patch by Evan Dandrea. @@ -782,10 +782,10 @@ Library comparisons that could lead to infinite recursion. - Issue #10686: the email package now :rfc:`2047`\ -encodes headers with - non-ASCII bytes (parsed by a Bytes Parser) when doing conversion to 7bit-clean + non-ASCII bytes (parsed by a BytesParser) when doing conversion to 7bit-clean presentation, instead of replacing them with ?s. -- email.header.Header was incorrectly encoding folding white space when +- email.header.Header was incorrectly encoding folding whitespace when rfc2047-encoding header values with embedded newlines, leaving them without folding whitespace. It now uses the continuation_ws, as it does for continuation lines that it creates itself. -- cgit v0.12 From 6a21f5527cab884e2d7dd4db025b5f9a7bd48dd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 03:46:31 +0200 Subject: =?UTF-8?q?Don=E2=80=99t=20misuse=20=E2=80=9Cpackage=20data?= =?UTF-8?q?=E2=80=9D=20in=20pprint=20example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/library/pprint.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 7bebd8c..4169b64 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -192,7 +192,7 @@ Example ------- To demonstrate several uses of the :func:`pprint` function and its parameters, -let's fetch information about a package from PyPI:: +let's fetch information about a project from PyPI:: >>> import json >>> import pprint @@ -200,8 +200,8 @@ let's fetch information about a package from PyPI:: >>> with urlopen('http://pypi.python.org/pypi/configparser/json') as url: ... http_info = url.info() ... raw_data = url.read().decode(http_info.get_content_charset()) - >>> package_data = json.loads(raw_data) - >>> result = {'headers': http_info.items(), 'body': package_data} + >>> project_info = json.loads(raw_data) + >>> result = {'headers': http_info.items(), 'body': project_info} In its basic form, :func:`pprint` shows the whole object:: -- cgit v0.12 From 87418afb3b6c6ef220df439004501093609abb2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sun, 29 May 2011 03:48:49 +0200 Subject: Fix typo in 0311f62714f7 --- Doc/c-api/veryhigh.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index d1940a8..73cd1d2 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -34,7 +34,7 @@ the same library that the Python runtime is using. according to the user's locale). It is important to note that the argument list may be modified (but the contents of the strings pointed to by the argument list are not). The return value will be - ```0``` if the interpreter exits normally (ie, without an + ``0`` if the interpreter exits normally (i.e. without an exception), ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter list does not represent a valid Python command line. -- cgit v0.12 From d5a91961dd0e501a448ec3a048146132121c24c6 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 29 May 2011 02:39:02 -0700 Subject: Issue #12205: Fix test_subprocess failure due to uninstalled test data. --- Makefile.pre.in | 2 +- Misc/NEWS | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 270dfb4..ed1dc33 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -841,7 +841,7 @@ MACHDEPS= $(PLATDIR) $(EXTRAPLATDIR) XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ tkinter/test/test_ttk site-packages test test/data \ - test/cjkencodings test/decimaltestdata \ + test/cjkencodings test/decimaltestdata test/subprocessdata \ test/tracedmodules \ encodings \ email email/mime email/test email/test/data \ diff --git a/Misc/NEWS b/Misc/NEWS index ec1bbf2..f04c643 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -421,6 +421,8 @@ Build Tests ----- +- Issue #12205: Fix test_subprocess failure due to uninstalled test data. + - Issue #11614: import __hello__ prints "Hello World!". Patch written by Andreas Stührk. -- cgit v0.12 From 2d517218320af6f752eaf1fbd96466e8d13d2d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 29 May 2011 16:36:44 +0200 Subject: Issue #12196: Add PIPE_MAX_SIZE to test.support, constant larger than the underlying OS pipe buffer size. --- Lib/test/support.py | 9 ++++++++- Lib/test/test_io.py | 2 +- Lib/test/test_subprocess.py | 11 ++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Lib/test/support.py b/Lib/test/support.py index b03069c..6724e9b 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -48,7 +48,7 @@ __all__ = [ "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail", "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754", "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", - "import_fresh_module", "requires_zlib" + "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE" ] class Error(Exception): @@ -409,6 +409,13 @@ def _is_ipv6_enabled(): IPV6_ENABLED = _is_ipv6_enabled() + +# A constant likely larger than the underlying OS pipe buffer size. +# Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe +# buffer size: take 1M to be sure. +PIPE_MAX_SIZE = 1024 * 1024 + + # decorator for skipping tests on non-IEEE 754 platforms requires_IEEE_754 = unittest.skipUnless( float.__getformat__("double").startswith("IEEE"), diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c0dc6ec..0ffb4a1 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2683,7 +2683,7 @@ class SignalsTest(unittest.TestCase): # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). self.assertRaises(ZeroDivisionError, - wio.write, item * (1024 * 1024)) + wio.write, item * (support.PIPE_MAX_SIZE // len(item))) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 2eeff72..686c1b1 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -489,24 +489,21 @@ class ProcessTestCase(BaseTestCase): # This test will probably deadlock rather than fail, if # communicate() does not work properly. x, y = os.pipe() - if mswindows: - pipe_buf = 512 - else: - pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") os.close(x) os.close(y) p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' 'sys.stdout.write(sys.stdin.read(47));' - 'sys.stderr.write("xyz"*%d);' - 'sys.stdout.write(sys.stdin.read())' % pipe_buf], + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) self.addCleanup(p.stdin.close) - string_to_write = b"abc"*pipe_buf + string_to_write = b"a" * support.PIPE_MAX_SIZE (stdout, stderr) = p.communicate(string_to_write) self.assertEqual(stdout, string_to_write) -- cgit v0.12 From 987475c9568c12d411e74e301ccfd12f7be4f06a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 29 May 2011 16:54:08 +0200 Subject: Explain that INET is IPv4 and STREAM is TCP. --- Doc/howto/sockets.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 04e9b98..80e98f7 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -23,8 +23,8 @@ It's not really a tutorial - you'll still have work to do in getting things working. It doesn't cover the fine points (and there are a lot of them), but I hope it will give you enough background to begin using them decently. -I'm only going to talk about INET sockets, but they account for at least 99% of -the sockets in use. And I'll only talk about STREAM sockets - unless you really +I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of +the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really know what you're doing (in which case this HOWTO isn't for you!), you'll get better behavior and performance from a STREAM socket than anything else. I will try to clear up the mystery of what a socket is, as well as some hints on how to -- cgit v0.12 From a7eaa41f7f54609847b3e0f22dff8400f874e796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 29 May 2011 17:15:44 +0200 Subject: Use bytes in myreceive. --- Doc/howto/sockets.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index 80e98f7..2416807 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -208,10 +208,10 @@ length message:: totalsent = totalsent + sent def myreceive(self): - msg = '' + msg = b'' while len(msg) < MSGLEN: chunk = self.sock.recv(MSGLEN-len(msg)) - if chunk == '': + if chunk == b'': raise RuntimeError("socket connection broken") msg = msg + chunk return msg -- cgit v0.12 From 43af12b0b488791d89a9c08b0ad9f1c5004b5ed7 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 May 2011 11:43:10 -0500 Subject: unify TryExcept and TryFinally (closes #12199) --- Include/Python-ast.h | 23 ++++----- Lib/test/test_ast.py | 4 +- Misc/NEWS | 3 ++ Parser/Python.asdl | 3 +- Python/Python-ast.c | 132 ++++++++++++--------------------------------------- Python/ast.c | 22 ++------- Python/compile.c | 34 +++++++++---- Python/symtable.c | 13 ++--- 8 files changed, 78 insertions(+), 156 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 74c5264..ea6455f 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -66,10 +66,9 @@ struct _mod { enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3, Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7, While_kind=8, If_kind=9, With_kind=10, Raise_kind=11, - TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14, - Import_kind=15, ImportFrom_kind=16, Global_kind=17, - Nonlocal_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21, - Continue_kind=22}; + Try_kind=12, Assert_kind=13, Import_kind=14, + ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17, + Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21}; struct _stmt { enum _stmt_kind kind; union { @@ -143,12 +142,8 @@ struct _stmt { asdl_seq *body; asdl_seq *handlers; asdl_seq *orelse; - } TryExcept; - - struct { - asdl_seq *body; asdl_seq *finalbody; - } TryFinally; + } Try; struct { expr_ty test; @@ -433,12 +428,10 @@ stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4) stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena); -#define TryExcept(a0, a1, a2, a3, a4, a5) _Py_TryExcept(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, - int lineno, int col_offset, PyArena *arena); -#define TryFinally(a0, a1, a2, a3, a4) _Py_TryFinally(a0, a1, a2, a3, a4) -stmt_ty _Py_TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int - col_offset, PyArena *arena); +#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6) +stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, + asdl_seq * finalbody, int lineno, int col_offset, PyArena + *arena); #define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4) stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena); diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index fde2cfe..5520e82 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -347,8 +347,8 @@ exec_results = [ ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), -('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), -('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), ('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), diff --git a/Misc/NEWS b/Misc/NEWS index 3304b03..09ad49f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12199: The TryExcept and TryFinally and AST nodes have been unified + into a Try node. + - Issue #9670: Increase the default stack size for secondary threads on Mac OS X and FreeBSD to reduce the chances of a crash instead of a "maximum recursion depth" RuntimeError exception. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index de48643..ed12cbc 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -31,8 +31,7 @@ module Python | With(withitem* items, stmt* body) | Raise(expr? exc, expr? cause) - | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - | TryFinally(stmt* body, stmt* finalbody) + | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) | Assert(expr test, expr? msg) | Import(alias* names) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f076c7e..66e09b7 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -103,15 +103,11 @@ static char *Raise_fields[]={ "exc", "cause", }; -static PyTypeObject *TryExcept_type; -static char *TryExcept_fields[]={ +static PyTypeObject *Try_type; +static char *Try_fields[]={ "body", "handlers", "orelse", -}; -static PyTypeObject *TryFinally_type; -static char *TryFinally_fields[]={ - "body", "finalbody", }; static PyTypeObject *Assert_type; @@ -689,11 +685,8 @@ static int init_types(void) if (!With_type) return 0; Raise_type = make_type("Raise", stmt_type, Raise_fields, 2); if (!Raise_type) return 0; - TryExcept_type = make_type("TryExcept", stmt_type, TryExcept_fields, 3); - if (!TryExcept_type) return 0; - TryFinally_type = make_type("TryFinally", stmt_type, TryFinally_fields, - 2); - if (!TryFinally_type) return 0; + Try_type = make_type("Try", stmt_type, Try_fields, 4); + if (!Try_type) return 0; Assert_type = make_type("Assert", stmt_type, Assert_fields, 2); if (!Assert_type) return 0; Import_type = make_type("Import", stmt_type, Import_fields, 1); @@ -1264,33 +1257,18 @@ Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena) } stmt_ty -TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int lineno, - int col_offset, PyArena *arena) +Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, asdl_seq * + finalbody, int lineno, int col_offset, PyArena *arena) { stmt_ty p; p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = TryExcept_kind; - p->v.TryExcept.body = body; - p->v.TryExcept.handlers = handlers; - p->v.TryExcept.orelse = orelse; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -stmt_ty -TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int col_offset, - PyArena *arena) -{ - stmt_ty p; - p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = TryFinally_kind; - p->v.TryFinally.body = body; - p->v.TryFinally.finalbody = finalbody; + p->kind = Try_kind; + p->v.Try.body = body; + p->v.Try.handlers = handlers; + p->v.Try.orelse = orelse; + p->v.Try.finalbody = finalbody; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -2434,35 +2412,25 @@ ast2obj_stmt(void* _o) goto failed; Py_DECREF(value); break; - case TryExcept_kind: - result = PyType_GenericNew(TryExcept_type, NULL, NULL); + case Try_kind: + result = PyType_GenericNew(Try_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_list(o->v.TryExcept.body, ast2obj_stmt); + value = ast2obj_list(o->v.Try.body, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryExcept.handlers, - ast2obj_excepthandler); + value = ast2obj_list(o->v.Try.handlers, ast2obj_excepthandler); if (!value) goto failed; if (PyObject_SetAttrString(result, "handlers", value) == -1) goto failed; Py_DECREF(value); - value = ast2obj_list(o->v.TryExcept.orelse, ast2obj_stmt); + value = ast2obj_list(o->v.Try.orelse, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttrString(result, "orelse", value) == -1) goto failed; Py_DECREF(value); - break; - case TryFinally_kind: - result = PyType_GenericNew(TryFinally_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_list(o->v.TryFinally.body, ast2obj_stmt); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "body", value) == -1) - goto failed; - Py_DECREF(value); - value = ast2obj_list(o->v.TryFinally.finalbody, ast2obj_stmt); + value = ast2obj_list(o->v.Try.finalbody, ast2obj_stmt); if (!value) goto failed; if (PyObject_SetAttrString(result, "finalbody", value) == -1) goto failed; @@ -4343,7 +4311,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)TryExcept_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Try_type); if (isinstance == -1) { return 1; } @@ -4351,6 +4319,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* handlers; asdl_seq* orelse; + asdl_seq* finalbody; if (PyObject_HasAttrString(obj, "body")) { int res; @@ -4359,7 +4328,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) tmp = PyObject_GetAttrString(obj, "body"); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4374,7 +4343,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from TryExcept"); + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Try"); return 1; } if (PyObject_HasAttrString(obj, "handlers")) { @@ -4384,7 +4353,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) tmp = PyObject_GetAttrString(obj, "handlers"); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4399,7 +4368,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from TryExcept"); + PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from Try"); return 1; } if (PyObject_HasAttrString(obj, "orelse")) { @@ -4409,7 +4378,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) tmp = PyObject_GetAttrString(obj, "orelse"); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryExcept field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4424,45 +4393,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from TryExcept"); - return 1; - } - *out = TryExcept(body, handlers, orelse, lineno, col_offset, - arena); - if (*out == NULL) goto failed; - return 0; - } - isinstance = PyObject_IsInstance(obj, (PyObject*)TryFinally_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - asdl_seq* body; - asdl_seq* finalbody; - - if (PyObject_HasAttrString(obj, "body")) { - int res; - Py_ssize_t len; - Py_ssize_t i; - tmp = PyObject_GetAttrString(obj, "body"); - if (tmp == NULL) goto failed; - if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryFinally field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); - goto failed; - } - len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); - } - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from TryFinally"); + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from Try"); return 1; } if (PyObject_HasAttrString(obj, "finalbody")) { @@ -4472,7 +4403,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) tmp = PyObject_GetAttrString(obj, "finalbody"); if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { - PyErr_Format(PyExc_TypeError, "TryFinally field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name); + PyErr_Format(PyExc_TypeError, "Try field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; } len = PyList_GET_SIZE(tmp); @@ -4487,10 +4418,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from TryFinally"); + PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from Try"); return 1; } - *out = TryFinally(body, finalbody, lineno, col_offset, arena); + *out = Try(body, handlers, orelse, finalbody, lineno, + col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -6853,10 +6785,8 @@ PyInit__ast(void) NULL; if (PyDict_SetItemString(d, "Raise", (PyObject*)Raise_type) < 0) return NULL; - if (PyDict_SetItemString(d, "TryExcept", (PyObject*)TryExcept_type) < - 0) return NULL; - if (PyDict_SetItemString(d, "TryFinally", (PyObject*)TryFinally_type) < - 0) return NULL; + if (PyDict_SetItemString(d, "Try", (PyObject*)Try_type) < 0) return + NULL; if (PyDict_SetItemString(d, "Assert", (PyObject*)Assert_type) < 0) return NULL; if (PyDict_SetItemString(d, "Import", (PyObject*)Import_type) < 0) diff --git a/Python/ast.c b/Python/ast.c index 882452b..485b7d6 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2893,7 +2893,7 @@ ast_for_try_stmt(struct compiling *c, const node *n) { const int nch = NCH(n); int n_except = (nch - 3)/3; - asdl_seq *body, *orelse = NULL, *finally = NULL; + asdl_seq *body, *handlers = NULL, *orelse = NULL, *finally = NULL; REQ(n, try_stmt); @@ -2934,9 +2934,8 @@ ast_for_try_stmt(struct compiling *c, const node *n) if (n_except > 0) { int i; - stmt_ty except_st; /* process except statements to create a try ... except */ - asdl_seq *handlers = asdl_seq_new(n_except, c->c_arena); + handlers = asdl_seq_new(n_except, c->c_arena); if (handlers == NULL) return NULL; @@ -2947,23 +2946,10 @@ ast_for_try_stmt(struct compiling *c, const node *n) return NULL; asdl_seq_SET(handlers, i, e); } - - except_st = TryExcept(body, handlers, orelse, LINENO(n), - n->n_col_offset, c->c_arena); - if (!finally) - return except_st; - - /* if a 'finally' is present too, we nest the TryExcept within a - TryFinally to emulate try ... except ... finally */ - body = asdl_seq_new(1, c->c_arena); - if (body == NULL) - return NULL; - asdl_seq_SET(body, 0, except_st); } - /* must be a try ... finally (except clauses are in body, if any exist) */ - assert(finally != NULL); - return TryFinally(body, finally, LINENO(n), n->n_col_offset, c->c_arena); + assert(finally != NULL || asdl_seq_LEN(handlers)); + return Try(body, handlers, orelse, finally, LINENO(n), n->n_col_offset, c->c_arena); } /* with_item: test ['as' expr] */ diff --git a/Python/compile.c b/Python/compile.c index d24528b..b655c25 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -185,6 +185,7 @@ static int compiler_call_helper(struct compiler *c, int n, asdl_seq *keywords, expr_ty starargs, expr_ty kwargs); +static int compiler_try_except(struct compiler *, stmt_ty); static PyCodeObject *assemble(struct compiler *, int addNone); static PyObject *__doc__; @@ -1898,7 +1899,13 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_use_next_block(c, body); if (!compiler_push_fblock(c, FINALLY_TRY, body)) return 0; - VISIT_SEQ(c, stmt, s->v.TryFinally.body); + if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) { + if (!compiler_try_except(c, s)) + return 0; + } + else { + VISIT_SEQ(c, stmt, s->v.Try.body); + } ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, FINALLY_TRY, body); @@ -1906,7 +1913,7 @@ compiler_try_finally(struct compiler *c, stmt_ty s) compiler_use_next_block(c, end); if (!compiler_push_fblock(c, FINALLY_END, end)) return 0; - VISIT_SEQ(c, stmt, s->v.TryFinally.finalbody); + VISIT_SEQ(c, stmt, s->v.Try.finalbody); ADDOP(c, END_FINALLY); compiler_pop_fblock(c, FINALLY_END, end); @@ -1960,15 +1967,15 @@ compiler_try_except(struct compiler *c, stmt_ty s) compiler_use_next_block(c, body); if (!compiler_push_fblock(c, EXCEPT, body)) return 0; - VISIT_SEQ(c, stmt, s->v.TryExcept.body); + VISIT_SEQ(c, stmt, s->v.Try.body); ADDOP(c, POP_BLOCK); compiler_pop_fblock(c, EXCEPT, body); ADDOP_JREL(c, JUMP_FORWARD, orelse); - n = asdl_seq_LEN(s->v.TryExcept.handlers); + n = asdl_seq_LEN(s->v.Try.handlers); compiler_use_next_block(c, except); for (i = 0; i < n; i++) { excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - s->v.TryExcept.handlers, i); + s->v.Try.handlers, i); if (!handler->v.ExceptHandler.type && i < n-1) return compiler_error(c, "default 'except:' must be last"); c->u->u_lineno_set = 0; @@ -2055,12 +2062,21 @@ compiler_try_except(struct compiler *c, stmt_ty s) } ADDOP(c, END_FINALLY); compiler_use_next_block(c, orelse); - VISIT_SEQ(c, stmt, s->v.TryExcept.orelse); + VISIT_SEQ(c, stmt, s->v.Try.orelse); compiler_use_next_block(c, end); return 1; } static int +compiler_try(struct compiler *c, stmt_ty s) { + if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody)) + return compiler_try_finally(c, s); + else + return compiler_try_except(c, s); +} + + +static int compiler_import_as(struct compiler *c, identifier name, identifier asname) { /* The IMPORT_NAME opcode was already generated. This function @@ -2307,10 +2323,8 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) } ADDOP_I(c, RAISE_VARARGS, n); break; - case TryExcept_kind: - return compiler_try_except(c, s); - case TryFinally_kind: - return compiler_try_finally(c, s); + case Try_kind: + return compiler_try(c, s); case Assert_kind: return compiler_assert(c, s); case Import_kind: diff --git a/Python/symtable.c b/Python/symtable.c index d276254..e31a2eb 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1211,14 +1211,11 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } } break; - case TryExcept_kind: - VISIT_SEQ(st, stmt, s->v.TryExcept.body); - VISIT_SEQ(st, stmt, s->v.TryExcept.orelse); - VISIT_SEQ(st, excepthandler, s->v.TryExcept.handlers); - break; - case TryFinally_kind: - VISIT_SEQ(st, stmt, s->v.TryFinally.body); - VISIT_SEQ(st, stmt, s->v.TryFinally.finalbody); + case Try_kind: + VISIT_SEQ(st, stmt, s->v.Try.body); + VISIT_SEQ(st, stmt, s->v.Try.orelse); + VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case Assert_kind: VISIT(st, expr, s->v.Assert.test); -- cgit v0.12 From 04a90b4611d3a20ac593860d7f4156a14723e84a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 May 2011 11:45:29 -0500 Subject: bump ast version --- Python/Python-ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 66e09b7..b597626 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2,7 +2,7 @@ /* - __version__ 9b11cc4e2918. + __version__ e0e663132363. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -6750,7 +6750,7 @@ PyInit__ast(void) NULL; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return NULL; - if (PyModule_AddStringConstant(m, "__version__", "9b11cc4e2918") < 0) + if (PyModule_AddStringConstant(m, "__version__", "e0e663132363") < 0) return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; -- cgit v0.12 From daafdd5bea1edb0fa980727cf8c52bfe7928d6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 29 May 2011 20:07:40 +0200 Subject: Issue #12196: Add pipe2() to the os module. --- Doc/library/os.rst | 13 +++++++++++++ Lib/test/test_posix.py | 25 +++++++++++++++++++++++++ Misc/NEWS | 2 ++ Modules/posixmodule.c | 28 ++++++++++++++++++++++++++++ configure | 10 +--------- configure.in | 4 +--- pyconfig.h.in | 2 +- 7 files changed, 71 insertions(+), 13 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 6ef6d9d..c62a00d 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1019,6 +1019,19 @@ as internal buffering of data. Availability: Unix, Windows. +.. function:: pipe2(flags=0) + + Create a pipe with *flags* set atomically. + *flags* is optional and can be constructed by ORing together zero or more of + these values: :data:`O_NONBLOCK`, :data:`O_CLOEXEC`. + Return a pair of file descriptors ``(r, w)`` usable for reading and writing, + respectively. + + Availability: some flavors of Unix. + + .. versionadded:: 3.3 + + .. function:: posix_fallocate(fd, offset, len) Ensures that enough disk space is allocated for the file specified by *fd* diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 9d9802b..e9103cd 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -478,6 +478,31 @@ class PosixTester(unittest.TestCase): os.close(reader) os.close(writer) + @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()") + def test_pipe2(self): + self.assertRaises(TypeError, os.pipe2, 'DEADBEEF') + self.assertRaises(TypeError, os.pipe2, 0, 0) + + # try calling without flag, like os.pipe() + r, w = os.pipe2() + os.close(r) + os.close(w) + + # test flags + r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + # try reading from an empty pipe: this should fail, not block + self.assertRaises(OSError, os.read, r, 1) + # try a write big enough to fill-up the pipe: this should either + # fail or perform a partial write, not block + try: + os.write(w, b'x' * support.PIPE_MAX_SIZE) + except OSError: + pass + def test_utime(self): if hasattr(posix, 'utime'): now = time.time() diff --git a/Misc/NEWS b/Misc/NEWS index 09ad49f..fde085b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -175,6 +175,8 @@ Core and Builtins Library ------- +- Issue #12196: Add pipe2() to the os module. + - Issue #985064: Make plistlib more resilient to faulty input plists. Patch by Mher Movsisyan. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index add3b35..e529afd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6547,6 +6547,31 @@ posix_pipe(PyObject *self, PyObject *noargs) } #endif /* HAVE_PIPE */ +#ifdef HAVE_PIPE2 +PyDoc_STRVAR(posix_pipe2__doc__, +"pipe2(flags=0) -> (read_end, write_end)\n\n\ +Create a pipe with flags set atomically.\ +flags is optional and can be constructed by ORing together zero or more\n\ +of these values: O_NONBLOCK, O_CLOEXEC.\n\ +"); + +static PyObject * +posix_pipe2(PyObject *self, PyObject *args) +{ + int flags = 0; + int fds[2]; + int res; + + if (!PyArg_ParseTuple(args, "|i:pipe2", &flags)) + return NULL; + + res = pipe2(fds, flags); + if (res != 0) + return posix_error(); + return Py_BuildValue("(ii)", fds[0], fds[1]); +} +#endif /* HAVE_PIPE2 */ + #ifdef HAVE_WRITEV PyDoc_STRVAR(posix_writev__doc__, "writev(fd, buffers) -> byteswritten\n\n\ @@ -9441,6 +9466,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_PIPE {"pipe", posix_pipe, METH_NOARGS, posix_pipe__doc__}, #endif +#ifdef HAVE_PIPE2 + {"pipe2", posix_pipe2, METH_VARARGS, posix_pipe2__doc__}, +#endif #ifdef HAVE_MKFIFO {"mkfifo", posix_mkfifo, METH_VARARGS, posix_mkfifo__doc__}, #endif diff --git a/configure b/configure index e1ab99b..89a5d70 100755 --- a/configure +++ b/configure @@ -9307,7 +9307,7 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mbrtowc mkdirat mkfifo \ - mkfifoat mknod mknodat mktime mremap nice openat pathconf pause plock poll \ + mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ select sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ @@ -13824,14 +13824,6 @@ $as_echo "#define HAVE_BROKEN_PIPE_BUF 1" >>confdefs.h esac -ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" -if test "x$ac_cv_func_pipe2" = x""yes; then : - -$as_echo "#define HAVE_PIPE2 1" >>confdefs.h - -fi - - for h in `(cd $srcdir;echo Python/thread_*.h)` diff --git a/configure.in b/configure.in index 3bf46b0..b5f155d 100644 --- a/configure.in +++ b/configure.in @@ -2531,7 +2531,7 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ if_nameindex \ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mbrtowc mkdirat mkfifo \ - mkfifoat mknod mknodat mktime mremap nice openat pathconf pause plock poll \ + mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \ select sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ @@ -4244,8 +4244,6 @@ AIX*) esac -AC_CHECK_FUNC(pipe2, AC_DEFINE(HAVE_PIPE2, 1, [Define if the OS supports pipe2()]), ) - AC_SUBST(THREADHEADERS) for h in `(cd $srcdir;echo Python/thread_*.h)` diff --git a/pyconfig.h.in b/pyconfig.h.in index 95d71c9..6c45460 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -560,7 +560,7 @@ /* Define to 1 if you have the `pause' function. */ #undef HAVE_PAUSE -/* Define if the OS supports pipe2() */ +/* Define to 1 if you have the `pipe2' function. */ #undef HAVE_PIPE2 /* Define to 1 if you have the `plock' function. */ -- cgit v0.12 From d92ccb1f75ad5852e280c90354e5f547db649418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Sun, 29 May 2011 20:46:27 +0200 Subject: Skip test_pipe2 on Linux kernels older than 2.6.27. --- Lib/test/test_posix.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index e9103cd..79e2d36 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -480,6 +480,10 @@ class PosixTester(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()") def test_pipe2(self): + version = support.linux_version() + if sys.platform == 'linux2' and version < (2, 6, 27): + self.skipTest("Linux kernel 2.6.27 or higher required, " + "not %s.%s.%s" % version) self.assertRaises(TypeError, os.pipe2, 'DEADBEEF') self.assertRaises(TypeError, os.pipe2, 0, 0) -- cgit v0.12 From 9a63745273522db31d6dea5a45db38d2af5fef55 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 May 2011 16:06:00 -0500 Subject: bump to 3.1.4rc1 --- Include/patchlevel.h | 8 ++++---- Lib/distutils/__init__.py | 2 +- Lib/idlelib/idlever.py | 2 +- Misc/NEWS | 6 +++--- Misc/RPM/python-3.1.spec | 2 +- README | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 58a79fe..db58c5f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 1 -#define PY_MICRO_VERSION 3 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL -#define PY_RELEASE_SERIAL 0 +#define PY_MICRO_VERSION 4 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.1.3+" +#define PY_VERSION "3.1.4rc1" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index 6f55f01..5f8425f 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -15,5 +15,5 @@ __revision__ = "$Id$" # Updated automatically by the Python release process. # #--start constants-- -__version__ = "3.1.3" +__version__ = "3.1.4rc1" #--end constants-- diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 8ee61a2..fa89048 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "3.1.3" +IDLE_VERSION = "3.1.4rc1" diff --git a/Misc/NEWS b/Misc/NEWS index f04c643..3982b2b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,10 +2,10 @@ Python News +++++++++++ -What's New in Python 3.1.4? -=========================== +What's New in Python 3.1.4 release candidate 1? +=============================================== -*Release date: XXXX-XX-XX* +*Release date: 2011-05-29* Core and Builtins ----------------- diff --git a/Misc/RPM/python-3.1.spec b/Misc/RPM/python-3.1.spec index 0f24a41..d3c9bf2 100644 --- a/Misc/RPM/python-3.1.spec +++ b/Misc/RPM/python-3.1.spec @@ -34,7 +34,7 @@ %define name python #--start constants-- -%define version 3.1.3 +%define version 3.1.4rc1 %define libvers 3.1 #--end constants-- %define release 1pydotorg diff --git a/README b/README index a59cb91..b136247 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -This is Python version 3.1.3 +This is Python version 3.1.4 ============================ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 -- cgit v0.12 From 5f250fbafea211b21e464d8d3c8f09af0a1434f8 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 May 2011 16:09:08 -0500 Subject: update pydoc-topics --- Lib/pydoc_data/topics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 78cea8c..385edea 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Thu Apr 28 07:37:19 2011 +# Autogenerated by Sphinx on Sun May 29 16:08:49 2011 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n | "*" target\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list, optionally enclosed in\nparentheses or square brackets, is recursively defined as follows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\n * If the target list contains one target prefixed with an asterisk,\n called a "starred" target: The object must be a sequence with at\n least as many items as there are targets in the target list, minus\n one. The first items of the sequence are assigned, from left to\n right, to the targets before the starred target. The final items\n of the sequence are assigned to the targets after the starred\n target. A list of the remaining items in the sequence is then\n assigned to the starred target (the list can be empty).\n\n * Else: The object must be a sequence with the same number of items\n as there are targets in the target list, and the items are\n assigned, from left to right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` or ``nonlocal``\n statement in the current code block: the name is bound to the\n object in the current local namespace.\n\n * Otherwise: the name is bound to the object in the global namespace\n or the outer namespace determined by ``nonlocal``, respectively.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield an integer. If it is negative, the sequence\'s\n length is added to it. The resulting value must be a nonnegative\n integer less than the sequence\'s length, and the sequence is asked\n to assign the assigned object to its item with that index. If the\n index is out of range, ``IndexError`` is raised (assignment to a\n subscripted sequence cannot add new items to a list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n For user-defined objects, the ``__setitem__()`` method is called\n with appropriate arguments.\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to integers. If either bound is\n negative, the sequence\'s length is added to it. The resulting\n bounds are clipped to lie between zero and the sequence\'s length,\n inclusive. Finally, the sequence object is asked to replace the\n slice with the items of the assigned sequence. The length of the\n slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print(x)\n\nSee also:\n\n **PEP 3132** - Extended Iterable Unpacking\n The specification for the ``*target`` feature.\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', @@ -60,7 +60,7 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss 'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice\n proper_slice ::= [lower_bound] ":" [upper_bound] [ ":" [stride] ]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice).\n\nThe semantics for a slicing are as follows. The primary must evaluate\nto a mapping object, and it is indexed (using the same\n``__getitem__()`` method as normal subscription) with a key that is\nconstructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of a proper slice is a\nslice object (see section *The standard type hierarchy*) whose\n``start``, ``stop`` and ``step`` attributes are the values of the\nexpressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': "\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object's\n (writable) attributes.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n []\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can't tell the type of the\n operands.\n\n[4] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n", 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``type(x).__getitem__(x,\ni)``. Except where mentioned, attempts to execute an operation raise\nan exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.last_traceback``. Circular references which are garbage are\n detected when the option cycle detector is enabled (it\'s on by\n default), but can only be cleaned up if there are no Python-\n level ``__del__()`` methods involved. Refer to the documentation\n for the ``gc`` module for more information about how\n ``__del__()`` methods are handled by the cycle detector,\n particularly the description of the ``garbage`` value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function to compute the\n "official" string representation of an object. If at all possible,\n this should look like a valid Python expression that could be used\n to recreate an object with the same value (given an appropriate\n environment). If this is not possible, a string of the form\n ``<...some useful description...>`` should be returned. The return\n value must be a string object. If a class defines ``__repr__()``\n but not ``__str__()``, then ``__repr__()`` is also used when an\n "informal" string representation of instances of that class is\n required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print()``\n function to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__format__(self, format_spec)\n\n Called by the ``format()`` built-in function (and by extension, the\n ``format()`` method of class ``str``) to produce a "formatted"\n string representation of an object. The ``format_spec`` argument is\n a string that contains a description of the formatting options\n desired. The interpretation of the ``format_spec`` argument is up\n to the type implementing ``__format__()``, however most classes\n will either delegate formatting to one of the built-in types, or\n use a similar formatting option syntax.\n\n See *Format Specification Mini-Language* for a description of the\n standard formatting syntax.\n\n The return value must be a string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n These are the so-called "rich comparison" methods. The\n correspondence between operator symbols and method names is as\n follows: ``xy`` calls ``x.__gt__(y)``, and ``x>=y`` calls\n ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see the Total Ordering recipe in the ASPN cookbook.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define an ``__eq__()`` method it should not\n define a ``__hash__()`` operation either; if it defines\n ``__eq__()`` but not ``__hash__()``, its instances will not be\n usable as items in hashable collections. If a class defines\n mutable objects and implements an ``__eq__()`` method, it should\n not implement ``__hash__()``, since the implementation of hashable\n collections requires that a key\'s hash value is immutable (if the\n object\'s hash value changes, it will be in the wrong hash bucket).\n\n User-defined classes have ``__eq__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__eq__()`` such that the hash value\n returned is no longer appropriate (e.g. by switching to a value-\n based concept of equality instead of the default identity based\n equality) can explicitly flag themselves as being unhashable by\n setting ``__hash__ = None`` in the class definition. Doing so means\n that not only will instances of the class raise an appropriate\n ``TypeError`` when a program attempts to retrieve their hash value,\n but they will also be correctly identified as unhashable when\n checking ``isinstance(obj, collections.Hashable)`` (unlike classes\n which define their own ``__hash__()`` to explicitly raise\n ``TypeError``).\n\n If a class that overrides ``__eq__()`` needs to retain the\n implementation of ``__hash__()`` from a parent class, the\n interpreter must be told this explicitly by setting ``__hash__ =\n .__hash__``. Otherwise the inheritance of\n ``__hash__()`` will be blocked, just as if ``__hash__`` had been\n explicitly set to ``None``.\n\nobject.__bool__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``. When this method\n is not defined, ``__len__()`` is called, if it is defined, and the\n object is considered true if its result is nonzero. If a class\n defines neither ``__len__()`` nor ``__bool__()``, all its instances\n are considered true.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control over attribute access.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup*.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should call the base class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\nobject.__dir__(self)\n\n Called when ``dir()`` is called on the object. A list must be\n returned.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called.\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to an object instance, ``a.x`` is transformed into the\n call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a class, ``A.x`` is transformed into the call:\n ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of classes have a dictionary for attribute\nstorage. This wastes space for objects having very few instance\nvariables. The space consumption can become acute when creating large\nnumbers of instances.\n\nThe default can be overridden by defining *__slots__* in a class\ndefinition. The *__slots__* declaration takes a sequence of instance\nvariables and reserves just enough space in each instance to hold a\nvalue for each variable. Space is saved because *__dict__* is not\ncreated for each instance.\n\nobject.__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n class, *__slots__* reserves space for the declared variables and\n prevents the automatic creation of *__dict__* and *__weakref__* for\n each instance.\n\n\nNotes on using *__slots__*\n~~~~~~~~~~~~~~~~~~~~~~~~~~\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``int``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, classes are constructed using ``type()``. A class\ndefinition is read into a separate namespace and the value of class\nname is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if a callable ``metaclass`` keyword\nargument is passed after the bases in the class definition, the\ncallable given will be called instead of ``type()``. If other keyword\narguments are passed, they will also be passed to the metaclass. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\nIf the metaclass has a ``__prepare__()`` attribute (usually\nimplemented as a class or static method), it is called before the\nclass body is evaluated with the name of the class and a tuple of its\nbases for arguments. It should return an object that supports the\nmapping interface that will be used to store the namespace of the\nclass. The default is a plain dictionary. This could be used, for\nexample, to keep track of the order that class attributes are declared\nin by returning an ordered dictionary.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If the ``metaclass`` keyword argument is passed with the bases, it\n is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used.\n\n* Otherwise, the default metaclass (``type``) is used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\nHere is an example of a metaclass that uses an\n``collections.OrderedDict`` to remember the order that class members\nwere defined:\n\n class OrderedClass(type):\n\n @classmethod\n def __prepare__(metacls, name, bases, **kwds):\n return collections.OrderedDict()\n\n def __new__(cls, name, bases, classdict):\n result = type.__new__(cls, name, bases, dict(classdict))\n result.members = tuple(classdict)\n return result\n\n class A(metaclass=OrderedClass):\n def one(self): pass\n def two(self): pass\n def three(self): pass\n def four(self): pass\n\n >>> A.members\n (\'__module__\', \'one\', \'two\', \'three\', \'four\')\n\nWhen the class definition for *A* gets executed, the process begins\nwith calling the metaclass\'s ``__prepare__()`` method which returns an\nempty ``collections.OrderedDict``. That mapping records the methods\nand attributes of *A* as they are defined within the body of the class\nstatement. Once those definitions are executed, the ordered dictionary\nis fully populated and the metaclass\'s ``__new__()`` method gets\ninvoked. That method builds the new type and it saves the ordered\ndictionary keys in an attribute called *members*.\n\n\nCustomizing instance and subclass checks\n========================================\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``get()``,\n``clear()``, ``setdefault()``, ``pop()``, ``popitem()``, ``copy()``,\nand ``update()`` behaving similar to those for Python\'s standard\ndictionary objects. The ``collections`` module provides a\n``MutableMapping`` abstract base class to help create those methods\nfrom a base set of ``__getitem__()``, ``__setitem__()``,\n``__delitem__()``, and ``keys()``. Mutable sequences should provide\nmethods ``append()``, ``count()``, ``index()``, ``extend()``,\n``insert()``, ``pop()``, ``remove()``, ``reverse()`` and ``sort()``,\nlike Python standard list objects. Finally, sequence types should\nimplement addition (meaning concatenation) and multiplication (meaning\nrepetition) by defining the methods ``__add__()``, ``__radd__()``,\n``__iadd__()``, ``__mul__()``, ``__rmul__()`` and ``__imul__()``\ndescribed below; they should not define other numerical operators. It\nis recommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should search the mapping\'s keys; for\nsequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``keys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__bool__()`` method and whose ``__len__()``\n method returns zero is considered to be false in a Boolean context.\n\nNote: Slicing is done exclusively with the following three methods. A\n call like\n\n a[1:2] = b\n\n is translated to\n\n a[slice(1, 2, None)] = b\n\n and so forth. Missing slice items are always filled in with\n ``None``.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``keys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__truediv__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n ``|``). For instance, to evaluate the expression ``x + y``, where\n *x* is an instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()``. Note that\n ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``//``, ``%``,\n ``divmod()``, ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``,\n ``|``) with reflected (swapped) operands. These functions are only\n called if the left operand does not support the corresponding\n operation and the operands are of different types. [2] For\n instance, to evaluate the expression ``x - y``, where *y* is an\n instance of a class that has an ``__rsub__()`` method,\n ``y.__rsub__(x)`` is called if ``x.__sub__(y)`` returns\n *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__float__(self)\nobject.__round__(self[, n])\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``float()`` and ``round()``. Should return a value of\n the appropriate type.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing, or in the\n built-in ``bin()``, ``hex()`` and ``oct()`` functions). Must return\n an integer.\n\n\nWith Statement Context Managers\n===============================\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup\n=====================\n\nFor custom classes, implicit invocations of special methods are only\nguaranteed to work correctly if defined on an object\'s type, not in\nthe object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception:\n\n >>> class C:\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print("Metaclass getattribute invoked")\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object, metaclass=Meta):\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print("Class getattribute invoked")\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', - 'string-methods': '\nString Methods\n**************\n\nString objects support the methods listed below. Note that none of\nthese methods take keyword arguments.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string as a bytes object. Default\n encoding is the current default string encoding. *errors* may be\n given to set a different error handling scheme. The default for\n *errors* is ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The *format_string*\n argument can contain literal text or replacement fields delimited\n by braces ``{}``. Each replacement field contains either the\n numeric index of a positional argument, or the name of a keyword\n argument. Returns a copy of *format_string* where each replacement\n field is replaced with the string value of the corresponding\n argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n', + 'string-methods': '\nString Methods\n**************\n\nString objects support the methods listed below. Note that none of\nthese methods take keyword arguments.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string as a bytes object. Default\n encoding is the current default string encoding. *errors* may be\n given to set a different error handling scheme. The default for\n *errors* is ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The *format_string*\n argument can contain literal text or replacement fields delimited\n by braces ``{}``. Each replacement field contains either the\n numeric index of a positional argument, or the name of a keyword\n argument. Returns a copy of *format_string* where each replacement\n field is replaced with the string value of the corresponding\n argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n', 'strings': '\nString and Bytes literals\n*************************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "R"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'" | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | stringescapeseq\n longstringitem ::= longstringchar | stringescapeseq\n shortstringchar ::= \n longstringchar ::= \n stringescapeseq ::= "\\" \n\n bytesliteral ::= bytesprefix(shortbytes | longbytes)\n bytesprefix ::= "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortbytes ::= "\'" shortbytesitem* "\'" | \'"\' shortbytesitem* \'"\'\n longbytes ::= "\'\'\'" longbytesitem* "\'\'\'" | \'"""\' longbytesitem* \'"""\'\n shortbytesitem ::= shortbyteschar | bytesescapeseq\n longbytesitem ::= longbyteschar | bytesescapeseq\n shortbyteschar ::= \n longbyteschar ::= \n bytesescapeseq ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the **stringprefix** or\n**bytesprefix** and the rest of the literal. The source character set\nis defined by the encoding declaration; it is UTF-8 if no encoding\ndeclaration is given in the source file; see section *Encoding\ndeclarations*.\n\nIn plain English: Both types of literals can be enclosed in matching\nsingle quotes (``\'``) or double quotes (``"``). They can also be\nenclosed in matching groups of three single or double quotes (these\nare generally referred to as *triple-quoted strings*). The backslash\n(``\\``) character is used to escape characters that otherwise have a\nspecial meaning, such as newline, backslash itself, or the quote\ncharacter.\n\nBytes literals are always prefixed with ``\'b\'`` or ``\'B\'``; they\nproduce an instance of the ``bytes`` type instead of the ``str`` type.\nThey may only contain ASCII characters; bytes with a numeric value of\n128 or greater must be expressed with escapes.\n\nBoth string and bytes literals may optionally be prefixed with a\nletter ``\'r\'`` or ``\'R\'``; such strings are called *raw strings* and\ntreat backslashes as literal characters. As a result, in string\nliterals, ``\'\\U\'`` and ``\'\\u\'`` escapes in raw strings are not treated\nspecially.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Backslash and newline ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (1,3) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (2,3) |\n+-------------------+-----------------------------------+---------+\n\nEscape sequences only recognized in string literals are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (4) |\n| | *xxxx* | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (5) |\n| | *xxxxxxxx* | |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. As in Standard C, up to three octal digits are accepted.\n\n2. Unlike in Standard C, exactly two hex digits are required.\n\n3. In a bytes literal, hexadecimal and octal escapes denote the byte\n with the given value. In a string literal, these escapes denote a\n Unicode character with the given value.\n\n4. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence. Exactly four hex digits are\n required.\n\n5. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Exactly eight hex digits are required.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences only recognized in string\nliterals fall into the category of unrecognized escapes for bytes\nliterals.\n\nEven in a raw string, string quotes can be escaped with a backslash,\nbut the backslash remains in the string; for example, ``r"\\""`` is a\nvalid string literal consisting of two characters: a backslash and a\ndouble quote; ``r"\\"`` is not a valid string literal (even a raw\nstring cannot end in an odd number of backslashes). Specifically, *a\nraw string cannot end in a single backslash* (since the backslash\nwould escape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n', 'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object that supports subscription,\ne.g. a list or dictionary. User-defined objects can support\nsubscription by defining a ``__getitem__()`` method.\n\nFor built-in objects, there are two types of objects that support\nsubscription:\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to\nan integer or a slice (as discussed in the following section).\n\nThe formal syntax makes no special provision for negative indices in\nsequences; however, built-in sequences all provide a ``__getitem__()``\nmethod that interprets negative indices by adding the length of the\nsequence to the index (so that ``x[-1]`` selects the last item of\n``x``). The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero). Since the support\nfor negative indices and slicing occurs in the object\'s\n``__getitem__()`` method, subclasses overriding this method will need\nto explicitly add that support.\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0.0``, ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__bool__()`` or ``__len__()`` method, when that method returns the\n integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", @@ -70,7 +70,7 @@ topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAss 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict([arg])\n\n Return a new dictionary initialized from an optional positional\n argument or from a set of keyword arguments. If no arguments are\n given, return a new empty dictionary. If the positional argument\n *arg* is a mapping object, return a dictionary mapping the same\n keys to the same values as does the mapping object. Otherwise the\n positional argument must be a sequence, a container that supports\n iteration, or an iterator object. The elements of the argument\n must each also be of one of those kinds, and each must in turn\n contain exactly two objects. The first is used as a key in the new\n dictionary, and the second as the key\'s value. If a given key is\n seen more than once, the last value associated with it is retained\n in the new dictionary.\n\n If keyword arguments are given, the keywords themselves with their\n associated values are added as items to the dictionary. If a key\n is specified both in the positional argument and as a keyword\n argument, the value associated with the keyword is retained in the\n dictionary. For example, these all return a dictionary equal to\n ``{"one": 1, "two": 2}``:\n\n * ``dict(one=1, two=2)``\n\n * ``dict({\'one\': 1, \'two\': 2})``\n\n * ``dict(zip((\'one\', \'two\'), (1, 2)))``\n\n * ``dict([[\'two\', 2], [\'one\', 1]])``\n\n The first example only works for keys that are valid Python\n identifiers; the others work with any valid keys.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n If a subclass of dict defines a method ``__missing__()``, if the\n key *key* is not present, the ``d[key]`` operation calls that\n method with the key *key* as argument. The ``d[key]`` operation\n then returns or raises whatever is returned or raised by the\n ``__missing__(key)`` call if the key is not present. No other\n operations or methods invoke ``__missing__()``. If\n ``__missing__()`` is not defined, ``KeyError`` is raised.\n ``__missing__()`` must be a method; it cannot be an instance\n variable. For an example, see ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iter(d.keys())``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n classmethod fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n items()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n keys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n values()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.keys()``, ``dict.values()`` and\n``dict.items()`` are *view objects*. They provide a dynamic view on\nthe dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that ``(key, value)`` pairs are unique\nand hashable, then the items view is also set-like. (Values views are\nnot treated as set-like since the entries are generally not unique.)\nFor set-like views, all of the operations defined for the abstract\nbase class ``collections.Set`` are available (for example, ``==``,\n``<``, or ``^``).\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.keys()\n >>> values = dishes.values()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n >>> keys ^ {\'sausage\', \'juice\'}\n {\'juice\', \'eggs\', \'bacon\', \'spam\'}\n', 'typesmethods': "\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nIf you access a method (a function defined in a class namespace)\nthrough an instance, you get a special object: a *bound method* (also\ncalled *instance method*) object. When called, it will add the\n``self`` argument to the argument list. Bound methods have two\nspecial read-only attributes: ``m.__self__`` is the object on which\nthe method operates, and ``m.__func__`` is the function implementing\nthe method. Calling ``m(arg-1, arg-2, ..., arg-n)`` is completely\nequivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ...,\narg-n)``.\n\nLike function objects, bound method objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.__func__``), setting method\nattributes on bound methods is disallowed. Attempting to set a method\nattribute results in a ``TypeError`` being raised. In order to set a\nmethod attribute, you need to explicitly set it on the underlying\nfunction object:\n\n class C:\n def method(self):\n pass\n\n c = C()\n c.method.__func__.whoami = 'my name is c'\n\nSee *The standard type hierarchy* for more information.\n", 'typesmodules': "\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special member of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ````. If loaded from a file, they are written as\n````.\n", - 'typesseq': '\nSequence Types --- ``str``, ``bytes``, ``bytearray``, ``list``, ``tuple``, ``range``\n************************************************************************************\n\nThere are six sequence types: strings, byte sequences (``bytes``\nobjects), byte arrays (``bytearray`` objects), lists, tuples, and\nrange objects. For other containers see the built in ``dict`` and\n``set`` classes, and the ``collections`` module.\n\nStrings contain Unicode characters. Their literals are written in\nsingle or double quotes: ``\'xyzzy\'``, ``"frobozz"``. See *String and\nBytes literals* for more about string literals. In addition to the\nfunctionality described here, there are also string-specific methods\ndescribed in the *String Methods* section.\n\nBytes and bytearray objects contain single bytes -- the former is\nimmutable while the latter is a mutable sequence. Bytes objects can\nbe constructed the constructor, ``bytes()``, and from literals; use a\n``b`` prefix with normal string syntax: ``b\'xyzzy\'``. To construct\nbyte arrays, use the ``bytearray()`` function.\n\nWarning: While string objects are sequences of characters (represented by\n strings of length 1), bytes and bytearray objects are sequences of\n *integers* (between 0 and 255), representing the ASCII value of\n single bytes. That means that for a bytes or bytearray object *b*,\n ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes or\n bytearray object of length 1. The representation of bytes objects\n uses the literal format (``b\'...\'``) since it is generally more\n useful than e.g. ``bytes([50, 19, 100])``. You can always convert a\n bytes object into a list of integers using ``list(b)``.Also, while\n in previous Python versions, byte strings and Unicode strings could\n be exchanged for each other rather freely (barring encoding issues),\n strings and bytes are now completely separate concepts. There\'s no\n implicit en-/decoding if you pass an object of the wrong type. A\n string always compares unequal to a bytes or bytearray object.\n\nLists are constructed with square brackets, separating items with\ncommas: ``[a, b, c]``. Tuples are constructed by the comma operator\n(not within square brackets), with or without enclosing parentheses,\nbut an empty tuple must have the enclosing parentheses, such as ``a,\nb, c`` or ``()``. A single item tuple must have a trailing comma,\nsuch as ``(d,)``.\n\nObjects of type range are created using the ``range()`` function.\nThey don\'t support slicing, concatenation or repetition, and using\n``in``, ``not in``, ``min()`` or ``max()`` on them is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must\ncompare equal and the two sequences must be of the same type and have\nthe same length. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string object, the ``in`` and ``not in`` operations\n act like a substring test.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n\nString Methods\n==============\n\nString objects support the methods listed below. Note that none of\nthese methods take keyword arguments.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string as a bytes object. Default\n encoding is the current default string encoding. *errors* may be\n given to set a different error handling scheme. The default for\n *errors* is ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The *format_string*\n argument can contain literal text or replacement fields delimited\n by braces ``{}``. Each replacement field contains either the\n numeric index of a positional argument, or the name of a keyword\n argument. Returns a copy of *format_string* where each replacement\n field is replaced with the string value of the corresponding\n argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n\nOld String Formatting Operations\n================================\n\nNote: The formatting operations described here are obsolete and may go\n away in future versions of Python. Use the new *String Formatting*\n in new code.\n\nString objects have one unique built-in operation: the ``%`` operator\n(modulo). This is also known as the string *formatting* or\n*interpolation* operator. Given ``format % values`` (where *format* is\na string), ``%`` conversion specifications in *format* are replaced\nwith zero or more elements of *values*. The effect is similar to the\nusing ``sprintf()`` in the C language.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print(\'%(language)s has %(number)03d quote types.\' %\n... {\'language\': "Python", "number": 2})\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The precision determines the maximal number of characters used.\n\n1. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 3.1: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nRange Type\n==========\n\nThe ``range`` type is an immutable sequence which is commonly used for\nlooping. The advantage of the ``range`` type is that an ``range``\nobject will always take the same amount of memory, no matter the size\nof the range it represents. There are no consistent performance\nadvantages.\n\nRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n\nBytes and Byte Array Methods\n============================\n\nBytes and bytearray objects, being "strings of bytes", have all\nmethods found on strings, with the exception of ``encode()``,\n``format()`` and ``isidentifier()``, which do not make sense with\nthese types. For converting the objects to strings, they have a\n``decode()`` method.\n\nWherever one of these methods needs to interpret the bytes as\ncharacters (e.g. the ``is...()`` methods), the ASCII character set is\nassumed.\n\nNote: The methods on bytes and bytearray objects don\'t accept strings as\n their arguments, just as the methods on strings don\'t accept bytes\n as their arguments. For example, you have to write\n\n a = "abc"\n b = a.replace("a", "f")\n\n and\n\n a = b"abc"\n b = a.replace(b"a", b"f")\n\nbytes.decode([encoding[, errors]])\nbytearray.decode([encoding[, errors]])\n\n Return a string decoded from the given bytes. Default encoding is\n the current default string encoding. *errors* may be given to set\n a different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'`` and any other name registered via\n ``codecs.register_error()``, see section *Codec Base Classes*. For\n a list of possible encodings, see section *Standard Encodings*.\n\nThe bytes and bytearray types have an additional class method:\n\nclassmethod bytes.fromhex(string)\nclassmethod bytearray.fromhex(string)\n\n This ``bytes`` class method returns a bytes or bytearray object,\n decoding the given string object. The string must contain two\n hexadecimal digits per byte, spaces are ignored.\n\n >>> bytes.fromhex(\'f0 f1f2 \')\n b\'\\xf0\\xf1\\xf2\'\n\nThe maketrans and translate methods differ in semantics from the\nversions available on strings:\n\nbytes.translate(table[, delete])\nbytearray.translate(table[, delete])\n\n Return a copy of the bytes or bytearray object where all bytes\n occurring in the optional argument *delete* are removed, and the\n remaining bytes have been mapped through the given translation\n table, which must be a bytes object of length 256.\n\n You can use the ``bytes.maketrans()`` method to create a\n translation table.\n\n Set the *table* argument to ``None`` for translations that only\n delete characters:\n\n >>> b\'read this short text\'.translate(None, b\'aeiou\')\n b\'rd ths shrt txt\'\n\nstatic bytes.maketrans(from, to)\nstatic bytearray.maketrans(from, to)\n\n This static method returns a translation table usable for\n ``bytes.translate()`` that will map each character in *from* into\n the character at the same position in *to*; *from* and *to* must be\n bytes objects and have the same length.\n\n New in version 3.1.\n', + 'typesseq': '\nSequence Types --- ``str``, ``bytes``, ``bytearray``, ``list``, ``tuple``, ``range``\n************************************************************************************\n\nThere are six sequence types: strings, byte sequences (``bytes``\nobjects), byte arrays (``bytearray`` objects), lists, tuples, and\nrange objects. For other containers see the built in ``dict`` and\n``set`` classes, and the ``collections`` module.\n\nStrings contain Unicode characters. Their literals are written in\nsingle or double quotes: ``\'xyzzy\'``, ``"frobozz"``. See *String and\nBytes literals* for more about string literals. In addition to the\nfunctionality described here, there are also string-specific methods\ndescribed in the *String Methods* section.\n\nBytes and bytearray objects contain single bytes -- the former is\nimmutable while the latter is a mutable sequence. Bytes objects can\nbe constructed the constructor, ``bytes()``, and from literals; use a\n``b`` prefix with normal string syntax: ``b\'xyzzy\'``. To construct\nbyte arrays, use the ``bytearray()`` function.\n\nWarning: While string objects are sequences of characters (represented by\n strings of length 1), bytes and bytearray objects are sequences of\n *integers* (between 0 and 255), representing the ASCII value of\n single bytes. That means that for a bytes or bytearray object *b*,\n ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes or\n bytearray object of length 1. The representation of bytes objects\n uses the literal format (``b\'...\'``) since it is generally more\n useful than e.g. ``bytes([50, 19, 100])``. You can always convert a\n bytes object into a list of integers using ``list(b)``.Also, while\n in previous Python versions, byte strings and Unicode strings could\n be exchanged for each other rather freely (barring encoding issues),\n strings and bytes are now completely separate concepts. There\'s no\n implicit en-/decoding if you pass an object of the wrong type. A\n string always compares unequal to a bytes or bytearray object.\n\nLists are constructed with square brackets, separating items with\ncommas: ``[a, b, c]``. Tuples are constructed by the comma operator\n(not within square brackets), with or without enclosing parentheses,\nbut an empty tuple must have the enclosing parentheses, such as ``a,\nb, c`` or ``()``. A single item tuple must have a trailing comma,\nsuch as ``(d,)``.\n\nObjects of type range are created using the ``range()`` function.\nThey don\'t support slicing, concatenation or repetition, and using\n``in``, ``not in``, ``min()`` or ``max()`` on them is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*\'th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must\ncompare equal and the two sequences must be of the same type and have\nthe same length. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string object, the ``in`` and ``not in`` operations\n act like a substring test.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n\nString Methods\n==============\n\nString objects support the methods listed below. Note that none of\nthese methods take keyword arguments.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, bytes, bytearray, list,\ntuple, range* section. To output formatted strings, see the *String\nFormatting* section. Also, see the ``re`` module for string functions\nbased on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string as a bytes object. Default\n encoding is the current default string encoding. *errors* may be\n given to set a different error handling scheme. The default for\n *errors* is ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n Changed in version 3.1: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The *format_string*\n argument can contain literal text or replacement fields delimited\n by braces ``{}``. Each replacement field contains either the\n numeric index of a positional argument, or the name of a keyword\n argument. Returns a copy of *format_string* where each replacement\n field is replaced with the string value of the corresponding\n argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise. A character\n ``c`` is alphanumeric if one of the following returns ``True``:\n ``c.isalpha()``, ``c.isdecimal()``, ``c.isdigit()``, or\n ``c.isnumeric()``.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise. Alphabetic\n characters are those characters defined in the Unicode character\n database as "Letter", i.e., those with general category property\n being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is\n different from the "Alphabetic" property defined in the Unicode\n Standard.\n\nstr.isdecimal()\n\n Return true if all characters in the string are decimal characters\n and there is at least one character, false otherwise. Decimal\n characters are those from general category "Nd". This category\n includes digit characters, and all characters that that can be used\n to form decimal-radix numbers, e.g. U+0660, ARABIC-INDIC DIGIT\n ZERO.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise. Digits include decimal\n characters and digits that need special handling, such as the\n compatibility superscript digits. Formally, a digit is a character\n that has the property value Numeric_Type=Digit or\n Numeric_Type=Decimal.\n\nstr.isidentifier()\n\n Return true if the string is a valid identifier according to the\n language definition, section *Identifiers and keywords*.\n\nstr.islower()\n\n Return true if all cased characters in the string are lowercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and lowercase characters are those with general\n category property "Ll".\n\nstr.isnumeric()\n\n Return true if all characters in the string are numeric characters,\n and there is at least one character, false otherwise. Numeric\n characters include digit characters, and all characters that have\n the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION\n ONE FIFTH. Formally, numeric characters are those with the\n property value Numeric_Type=Digit, Numeric_Type=Decimal or\n Numeric_Type=Numeric.\n\nstr.isprintable()\n\n Return true if all characters in the string are printable or the\n string is empty, false otherwise. Nonprintable characters are\n those characters defined in the Unicode character database as\n "Other" or "Separator", excepting the ASCII space (0x20) which is\n considered printable. (Note that printable characters in this\n context are those which should not be escaped when ``repr()`` is\n invoked on a string. It has no bearing on the handling of strings\n written to ``sys.stdout`` or ``sys.stderr``.)\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise. Whitespace\n characters are those characters defined in the Unicode character\n database as "Other" or "Separator" and those with bidirectional\n property being one of "WS", "B", or "S".\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\nstr.isupper()\n\n Return true if all cased characters in the string are uppercase and\n there is at least one cased character, false otherwise. Cased\n characters are those with general category property being one of\n "Lu", "Ll", or "Lt" and uppercase characters are those with general\n category property "Lu".\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. A ``TypeError`` will be raised if there are\n any non-string values in *seq*, including ``bytes`` objects. The\n separator between elements is the string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.lower()\n\n Return a copy of the string converted to lowercase.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\nstatic str.maketrans(x[, y[, z]])\n\n This static method returns a translation table usable for\n ``str.translate()``.\n\n If there is only one argument, it must be a dictionary mapping\n Unicode ordinals (integers) or characters (strings of length 1) to\n Unicode ordinals, strings (of arbitrary lengths) or None.\n Character keys will then be converted to ordinals.\n\n If there are two arguments, they must be strings of equal length,\n and in the resulting dictionary, each character in x will be mapped\n to the character at the same position in y. If there is a third\n argument, it must be a string, whose characters will be mapped to\n None in the result.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than\n ``len(s)``.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified, then there is no limit\n on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n lambda mo: mo.group(0)[0].upper() +\n mo.group(0)[1:].lower(),\n s)\n\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\nstr.translate(map)\n\n Return a copy of the *s* where all characters have been mapped\n through the *map* which must be a dictionary of Unicode ordinals\n (integers) to Unicode ordinals, strings or ``None``. Unmapped\n characters are left untouched. Characters mapped to ``None`` are\n deleted.\n\n You can use ``str.maketrans()`` to create a translation map from\n character-to-character mappings in different formats.\n\n Note: An even more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see\n ``encodings.cp1251`` for an example).\n\nstr.upper()\n\n Return a copy of the string converted to uppercase.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than ``len(s)``.\n\n\nOld String Formatting Operations\n================================\n\nNote: The formatting operations described here are obsolete and may go\n away in future versions of Python. Use the new *String Formatting*\n in new code.\n\nString objects have one unique built-in operation: the ``%`` operator\n(modulo). This is also known as the string *formatting* or\n*interpolation* operator. Given ``format % values`` (where *format* is\na string), ``%`` conversion specifications in *format* are replaced\nwith zero or more elements of *values*. The effect is similar to the\nusing ``sprintf()`` in the C language.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [4] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print(\'%(language)s has %(number)03d quote types.\' %\n... {\'language\': "Python", "number": 2})\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The precision determines the maximal number of characters used.\n\n1. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 3.1: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nRange Type\n==========\n\nThe ``range`` type is an immutable sequence which is commonly used for\nlooping. The advantage of the ``range`` type is that an ``range``\nobject will always take the same amount of memory, no matter the size\nof the range it represents. There are no consistent performance\nadvantages.\n\nRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n\n\nBytes and Byte Array Methods\n============================\n\nBytes and bytearray objects, being "strings of bytes", have all\nmethods found on strings, with the exception of ``encode()``,\n``format()`` and ``isidentifier()``, which do not make sense with\nthese types. For converting the objects to strings, they have a\n``decode()`` method.\n\nWherever one of these methods needs to interpret the bytes as\ncharacters (e.g. the ``is...()`` methods), the ASCII character set is\nassumed.\n\nNote: The methods on bytes and bytearray objects don\'t accept strings as\n their arguments, just as the methods on strings don\'t accept bytes\n as their arguments. For example, you have to write\n\n a = "abc"\n b = a.replace("a", "f")\n\n and\n\n a = b"abc"\n b = a.replace(b"a", b"f")\n\nbytes.decode([encoding[, errors]])\nbytearray.decode([encoding[, errors]])\n\n Return a string decoded from the given bytes. Default encoding is\n the current default string encoding. *errors* may be given to set\n a different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'`` and any other name registered via\n ``codecs.register_error()``, see section *Codec Base Classes*. For\n a list of possible encodings, see section *Standard Encodings*.\n\nThe bytes and bytearray types have an additional class method:\n\nclassmethod bytes.fromhex(string)\nclassmethod bytearray.fromhex(string)\n\n This ``bytes`` class method returns a bytes or bytearray object,\n decoding the given string object. The string must contain two\n hexadecimal digits per byte, spaces are ignored.\n\n >>> bytes.fromhex(\'f0 f1f2 \')\n b\'\\xf0\\xf1\\xf2\'\n\nThe maketrans and translate methods differ in semantics from the\nversions available on strings:\n\nbytes.translate(table[, delete])\nbytearray.translate(table[, delete])\n\n Return a copy of the bytes or bytearray object where all bytes\n occurring in the optional argument *delete* are removed, and the\n remaining bytes have been mapped through the given translation\n table, which must be a bytes object of length 256.\n\n You can use the ``bytes.maketrans()`` method to create a\n translation table.\n\n Set the *table* argument to ``None`` for translations that only\n delete characters:\n\n >>> b\'read this short text\'.translate(None, b\'aeiou\')\n b\'rd ths shrt txt\'\n\nstatic bytes.maketrans(from, to)\nstatic bytearray.maketrans(from, to)\n\n This static method returns a translation table usable for\n ``bytes.translate()`` that will map each character in *from* into\n the character at the same position in *to*; *from* and *to* must be\n bytes objects and have the same length.\n\n New in version 3.1.\n', 'typesseq-mutable': '\nMutable Sequence Types\n**********************\n\nList and bytearray objects support additional operations that allow\nin-place modification of the object. Other mutable sequence types\n(when added to the language) should also support these operations.\nStrings and tuples are immutable sequence types: such objects cannot\nbe modified once created. The following operations are defined on\nmutable sequence types (where *x* is an arbitrary object).\n\nNote that while lists allow their items to be of any type, bytearray\nobject "items" are all integers in the range 0 <= x < 256.\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (2) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (3) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (5) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (6) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([key[, reverse]])`` | sort the items of *s* in place | (6), (7), (8) |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. *x* can be any iterable object.\n\n3. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the sequence length is added, as for slice indices. If it\n is still negative, it is truncated to zero, as for slice indices.\n\n4. When a negative index is passed as the first parameter to the\n ``insert()`` method, the sequence length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n5. The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n6. The ``sort()`` and ``reverse()`` methods modify the sequence in\n place for economy of space when sorting or reversing a large\n sequence. To remind you that they operate by side effect, they\n don\'t return the sorted or reversed sequence.\n\n7. The ``sort()`` method takes optional arguments for controlling the\n comparisons. Each must be specified as a keyword argument.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n The ``sort()`` method is guaranteed to be stable. A sort is stable\n if it guarantees not to change the relative order of elements that\n compare equal --- this is helpful for sorting in multiple passes\n (for example, sort by department, then by salary grade).\n\n **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python makes the list appear\n empty for the duration, and raises ``ValueError`` if it can detect\n that the list has been mutated during a sort.\n\n8. ``sort()`` is not supported by ``bytearray`` objects.\n', 'unary': '\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\ninteger argument. The bitwise inversion of ``x`` is defined as\n``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': '\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', -- cgit v0.12 From bb11bb0dbb9fc05072317d2bd59d8d4fa470d41e Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 29 May 2011 16:10:15 -0500 Subject: Added tag v3.1.4rc1 for changeset 32fcb9e94985 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 720a533..680baaf 100644 --- a/.hgtags +++ b/.hgtags @@ -75,3 +75,4 @@ a69a031ac1402dede8b1ef80096436bca6d371f3 v3.1 960efa327c5d9c18df995437b0ac550cb89c9f85 v3.1.2 d18e9d71f369d8211f6ac87252c6d3211f9bd09f v3.1.3rc1 a4f75773c0060cee38b0bb651a7aba6f56b0e996 v3.1.3 +32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1 -- cgit v0.12 From d8e08067b96d677cf68e18512cb5b7b60f18c558 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 29 May 2011 15:28:54 -0700 Subject: Issue #10736: Fix test_ttk test_widgets failures with Cocoa Tk 8.5.9 on Mac OS X. (Patch by Ronald Oussoren) --- Lib/tkinter/test/test_ttk/test_widgets.py | 20 ++++++++++++++++---- Misc/NEWS | 3 +++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index cd00a61..f5c0f17 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -3,6 +3,7 @@ import tkinter import os from tkinter import ttk from test.support import requires, run_unittest +import sys import tkinter.test.support as support from tkinter.test.test_ttk.test_functions import MockTclObj, MockStateSpec @@ -561,11 +562,19 @@ class NotebookTest(unittest.TestCase): self.nb.pack() self.nb.wait_visibility() - self.assertEqual(self.nb.tab('@5,5'), self.nb.tab('current')) + if sys.platform == 'darwin': + tb_idx = "@20,5" + else: + tb_idx = "@5,5" + self.assertEqual(self.nb.tab(tb_idx), self.nb.tab('current')) for i in range(5, 100, 5): - if self.nb.tab('@%d, 5' % i, text=None) == 'a': - break + try: + if self.nb.tab('@%d, 5' % i, text=None) == 'a': + break + except tkinter.TclError: + pass + else: self.fail("Tab with text 'a' not found") @@ -722,7 +731,10 @@ class NotebookTest(unittest.TestCase): self.nb.enable_traversal() self.nb.focus_force() support.simulate_mouse_click(self.nb, 5, 5) - self.nb.event_generate('') + if sys.platform == 'darwin': + self.nb.event_generate('') + else: + self.nb.event_generate('') self.assertEqual(self.nb.select(), str(self.child1)) diff --git a/Misc/NEWS b/Misc/NEWS index fde085b..c9bcebb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -770,6 +770,9 @@ Extension Modules Tests ----- +- Issue #10736: Fix test_ttk test_widgets failures with Cocoa Tk 8.5.9 + on Mac OS X. (Patch by Ronald Oussoren) + - Issue #12180: Fixed a few remaining errors in test_packaging when no threading. -- cgit v0.12 From 65bf417fe1e061169c93fd5030611410ad9d0e0e Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Sun, 29 May 2011 15:39:11 -0700 Subject: Issue #10736: Revert changeset de4d34a95603 as it causes failures with old Apple-supplied Carbon Tk 8.4. --- Lib/tkinter/test/test_ttk/test_widgets.py | 20 ++++---------------- Misc/NEWS | 3 --- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index f5c0f17..cd00a61 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -3,7 +3,6 @@ import tkinter import os from tkinter import ttk from test.support import requires, run_unittest -import sys import tkinter.test.support as support from tkinter.test.test_ttk.test_functions import MockTclObj, MockStateSpec @@ -562,19 +561,11 @@ class NotebookTest(unittest.TestCase): self.nb.pack() self.nb.wait_visibility() - if sys.platform == 'darwin': - tb_idx = "@20,5" - else: - tb_idx = "@5,5" - self.assertEqual(self.nb.tab(tb_idx), self.nb.tab('current')) + self.assertEqual(self.nb.tab('@5,5'), self.nb.tab('current')) for i in range(5, 100, 5): - try: - if self.nb.tab('@%d, 5' % i, text=None) == 'a': - break - except tkinter.TclError: - pass - + if self.nb.tab('@%d, 5' % i, text=None) == 'a': + break else: self.fail("Tab with text 'a' not found") @@ -731,10 +722,7 @@ class NotebookTest(unittest.TestCase): self.nb.enable_traversal() self.nb.focus_force() support.simulate_mouse_click(self.nb, 5, 5) - if sys.platform == 'darwin': - self.nb.event_generate('') - else: - self.nb.event_generate('') + self.nb.event_generate('') self.assertEqual(self.nb.select(), str(self.child1)) diff --git a/Misc/NEWS b/Misc/NEWS index c9bcebb..fde085b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -770,9 +770,6 @@ Extension Modules Tests ----- -- Issue #10736: Fix test_ttk test_widgets failures with Cocoa Tk 8.5.9 - on Mac OS X. (Patch by Ronald Oussoren) - - Issue #12180: Fixed a few remaining errors in test_packaging when no threading. -- cgit v0.12 From 98838bac9c1ac67a910e19812e4c9f6e77ef2378 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Mon, 30 May 2011 01:12:24 +0200 Subject: Miscellaneous cleanups to bz2 and test_bz2 following issue #1625. * In bz2.decompress(), concatenate partial results in a way that should be more friendly to other Python implementations * Remove redundant comments in test_bz2 * Use 'while True:' instead of 'while 1:' --- Lib/bz2.py | 6 ++-- Lib/test/test_bz2.py | 89 +++++++++++----------------------------------------- 2 files changed, 22 insertions(+), 73 deletions(-) diff --git a/Lib/bz2.py b/Lib/bz2.py index 4b25f5d..cc71ae0 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -400,14 +400,14 @@ def decompress(data): if len(data) == 0: return b"" - result = b"" + results = [] while True: decomp = BZ2Decompressor() - result += decomp.decompress(data) + results.append(decomp.decompress(data)) if not decomp.eof: raise ValueError("Compressed data ended before the " "end-of-stream marker was reached") if not decomp.unused_data: - return result + return b"".join(results) # There is unused data left over. Proceed to next stream. data = decomp.unused_data diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 4d66840..5783cae 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -89,22 +89,20 @@ class BZ2FileTest(BaseTest): f.write(self.getData(crlf) * streams) def testRead(self): - # "Test BZ2File.read()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT) def testReadMultiStream(self): - # "Test BZ2File.read() with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT * 5) def testReadMonkeyMultiStream(self): - # "Test BZ2File.read() with a multi stream archive in which stream" - # "end is alined with internal buffer size" + # Test BZ2File.read() on a multi-stream archive where a stream + # boundary coincides with the end of the raw read buffer. buffer_size = bz2._BUFFER_SIZE bz2._BUFFER_SIZE = len(self.DATA) try: @@ -116,18 +114,16 @@ class BZ2FileTest(BaseTest): bz2._BUFFER_SIZE = buffer_size def testRead0(self): - # "Test BBZ2File.read(0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(0), b"") def testReadChunk10(self): - # "Test BZ2File.read() in chunks of 10 bytes" self.createTempFile() with BZ2File(self.filename) as bz2f: text = b'' - while 1: + while True: str = bz2f.read(10) if not str: break @@ -135,11 +131,10 @@ class BZ2FileTest(BaseTest): self.assertEqual(text, self.TEXT) def testReadChunk10MultiStream(self): - # "Test BZ2File.read() in chunks of 10 bytes with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: text = b'' - while 1: + while True: str = bz2f.read(10) if not str: break @@ -147,13 +142,11 @@ class BZ2FileTest(BaseTest): self.assertEqual(text, self.TEXT * 5) def testRead100(self): - # "Test BZ2File.read(100)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(100), self.TEXT[:100]) def testPeek(self): - # "Test BZ2File.peek()" self.createTempFile() with BZ2File(self.filename) as bz2f: pdata = bz2f.peek() @@ -162,7 +155,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT) def testReadInto(self): - # "Test BZ2File.readinto()" self.createTempFile() with BZ2File(self.filename) as bz2f: n = 128 @@ -175,7 +167,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(b[:n], self.TEXT[-n:]) def testReadLine(self): - # "Test BZ2File.readline()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) @@ -184,7 +175,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.readline(), line) def testReadLineMultiStream(self): - # "Test BZ2File.readline() with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) @@ -193,7 +183,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.readline(), line) def testReadLines(self): - # "Test BZ2File.readlines()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) @@ -201,7 +190,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.readlines(), sio.readlines()) def testReadLinesMultiStream(self): - # "Test BZ2File.readlines() with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) @@ -209,22 +197,19 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.readlines(), sio.readlines()) def testIterator(self): - # "Test iter(BZ2File)" self.createTempFile() with BZ2File(self.filename) as bz2f: sio = BytesIO(self.TEXT) self.assertEqual(list(iter(bz2f)), sio.readlines()) def testIteratorMultiStream(self): - # "Test iter(BZ2File) with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: sio = BytesIO(self.TEXT * 5) self.assertEqual(list(iter(bz2f)), sio.readlines()) def testClosedIteratorDeadlock(self): - # "Test that iteration on a closed bz2file releases the lock." - # http://bugs.python.org/issue3309 + # Issue #3309: Iteration on a closed BZ2File should release the lock. self.createTempFile() bz2f = BZ2File(self.filename) bz2f.close() @@ -234,7 +219,6 @@ class BZ2FileTest(BaseTest): self.assertRaises(ValueError, bz2f.readlines) def testWrite(self): - # "Test BZ2File.write()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) @@ -242,10 +226,9 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT) def testWriteChunks10(self): - # "Test BZ2File.write() with chunks of 10 bytes" with BZ2File(self.filename, "w") as bz2f: n = 0 - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -255,12 +238,12 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT) def testWriteLines(self): - # "Test BZ2File.writelines()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.writelines) sio = BytesIO(self.TEXT) bz2f.writelines(sio.readlines()) - # patch #1535500 + # Issue #1535500: Calling writelines() on a closed BZ2File + # should raise an exception. self.assertRaises(ValueError, bz2f.writelines, ["a"]) with open(self.filename, 'rb') as f: self.assertEqual(self.decompress(f.read()), self.TEXT) @@ -274,7 +257,6 @@ class BZ2FileTest(BaseTest): self.assertRaises(IOError, bz2f.writelines, [b"a"]) def testAppend(self): - # "Test BZ2File.write()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) @@ -285,15 +267,13 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT * 2) def testSeekForward(self): - # "Test BZ2File.seek(150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.seek) bz2f.seek(150) self.assertEqual(bz2f.read(), self.TEXT[150:]) - def testSeekForwardMultiStream(self): - # "Test BZ2File.seek(150, 0) across stream boundaries" + def testSeekForwardAcrossStreams(self): self.createTempFile(streams=2) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.seek) @@ -301,15 +281,13 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT[150:]) def testSeekBackwards(self): - # "Test BZ2File.seek(-150, 1)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.read(500) bz2f.seek(-150, 1) self.assertEqual(bz2f.read(), self.TEXT[500-150:]) - def testSeekBackwardsMultiStream(self): - # "Test BZ2File.seek(-150, 1) across stream boundaries" + def testSeekBackwardsAcrossStreams(self): self.createTempFile(streams=2) with BZ2File(self.filename) as bz2f: readto = len(self.TEXT) + 100 @@ -319,21 +297,18 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT[100-150:] + self.TEXT) def testSeekBackwardsFromEnd(self): - # "Test BZ2File.seek(-150, 2)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150, 2) self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) - def testSeekBackwardsFromEndMultiStream(self): - # "Test BZ2File.seek(-1000, 2) across stream boundaries" + def testSeekBackwardsFromEndAcrossStreams(self): self.createTempFile(streams=2) with BZ2File(self.filename) as bz2f: bz2f.seek(-1000, 2) self.assertEqual(bz2f.read(), (self.TEXT * 2)[-1000:]) def testSeekPostEnd(self): - # "Test BZ2File.seek(150000)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -341,7 +316,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), b"") def testSeekPostEndMultiStream(self): - # "Test BZ2File.seek(150000)" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -349,7 +323,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), b"") def testSeekPostEndTwice(self): - # "Test BZ2File.seek(150000) twice" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -358,7 +331,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), b"") def testSeekPostEndTwiceMultiStream(self): - # "Test BZ2File.seek(150000) twice with a multi stream archive" self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -367,7 +339,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), b"") def testSeekPreStart(self): - # "Test BZ2File.seek(-150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150) @@ -375,7 +346,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT) def testSeekPreStartMultiStream(self): - # "Test BZ2File.seek(-150, 0) with a multi stream archive" self.createTempFile(streams=2) with BZ2File(self.filename) as bz2f: bz2f.seek(-150) @@ -383,25 +353,22 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT * 2) def testFileno(self): - # "Test BZ2File.fileno()" self.createTempFile() with open(self.filename) as rawf: with BZ2File(fileobj=rawf) as bz2f: self.assertEqual(bz2f.fileno(), rawf.fileno()) def testOpenDel(self): - # "Test opening and deleting a file many times" self.createTempFile() for i in range(10000): o = BZ2File(self.filename) del o def testOpenNonexistent(self): - # "Test opening a nonexistent file" self.assertRaises(IOError, BZ2File, "/non/existent") - def testBug1191043(self): - # readlines() for files containing no newline + def testReadlinesNoNewline(self): + # Issue #1191043: readlines() fails on a file containing no newline. data = b'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' with open(self.filename, "wb") as f: f.write(data) @@ -413,7 +380,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - # BZ2File supports the context management protocol f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") @@ -436,7 +402,7 @@ class BZ2FileTest(BaseTest): @unittest.skipUnless(threading, 'Threading required for this test.') def testThreading(self): - # Using a BZ2File from several threads doesn't deadlock (issue #7205). + # Issue #7205: Using a BZ2File from several threads shouldn't deadlock. data = b"1" * 2**20 nthreads = 10 with bz2.BZ2File(self.filename, 'wb') as f: @@ -449,8 +415,7 @@ class BZ2FileTest(BaseTest): for t in threads: t.join() - def testMixedIterationReads(self): - # "Test mixed iteration and reads." + def testMixedIterationAndReads(self): self.createTempFile() linelen = len(self.TEXT_LINES[0]) halflen = linelen // 2 @@ -468,8 +433,9 @@ class BZ2FileTest(BaseTest): next(bz2f) self.assertEqual(bz2f.readlines(), []) + # Tests for a BZ2File wrapping another file object: + def testReadBytesIO(self): - # "Test BZ2File.read() with BytesIO source" with BytesIO(self.getData()) as bio: with BZ2File(fileobj=bio) as bz2f: self.assertRaises(TypeError, bz2f.read, None) @@ -477,7 +443,6 @@ class BZ2FileTest(BaseTest): self.assertFalse(bio.closed) def testPeekBytesIO(self): - # "Test BZ2File.peek() with BytesIO source" with BytesIO(self.getData()) as bio: with BZ2File(fileobj=bio) as bz2f: pdata = bz2f.peek() @@ -486,7 +451,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT) def testWriteBytesIO(self): - # "Test BZ2File.write() with BytesIO destination" with BytesIO() as bio: with BZ2File(fileobj=bio, mode="w") as bz2f: self.assertRaises(TypeError, bz2f.write) @@ -495,7 +459,6 @@ class BZ2FileTest(BaseTest): self.assertFalse(bio.closed) def testSeekForwardBytesIO(self): - # "Test BZ2File.seek(150, 0) with BytesIO source" with BytesIO(self.getData()) as bio: with BZ2File(fileobj=bio) as bz2f: self.assertRaises(TypeError, bz2f.seek) @@ -503,7 +466,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.read(), self.TEXT[150:]) def testSeekBackwardsBytesIO(self): - # "Test BZ2File.seek(-150, 1) with BytesIO source" with BytesIO(self.getData()) as bio: with BZ2File(fileobj=bio) as bz2f: bz2f.read(500) @@ -512,7 +474,6 @@ class BZ2FileTest(BaseTest): class BZ2CompressorTest(BaseTest): def testCompress(self): - # "Test BZ2Compressor.compress()/flush()" bz2c = BZ2Compressor() self.assertRaises(TypeError, bz2c.compress) data = bz2c.compress(self.TEXT) @@ -520,11 +481,10 @@ class BZ2CompressorTest(BaseTest): self.assertEqual(self.decompress(data), self.TEXT) def testCompressChunks10(self): - # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" bz2c = BZ2Compressor() n = 0 data = b'' - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -555,18 +515,16 @@ class BZ2DecompressorTest(BaseTest): self.assertRaises(TypeError, BZ2Decompressor, 42) def testDecompress(self): - # "Test BZ2Decompressor.decompress()" bz2d = BZ2Decompressor() self.assertRaises(TypeError, bz2d.decompress) text = bz2d.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressChunks10(self): - # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" bz2d = BZ2Decompressor() text = b'' n = 0 - while 1: + while True: str = self.DATA[n*10:(n+1)*10] if not str: break @@ -575,7 +533,6 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(text, self.TEXT) def testDecompressUnusedData(self): - # "Test BZ2Decompressor.decompress() with unused data" bz2d = BZ2Decompressor() unused_data = b"this is unused data" text = bz2d.decompress(self.DATA+unused_data) @@ -583,7 +540,6 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(bz2d.unused_data, unused_data) def testEOFError(self): - # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError" bz2d = BZ2Decompressor() text = bz2d.decompress(self.DATA) self.assertRaises(EOFError, bz2d.decompress, b"anything") @@ -609,26 +565,21 @@ class FuncTest(BaseTest): "Test module functions" def testCompress(self): - # "Test compress() function" data = bz2.compress(self.TEXT) self.assertEqual(self.decompress(data), self.TEXT) def testDecompress(self): - # "Test decompress() function" text = bz2.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressEmpty(self): - # "Test decompress() function with empty string" text = bz2.decompress(b"") self.assertEqual(text, b"") def testDecompressIncomplete(self): - # "Test decompress() function with incomplete data" self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) def testDecompressMultiStream(self): - # "Test decompress() function for data with multiple streams" text = bz2.decompress(self.DATA * 5) self.assertEqual(text, self.TEXT * 5) @@ -643,5 +594,3 @@ def test_main(): if __name__ == '__main__': test_main() - -# vim:ts=4:sw=4 -- cgit v0.12 From 07d291ae595ef2414e35688dfba9a9f25361cda5 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Mon, 30 May 2011 01:44:45 +0200 Subject: Remove unused data from test_bz2. DATA_CRLF was used to test BZ2File's universal newline logic, which was removed in changeset ce63a5dcb0af. The tests themselves were removed in fbabdb0d7dd2. --- Lib/test/test_bz2.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 5783cae..350d2de 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -47,7 +47,6 @@ class BaseTest(unittest.TestCase): ] TEXT = b''.join(TEXT_LINES) DATA = b'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' if has_cmdline_bunzip2: def decompress(self, data): @@ -78,15 +77,9 @@ class BZ2FileTest(BaseTest): if os.path.isfile(self.filename): os.unlink(self.filename) - def getData(self, crlf=False): - if crlf: - return self.DATA_CRLF - else: - return self.DATA - - def createTempFile(self, crlf=False, streams=1): + def createTempFile(self, streams=1): with open(self.filename, "wb") as f: - f.write(self.getData(crlf) * streams) + f.write(self.DATA * streams) def testRead(self): self.createTempFile() @@ -436,14 +429,14 @@ class BZ2FileTest(BaseTest): # Tests for a BZ2File wrapping another file object: def testReadBytesIO(self): - with BytesIO(self.getData()) as bio: + with BytesIO(self.DATA) as bio: with BZ2File(fileobj=bio) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT) self.assertFalse(bio.closed) def testPeekBytesIO(self): - with BytesIO(self.getData()) as bio: + with BytesIO(self.DATA) as bio: with BZ2File(fileobj=bio) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) @@ -459,14 +452,14 @@ class BZ2FileTest(BaseTest): self.assertFalse(bio.closed) def testSeekForwardBytesIO(self): - with BytesIO(self.getData()) as bio: + with BytesIO(self.DATA) as bio: with BZ2File(fileobj=bio) as bz2f: self.assertRaises(TypeError, bz2f.seek) bz2f.seek(150) self.assertEqual(bz2f.read(), self.TEXT[150:]) def testSeekBackwardsBytesIO(self): - with BytesIO(self.getData()) as bio: + with BytesIO(self.DATA) as bio: with BZ2File(fileobj=bio) as bz2f: bz2f.read(500) bz2f.seek(-150, 1) -- cgit v0.12 From f3ecb8382222577ce9aef9901b91c9ba69a41f32 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Mon, 30 May 2011 01:58:12 +0200 Subject: Issue #1625: Add stream ordering test to test_bz2. --- Lib/test/test_bz2.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 350d2de..e2c34d5 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -426,6 +426,17 @@ class BZ2FileTest(BaseTest): next(bz2f) self.assertEqual(bz2f.readlines(), []) + def testMultiStreamOrdering(self): + # Test the ordering of streams when reading a multi-stream archive. + data1 = b"foo" * 1000 + data2 = b"bar" * 1000 + with BZ2File(self.filename, "w") as bz2f: + bz2f.write(data1) + with BZ2File(self.filename, "a") as bz2f: + bz2f.write(data2) + with BZ2File(self.filename) as bz2f: + self.assertEqual(bz2f.read(), data1 + data2) + # Tests for a BZ2File wrapping another file object: def testReadBytesIO(self): -- cgit v0.12 From 72e58651b27e8b0a8f4170531ca6e985d9947d1b Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Mon, 30 May 2011 02:09:54 +0200 Subject: Simplify line-oriented tests in test_bz2. --- Lib/test/test_bz2.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index e2c34d5..75078be 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -163,43 +163,37 @@ class BZ2FileTest(BaseTest): self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) - sio = BytesIO(self.TEXT) - for line in sio.readlines(): + for line in self.TEXT_LINES: self.assertEqual(bz2f.readline(), line) def testReadLineMultiStream(self): self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) - sio = BytesIO(self.TEXT * 5) - for line in sio.readlines(): + for line in self.TEXT_LINES * 5: self.assertEqual(bz2f.readline(), line) def testReadLines(self): self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) - sio = BytesIO(self.TEXT) - self.assertEqual(bz2f.readlines(), sio.readlines()) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES) def testReadLinesMultiStream(self): self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) - sio = BytesIO(self.TEXT * 5) - self.assertEqual(bz2f.readlines(), sio.readlines()) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES * 5) def testIterator(self): self.createTempFile() with BZ2File(self.filename) as bz2f: - sio = BytesIO(self.TEXT) - self.assertEqual(list(iter(bz2f)), sio.readlines()) + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES) def testIteratorMultiStream(self): self.createTempFile(streams=5) with BZ2File(self.filename) as bz2f: - sio = BytesIO(self.TEXT * 5) - self.assertEqual(list(iter(bz2f)), sio.readlines()) + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES * 5) def testClosedIteratorDeadlock(self): # Issue #3309: Iteration on a closed BZ2File should release the lock. @@ -233,8 +227,7 @@ class BZ2FileTest(BaseTest): def testWriteLines(self): with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.writelines) - sio = BytesIO(self.TEXT) - bz2f.writelines(sio.readlines()) + bz2f.writelines(self.TEXT_LINES) # Issue #1535500: Calling writelines() on a closed BZ2File # should raise an exception. self.assertRaises(ValueError, bz2f.writelines, ["a"]) -- cgit v0.12 From a17d8883fd6b001bef57cc1f7cfdb0504446d9db Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 10:57:44 +0200 Subject: a resource module for 2 functions is overkill. database is the right place for those --- Lib/packaging/database.py | 15 +++ Lib/packaging/resources.py | 25 ----- Lib/packaging/tests/test_database.py | 159 +++++++++++++++++++++++++++++++- Lib/packaging/tests/test_resources.py | 167 ---------------------------------- 4 files changed, 171 insertions(+), 195 deletions(-) delete mode 100644 Lib/packaging/resources.py delete mode 100644 Lib/packaging/tests/test_resources.py diff --git a/Lib/packaging/database.py b/Lib/packaging/database.py index b107148..010c4eb 100644 --- a/Lib/packaging/database.py +++ b/Lib/packaging/database.py @@ -18,6 +18,7 @@ __all__ = [ 'get_distributions', 'get_distribution', 'get_file_users', 'provides_distribution', 'obsoletes_distribution', 'enable_cache', 'disable_cache', 'clear_cache', + 'get_file_path', 'get_file' ] @@ -627,3 +628,17 @@ def get_file_users(path): for dist in get_distributions(): if dist.uses(path): yield dist + + +def get_file_path(distribution_name, relative_path): + """Return the path to a resource file.""" + dist = get_distribution(distribution_name) + if dist != None: + return dist.get_resource_path(relative_path) + raise LookupError('no distribution named %r found' % distribution_name) + + +def get_file(distribution_name, relative_path, *args, **kwargs): + """Open and return a resource file.""" + return open(get_file_path(distribution_name, relative_path), + *args, **kwargs) diff --git a/Lib/packaging/resources.py b/Lib/packaging/resources.py deleted file mode 100644 index e5904f3..0000000 --- a/Lib/packaging/resources.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Data file path abstraction. - -Functions in this module use sysconfig to find the paths to the resource -files registered in project's setup.cfg file. See the documentation for -more information. -""" -# TODO write that documentation - -from packaging.database import get_distribution - -__all__ = ['get_file_path', 'get_file'] - - -def get_file_path(distribution_name, relative_path): - """Return the path to a resource file.""" - dist = get_distribution(distribution_name) - if dist != None: - return dist.get_resource_path(relative_path) - raise LookupError('no distribution named %r found' % distribution_name) - - -def get_file(distribution_name, relative_path, *args, **kwargs): - """Open and return a resource file.""" - return open(get_file_path(distribution_name, relative_path), - *args, **kwargs) diff --git a/Lib/packaging/tests/test_database.py b/Lib/packaging/tests/test_database.py index ea63d3e..e0c16de 100644 --- a/Lib/packaging/tests/test_database.py +++ b/Lib/packaging/tests/test_database.py @@ -8,16 +8,21 @@ import zipfile import tempfile from os.path import relpath # separate import for backport concerns from hashlib import md5 +from textwrap import dedent +from packaging.tests import unittest +from packaging.tests.test_util import GlobTestCaseBase +from packaging.tests.support import requires_zlib + +from packaging.config import get_resources_dests from packaging.errors import PackagingError from packaging.metadata import Metadata from packaging.tests import unittest, run_unittest, support, TESTFN -from packaging.tests.support import requires_zlib - from packaging.database import ( Distribution, EggInfoDistribution, get_distribution, get_distributions, provides_distribution, obsoletes_distribution, get_file_users, - enable_cache, disable_cache, distinfo_dirname, _yield_distributions) + enable_cache, disable_cache, distinfo_dirname, _yield_distributions, + get_file, get_file_path) # TODO Add a test for getting a distribution provided by another distribution # TODO Add a test for absolute pathed RECORD items (e.g. /etc/myapp/config.ini) @@ -504,12 +509,160 @@ class TestDatabase(support.LoggingCatcher, checkLists(dists + eggs, found) +class DataFilesTestCase(GlobTestCaseBase): + + def assertRulesMatch(self, rules, spec): + tempdir = self.build_files_tree(spec) + expected = self.clean_tree(spec) + result = get_resources_dests(tempdir, rules) + self.assertEqual(expected, result) + + def clean_tree(self, spec): + files = {} + for path, value in spec.items(): + if value is not None: + files[path] = value + return files + + def test_simple_glob(self): + rules = [('', '*.tpl', '{data}')] + spec = {'coucou.tpl': '{data}/coucou.tpl', + 'Donotwant': None} + self.assertRulesMatch(rules, spec) + + def test_multiple_match(self): + rules = [('scripts', '*.bin', '{appdata}'), + ('scripts', '*', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match(self): + rules = [('scripts', '*.{bin,sh}', '{appscript}')] + spec = {'scripts/script.bin': '{appscript}/script.bin', + 'scripts/babar.sh': '{appscript}/babar.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_multiple(self): + rules = [('scripts', 'script{s,}.{bin,sh}', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': '{appscript}/script.sh', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_set_match_exclude(self): + rules = [('scripts', '*', '{appscript}'), + ('', os.path.join('**', '*.sh'), None)] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripts/script.sh': None, + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_glob_in_base(self): + rules = [('scrip*', '*.bin', '{appscript}')] + spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', + 'scripouille/babar.bin': '{appscript}/babar.bin', + 'scriptortu/lotus.bin': '{appscript}/lotus.bin', + 'Babarlikestrawberry': None} + self.assertRulesMatch(rules, spec) + + def test_recursive_glob(self): + rules = [('', os.path.join('**', '*.bin'), '{binary}')] + spec = {'binary0.bin': '{binary}/binary0.bin', + 'scripts/binary1.bin': '{binary}/scripts/binary1.bin', + 'scripts/bin/binary2.bin': '{binary}/scripts/bin/binary2.bin', + 'you/kill/pandabear.guy': None} + self.assertRulesMatch(rules, spec) + + def test_final_exemple_glob(self): + rules = [ + ('mailman/database/schemas/', '*', '{appdata}/schemas'), + ('', os.path.join('**', '*.tpl'), '{appdata}/templates'), + ('', os.path.join('developer-docs', '**', '*.txt'), '{doc}'), + ('', 'README', '{doc}'), + ('mailman/etc/', '*', '{config}'), + ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), '{config}/baz'), + ('mailman/foo/', os.path.join('**', '*.cfg'), '{config}/hmm'), + ('', 'some-new-semantic.sns', '{funky-crazy-category}'), + ] + spec = { + 'README': '{doc}/README', + 'some.tpl': '{appdata}/templates/some.tpl', + 'some-new-semantic.sns': + '{funky-crazy-category}/some-new-semantic.sns', + 'mailman/database/mailman.db': None, + 'mailman/database/schemas/blah.schema': + '{appdata}/schemas/blah.schema', + 'mailman/etc/my.cnf': '{config}/my.cnf', + 'mailman/foo/some/path/bar/my.cfg': + '{config}/hmm/some/path/bar/my.cfg', + 'mailman/foo/some/path/other.cfg': + '{config}/hmm/some/path/other.cfg', + 'developer-docs/index.txt': '{doc}/developer-docs/index.txt', + 'developer-docs/api/toc.txt': '{doc}/developer-docs/api/toc.txt', + } + self.maxDiff = None + self.assertRulesMatch(rules, spec) + + def test_get_file(self): + # Create a fake dist + temp_site_packages = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, temp_site_packages) + + dist_name = 'test' + dist_info = os.path.join(temp_site_packages, 'test-0.1.dist-info') + os.mkdir(dist_info) + + metadata_path = os.path.join(dist_info, 'METADATA') + resources_path = os.path.join(dist_info, 'RESOURCES') + + with open(metadata_path, 'w') as fp: + fp.write(dedent("""\ + Metadata-Version: 1.2 + Name: test + Version: 0.1 + Summary: test + Author: me + """)) + + test_path = 'test.cfg' + + fd, test_resource_path = tempfile.mkstemp() + os.close(fd) + self.addCleanup(os.remove, test_resource_path) + + with open(test_resource_path, 'w') as fp: + fp.write('Config') + + with open(resources_path, 'w') as fp: + fp.write('%s,%s' % (test_path, test_resource_path)) + + # Add fake site-packages to sys.path to retrieve fake dist + self.addCleanup(sys.path.remove, temp_site_packages) + sys.path.insert(0, temp_site_packages) + + # Force packaging.database to rescan the sys.path + self.addCleanup(enable_cache) + disable_cache() + + # Try to retrieve resources paths and files + self.assertEqual(get_file_path(dist_name, test_path), + test_resource_path) + self.assertRaises(KeyError, get_file_path, dist_name, 'i-dont-exist') + + with get_file(dist_name, test_path) as fp: + self.assertEqual(fp.read(), 'Config') + self.assertRaises(KeyError, get_file, dist_name, 'i-dont-exist') + + def test_suite(): suite = unittest.TestSuite() load = unittest.defaultTestLoader.loadTestsFromTestCase suite.addTest(load(TestDistribution)) suite.addTest(load(TestEggInfoDistribution)) suite.addTest(load(TestDatabase)) + suite.addTest(load(DataFilesTestCase)) return suite diff --git a/Lib/packaging/tests/test_resources.py b/Lib/packaging/tests/test_resources.py deleted file mode 100644 index 1e92f2a..0000000 --- a/Lib/packaging/tests/test_resources.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Tests for packaging.resources.""" - -import os -import sys -import shutil -import tempfile -from textwrap import dedent -from packaging.config import get_resources_dests -from packaging.database import disable_cache, enable_cache -from packaging.resources import get_file, get_file_path - -from packaging.tests import unittest -from packaging.tests.test_util import GlobTestCaseBase - - -class DataFilesTestCase(GlobTestCaseBase): - - def assertRulesMatch(self, rules, spec): - tempdir = self.build_files_tree(spec) - expected = self.clean_tree(spec) - result = get_resources_dests(tempdir, rules) - self.assertEqual(expected, result) - - def clean_tree(self, spec): - files = {} - for path, value in spec.items(): - if value is not None: - files[path] = value - return files - - def test_simple_glob(self): - rules = [('', '*.tpl', '{data}')] - spec = {'coucou.tpl': '{data}/coucou.tpl', - 'Donotwant': None} - self.assertRulesMatch(rules, spec) - - def test_multiple_match(self): - rules = [('scripts', '*.bin', '{appdata}'), - ('scripts', '*', '{appscript}')] - spec = {'scripts/script.bin': '{appscript}/script.bin', - 'Babarlikestrawberry': None} - self.assertRulesMatch(rules, spec) - - def test_set_match(self): - rules = [('scripts', '*.{bin,sh}', '{appscript}')] - spec = {'scripts/script.bin': '{appscript}/script.bin', - 'scripts/babar.sh': '{appscript}/babar.sh', - 'Babarlikestrawberry': None} - self.assertRulesMatch(rules, spec) - - def test_set_match_multiple(self): - rules = [('scripts', 'script{s,}.{bin,sh}', '{appscript}')] - spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', - 'scripts/script.sh': '{appscript}/script.sh', - 'Babarlikestrawberry': None} - self.assertRulesMatch(rules, spec) - - def test_set_match_exclude(self): - rules = [('scripts', '*', '{appscript}'), - ('', os.path.join('**', '*.sh'), None)] - spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', - 'scripts/script.sh': None, - 'Babarlikestrawberry': None} - self.assertRulesMatch(rules, spec) - - def test_glob_in_base(self): - rules = [('scrip*', '*.bin', '{appscript}')] - spec = {'scripts/scripts.bin': '{appscript}/scripts.bin', - 'scripouille/babar.bin': '{appscript}/babar.bin', - 'scriptortu/lotus.bin': '{appscript}/lotus.bin', - 'Babarlikestrawberry': None} - self.assertRulesMatch(rules, spec) - - def test_recursive_glob(self): - rules = [('', os.path.join('**', '*.bin'), '{binary}')] - spec = {'binary0.bin': '{binary}/binary0.bin', - 'scripts/binary1.bin': '{binary}/scripts/binary1.bin', - 'scripts/bin/binary2.bin': '{binary}/scripts/bin/binary2.bin', - 'you/kill/pandabear.guy': None} - self.assertRulesMatch(rules, spec) - - def test_final_exemple_glob(self): - rules = [ - ('mailman/database/schemas/', '*', '{appdata}/schemas'), - ('', os.path.join('**', '*.tpl'), '{appdata}/templates'), - ('', os.path.join('developer-docs', '**', '*.txt'), '{doc}'), - ('', 'README', '{doc}'), - ('mailman/etc/', '*', '{config}'), - ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), '{config}/baz'), - ('mailman/foo/', os.path.join('**', '*.cfg'), '{config}/hmm'), - ('', 'some-new-semantic.sns', '{funky-crazy-category}'), - ] - spec = { - 'README': '{doc}/README', - 'some.tpl': '{appdata}/templates/some.tpl', - 'some-new-semantic.sns': - '{funky-crazy-category}/some-new-semantic.sns', - 'mailman/database/mailman.db': None, - 'mailman/database/schemas/blah.schema': - '{appdata}/schemas/blah.schema', - 'mailman/etc/my.cnf': '{config}/my.cnf', - 'mailman/foo/some/path/bar/my.cfg': - '{config}/hmm/some/path/bar/my.cfg', - 'mailman/foo/some/path/other.cfg': - '{config}/hmm/some/path/other.cfg', - 'developer-docs/index.txt': '{doc}/developer-docs/index.txt', - 'developer-docs/api/toc.txt': '{doc}/developer-docs/api/toc.txt', - } - self.maxDiff = None - self.assertRulesMatch(rules, spec) - - def test_get_file(self): - # Create a fake dist - temp_site_packages = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, temp_site_packages) - - dist_name = 'test' - dist_info = os.path.join(temp_site_packages, 'test-0.1.dist-info') - os.mkdir(dist_info) - - metadata_path = os.path.join(dist_info, 'METADATA') - resources_path = os.path.join(dist_info, 'RESOURCES') - - with open(metadata_path, 'w') as fp: - fp.write(dedent("""\ - Metadata-Version: 1.2 - Name: test - Version: 0.1 - Summary: test - Author: me - """)) - - test_path = 'test.cfg' - - fd, test_resource_path = tempfile.mkstemp() - os.close(fd) - self.addCleanup(os.remove, test_resource_path) - - with open(test_resource_path, 'w') as fp: - fp.write('Config') - - with open(resources_path, 'w') as fp: - fp.write('%s,%s' % (test_path, test_resource_path)) - - # Add fake site-packages to sys.path to retrieve fake dist - self.addCleanup(sys.path.remove, temp_site_packages) - sys.path.insert(0, temp_site_packages) - - # Force packaging.database to rescan the sys.path - self.addCleanup(enable_cache) - disable_cache() - - # Try to retrieve resources paths and files - self.assertEqual(get_file_path(dist_name, test_path), - test_resource_path) - self.assertRaises(KeyError, get_file_path, dist_name, 'i-dont-exist') - - with get_file(dist_name, test_path) as fp: - self.assertEqual(fp.read(), 'Config') - self.assertRaises(KeyError, get_file, dist_name, 'i-dont-exist') - - -def test_suite(): - return unittest.makeSuite(DataFilesTestCase) - -if __name__ == '__main__': - unittest.main(defaultTest='test_suite') -- cgit v0.12 From 43f289ad8882a743d03b92a430f7ab7616e96673 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 11:07:54 +0200 Subject: cleanup packaging.database --- Lib/packaging/database.py | 3 +-- Lib/packaging/tests/test_database.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/packaging/database.py b/Lib/packaging/database.py index 010c4eb..22d4b13 100644 --- a/Lib/packaging/database.py +++ b/Lib/packaging/database.py @@ -18,8 +18,7 @@ __all__ = [ 'get_distributions', 'get_distribution', 'get_file_users', 'provides_distribution', 'obsoletes_distribution', 'enable_cache', 'disable_cache', 'clear_cache', - 'get_file_path', 'get_file' -] + 'get_file_path', 'get_file'] # TODO update docs diff --git a/Lib/packaging/tests/test_database.py b/Lib/packaging/tests/test_database.py index e0c16de..e965c60 100644 --- a/Lib/packaging/tests/test_database.py +++ b/Lib/packaging/tests/test_database.py @@ -1,23 +1,20 @@ import os import io import csv -import imp import sys import shutil -import zipfile import tempfile from os.path import relpath # separate import for backport concerns from hashlib import md5 from textwrap import dedent -from packaging.tests import unittest from packaging.tests.test_util import GlobTestCaseBase from packaging.tests.support import requires_zlib from packaging.config import get_resources_dests from packaging.errors import PackagingError from packaging.metadata import Metadata -from packaging.tests import unittest, run_unittest, support, TESTFN +from packaging.tests import unittest, support from packaging.database import ( Distribution, EggInfoDistribution, get_distribution, get_distributions, provides_distribution, obsoletes_distribution, get_file_users, @@ -582,7 +579,8 @@ class DataFilesTestCase(GlobTestCaseBase): ('', os.path.join('developer-docs', '**', '*.txt'), '{doc}'), ('', 'README', '{doc}'), ('mailman/etc/', '*', '{config}'), - ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), '{config}/baz'), + ('mailman/foo/', os.path.join('**', 'bar', '*.cfg'), + '{config}/baz'), ('mailman/foo/', os.path.join('**', '*.cfg'), '{config}/hmm'), ('', 'some-new-semantic.sns', '{funky-crazy-category}'), ] -- cgit v0.12 From b1b6e1384cf180657e8301643a4563077183a9e4 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 12:07:49 +0200 Subject: Cleaned up the installer output behavior. This change also makes sure the logger handlers are not alterated after an installation. That also fixes the remaining environment alteration issue in test_packaging. --- Lib/packaging/install.py | 32 +++++++++++------------ Lib/packaging/pypi/simple.py | 10 ++++--- Lib/packaging/run.py | 45 ++++++++++++++++++++++---------- Lib/packaging/tests/test_command_test.py | 3 +-- Lib/packaging/tests/test_util.py | 44 +++++++++++++++---------------- Lib/packaging/util.py | 19 +++++++------- 6 files changed, 86 insertions(+), 67 deletions(-) diff --git a/Lib/packaging/install.py b/Lib/packaging/install.py index 92657ea..cd2bbb6 100644 --- a/Lib/packaging/install.py +++ b/Lib/packaging/install.py @@ -118,15 +118,15 @@ def install_local_project(path): """ path = os.path.abspath(path) if os.path.isdir(path): - logger.info('installing from source directory: %s', path) + logger.info('Installing from source directory: %s', path) _run_install_from_dir(path) elif _is_archive_file(path): - logger.info('installing from archive: %s', path) + logger.info('Installing from archive: %s', path) _unpacked_dir = tempfile.mkdtemp() shutil.unpack_archive(path, _unpacked_dir) _run_install_from_archive(_unpacked_dir) else: - logger.warning('no projects to install') + logger.warning('No projects to install.') def _run_install_from_archive(source_dir): @@ -174,16 +174,16 @@ def install_dists(dists, path, paths=sys.path): installed_dists = [] for dist in dists: - logger.info('installing %s %s', dist.name, dist.version) + logger.info('Installing %r %s...', dist.name, dist.version) try: _install_dist(dist, path) installed_dists.append(dist) except Exception as e: - logger.info('failed: %s', e) + logger.info('Failed: %s', e) # reverting for installed_dist in installed_dists: - logger.info('reverting %s', installed_dist) + logger.info('Reverting %s', installed_dist) _remove_dist(installed_dist, paths) raise e return installed_dists @@ -292,7 +292,7 @@ def get_infos(requirements, index=None, installed=None, prefer_final=True): # or remove if not installed: - logger.info('reading installed distributions') + logger.debug('Reading installed distributions') installed = list(get_distributions(use_egg_info=True)) infos = {'install': [], 'remove': [], 'conflict': []} @@ -306,7 +306,7 @@ def get_infos(requirements, index=None, installed=None, prefer_final=True): if predicate.name.lower() != installed_project.name.lower(): continue found = True - logger.info('found %s %s', installed_project.name, + logger.info('Found %s %s', installed_project.name, installed_project.metadata['version']) # if we already have something installed, check it matches the @@ -316,7 +316,7 @@ def get_infos(requirements, index=None, installed=None, prefer_final=True): break if not found: - logger.info('project not installed') + logger.debug('Project not installed') if not index: index = wrapper.ClientWrapper() @@ -331,7 +331,7 @@ def get_infos(requirements, index=None, installed=None, prefer_final=True): raise InstallationException('Release not found: "%s"' % requirements) if release is None: - logger.info('could not find a matching project') + logger.info('Could not find a matching project') return infos metadata = release.fetch_metadata() @@ -348,7 +348,7 @@ def get_infos(requirements, index=None, installed=None, prefer_final=True): # Get what the missing deps are dists = depgraph.missing[release] if dists: - logger.info("missing dependencies found, retrieving metadata") + logger.info("Missing dependencies found, retrieving metadata") # we have missing deps for dist in dists: _update_infos(infos, get_infos(dist, index, installed)) @@ -401,7 +401,7 @@ def remove(project_name, paths=sys.path, auto_confirm=True): finally: shutil.rmtree(tmp) - logger.info('removing %r: ', project_name) + logger.info('Removing %r: ', project_name) for file_ in rmfiles: logger.info(' %s', file_) @@ -444,20 +444,20 @@ def remove(project_name, paths=sys.path, auto_confirm=True): if os.path.exists(dist.path): shutil.rmtree(dist.path) - logger.info('success: removed %d files and %d dirs', + logger.info('Success: removed %d files and %d dirs', file_count, dir_count) def install(project): - logger.info('getting information about %r', project) + logger.info('Getting information about %r...', project) try: info = get_infos(project) except InstallationException: - logger.info('cound not find %r', project) + logger.info('Cound not find %r', project) return if info['install'] == []: - logger.info('nothing to install') + logger.info('Nothing to install') return install_path = get_config_var('base') diff --git a/Lib/packaging/pypi/simple.py b/Lib/packaging/pypi/simple.py index ee7a113..983d477 100644 --- a/Lib/packaging/pypi/simple.py +++ b/Lib/packaging/pypi/simple.py @@ -118,9 +118,10 @@ class Crawler(BaseClient): def __init__(self, index_url=DEFAULT_SIMPLE_INDEX_URL, prefer_final=False, prefer_source=True, hosts=DEFAULT_HOSTS, follow_externals=False, mirrors_url=None, mirrors=None, - timeout=SOCKET_TIMEOUT, mirrors_max_tries=0): + timeout=SOCKET_TIMEOUT, mirrors_max_tries=0, verbose=False): super(Crawler, self).__init__(prefer_final, prefer_source) self.follow_externals = follow_externals + self.verbose = verbose # mirroring attributes. parsed = urllib.parse.urlparse(index_url) @@ -184,7 +185,7 @@ class Crawler(BaseClient): if predicate.name.lower() in self._projects and not force_update: return self._projects.get(predicate.name.lower()) prefer_final = self._get_prefer_final(prefer_final) - logger.info('reading info on PyPI about %s', predicate.name) + logger.debug('Reading info on PyPI about %s', predicate.name) self._process_index_page(predicate.name) if predicate.name.lower() not in self._projects: @@ -321,8 +322,9 @@ class Crawler(BaseClient): infos = get_infos_from_url(link, project_name, is_external=not self.index_url in url) except CantParseArchiveName as e: - logger.warning( - "version has not been parsed: %s", e) + if self.verbose: + logger.warning( + "version has not been parsed: %s", e) else: self._register_release(release_info=infos) else: diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py index 1d4fadb..03b80c6 100644 --- a/Lib/packaging/run.py +++ b/Lib/packaging/run.py @@ -5,6 +5,7 @@ import re import sys import getopt import logging +from copy import copy from packaging import logger from packaging.dist import Distribution @@ -227,12 +228,13 @@ def _install(dispatcher, args, **kw): logger.warning('no project to install') return + target = args[1] # installing from a source dir or archive file? - if os.path.isdir(args[1]) or _is_archive_file(args[1]): - install_local_project(args[1]) + if os.path.isdir(target) or _is_archive_file(target): + install_local_project(target) else: # download from PyPI - install(args[1]) + install(target) @action_help(metadata_usage) @@ -399,6 +401,17 @@ class Dispatcher: msg = 'Unrecognized action "%s"' % self.action raise PackagingArgError(msg) + self._set_logger() + + # for display options we return immediately + option_order = self.parser.get_option_order() + + self.args = args + + if self.help or self.action is None: + self._show_help(self.parser, display_options_=False) + + def _set_logger(self): # setting up the logging level from the command-line options # -q gets warning, error and critical if self.verbose == 0: @@ -416,13 +429,11 @@ class Dispatcher: else: # -vv and more for debug level = logging.DEBUG - # for display options we return immediately - option_order = self.parser.get_option_order() - - self.args = args - - if self.help or self.action is None: - self._show_help(self.parser, display_options_=False) + # setting up the stream handler + handler = logging.StreamHandler(sys.stderr) + handler.setLevel(level) + logger.addHandler(handler) + logger.setLevel(level) def _parse_command_opts(self, parser, args): # Pull the current command from the head of the command line @@ -635,11 +646,17 @@ class Dispatcher: def main(args=None): - dispatcher = Dispatcher(args) - if dispatcher.action is None: - return + old_level = logger.level + old_handlers = copy(logger.handlers) + try: + dispatcher = Dispatcher(args) + if dispatcher.action is None: + return + return dispatcher() + finally: + logger.setLevel(old_level) + logger.handlers[:] = old_handlers - return dispatcher() if __name__ == '__main__': sys.exit(main()) diff --git a/Lib/packaging/tests/test_command_test.py b/Lib/packaging/tests/test_command_test.py index 4fd8452..f780723 100644 --- a/Lib/packaging/tests/test_command_test.py +++ b/Lib/packaging/tests/test_command_test.py @@ -150,8 +150,7 @@ class TestTest(TempdirManager, cmd.tests_require = [phony_project] cmd.ensure_finalized() logs = self.get_logs(logging.WARNING) - self.assertEqual(1, len(logs)) - self.assertIn(phony_project, logs[0]) + self.assertIn(phony_project, logs[-1]) def prepare_a_module(self): tmp_dir = self.mkdtemp() diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py index 61b4ec7..5a94a73 100644 --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -818,51 +818,51 @@ class PackagingLibChecks(support.TempdirManager, def test_is_setuptools_logs_setup_py_text_found(self): is_setuptools(self._setuptools_setup_py_pkg()) - expected = ['setup.py file found', 'found setuptools text in setup.py'] - self.assertEqual(expected, self.get_logs(logging.INFO)) + expected = ['setup.py file found.', + 'No egg-info directory found.', + 'Found setuptools text in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_setuptools_logs_setup_py_text_not_found(self): directory = self._random_setup_py_pkg() is_setuptools(directory) - info_expected = ['setup.py file found'] - warn_expected = ['no egg-info directory found', - 'no setuptools text found in setup.py'] - self.assertEqual(info_expected, self.get_logs(logging.INFO)) - self.assertEqual(warn_expected, self.get_logs(logging.WARN)) + expected = ['setup.py file found.', 'No egg-info directory found.', + 'No setuptools text found in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_setuptools_logs_egg_info_dir_found(self): is_setuptools(self._setuptools_egg_info_pkg()) - expected = ['setup.py file found', 'found egg-info directory'] - self.assertEqual(expected, self.get_logs(logging.INFO)) + expected = ['setup.py file found.', 'Found egg-info directory.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_distutils_logs_setup_py_text_found(self): is_distutils(self._distutils_setup_py_pkg()) - expected = ['setup.py file found', 'found distutils text in setup.py'] - self.assertEqual(expected, self.get_logs(logging.INFO)) + expected = ['setup.py file found.', + 'No PKG-INFO file found.', + 'Found distutils text in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_distutils_logs_setup_py_text_not_found(self): directory = self._random_setup_py_pkg() is_distutils(directory) - info_expected = ['setup.py file found'] - warn_expected = ['no PKG-INFO file found', - 'no distutils text found in setup.py'] - self.assertEqual(info_expected, self.get_logs(logging.INFO)) - self.assertEqual(warn_expected, self.get_logs(logging.WARN)) + expected = ['setup.py file found.', 'No PKG-INFO file found.', + 'No distutils text found in setup.py.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_distutils_logs_pkg_info_file_found(self): is_distutils(self._distutils_pkg_info()) - expected = ['setup.py file found', 'PKG-INFO file found'] - self.assertEqual(expected, self.get_logs(logging.INFO)) + expected = ['setup.py file found.', 'PKG-INFO file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_packaging_logs_setup_cfg_found(self): is_packaging(self._valid_setup_cfg_pkg()) - expected = ['setup.cfg file found'] - self.assertEqual(expected, self.get_logs(logging.INFO)) + expected = ['setup.cfg file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def test_is_packaging_logs_setup_cfg_not_found(self): is_packaging(self._empty_dir) - expected = ['no setup.cfg file found'] - self.assertEqual(expected, self.get_logs(logging.WARN)) + expected = ['No setup.cfg file found.'] + self.assertEqual(expected, self.get_logs(logging.DEBUG)) def _write_setuptools_setup_py(self, directory): self.write_file((directory, 'setup.py'), diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index 15da9e6..e839320 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -1224,9 +1224,9 @@ def _has_egg_info(srcdir): for item in os.listdir(srcdir): full_path = os.path.join(srcdir, item) if item.endswith('.egg-info') and os.path.isdir(full_path): - logger.info("found egg-info directory") + logger.debug("Found egg-info directory.") return True - logger.warning("no egg-info directory found") + logger.debug("No egg-info directory found.") return False @@ -1243,9 +1243,9 @@ def _has_text(setup_py, installer): with open(setup_py, 'r', encoding='utf-8') as setup: for line in setup: if re.search(installer_pattern, line): - logger.info("found %s text in setup.py", installer) + logger.debug("Found %s text in setup.py.", installer) return True - logger.warning("no %s text found in setup.py", installer) + logger.debug("No %s text found in setup.py.", installer) return False @@ -1261,15 +1261,16 @@ def _has_pkg_info(srcdir): pkg_info = os.path.join(srcdir, 'PKG-INFO') has_pkg_info = os.path.isfile(pkg_info) if has_pkg_info: - logger.info("PKG-INFO file found") - logger.warning("no PKG-INFO file found") + logger.debug("PKG-INFO file found.") + else: + logger.debug("No PKG-INFO file found.") return has_pkg_info def _has_setup_py(srcdir): setup_py = os.path.join(srcdir, 'setup.py') if os.path.isfile(setup_py): - logger.info('setup.py file found') + logger.debug('setup.py file found.') return True return False @@ -1277,9 +1278,9 @@ def _has_setup_py(srcdir): def _has_setup_cfg(srcdir): setup_cfg = os.path.join(srcdir, 'setup.cfg') if os.path.isfile(setup_cfg): - logger.info('setup.cfg file found') + logger.debug('setup.cfg file found.') return True - logger.warning("no setup.cfg file found") + logger.debug("No setup.cfg file found.") return False -- cgit v0.12 From 2da756592bb7b89e93c8cf2f7ab11a8eb27268fc Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 12:25:21 +0200 Subject: removed spurious output --- Lib/packaging/tests/test_uninstall.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/tests/test_uninstall.py b/Lib/packaging/tests/test_uninstall.py index 578b10d..d5a2521 100644 --- a/Lib/packaging/tests/test_uninstall.py +++ b/Lib/packaging/tests/test_uninstall.py @@ -1,6 +1,7 @@ """Tests for the uninstall command.""" import os import sys +from io import StringIO from packaging.database import disable_cache, enable_cache from packaging.run import main @@ -79,7 +80,12 @@ class UninstallTestCase(support.TempdirManager, if not dirname: dirname = self.make_dist(name, **kw) os.chdir(dirname) - dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) + old_out = sys.stdout + sys.stderr = StringIO() + try: + dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) + finally: + sys.sterr = old_out install_lib = self.get_path(dist, 'purelib') return dist, install_lib -- cgit v0.12 From b9c09878ab8390aa9b78ff468f42ed91bae2b1fe Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 12:25:38 +0200 Subject: displaying the faulty project name is a good idea --- Lib/packaging/metadata.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/packaging/metadata.py b/Lib/packaging/metadata.py index 596eec7..53d91f7 100644 --- a/Lib/packaging/metadata.py +++ b/Lib/packaging/metadata.py @@ -396,22 +396,24 @@ class Metadata: value = [] if logger.isEnabledFor(logging.WARNING): + project_name = self['Name'] + if name in _PREDICATE_FIELDS and value is not None: for v in value: # check that the values are valid predicates if not is_valid_predicate(v.split(';')[0]): logger.warning( - '%r is not a valid predicate (field %r)', - v, name) + '%r: %r is not a valid predicate (field %r)', + project_name, v, name) # FIXME this rejects UNKNOWN, is that right? elif name in _VERSIONS_FIELDS and value is not None: if not is_valid_versions(value): - logger.warning('%r is not a valid version (field %r)', - value, name) + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) elif name in _VERSION_FIELDS and value is not None: if not is_valid_version(value): - logger.warning('%r is not a valid version (field %r)', - value, name) + logger.warning('%r: %r is not a valid version (field %r)', + project_name, value, name) if name in _UNICODEFIELDS: if name == 'Description': -- cgit v0.12 From 1ae2998f3edaac9d0e98a62847b5d541f5f50ce0 Mon Sep 17 00:00:00 2001 From: Brian Quinlan Date: Mon, 30 May 2011 21:52:24 +1000 Subject: Increases some test timeouts to fix issue 11864. --- Lib/test/test_concurrent_futures.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index ec84a66..7457f39 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -260,14 +260,14 @@ class WaitTests(unittest.TestCase): def test_timeout(self): future1 = self.executor.submit(mul, 6, 7) - future2 = self.executor.submit(time.sleep, 3) + future2 = self.executor.submit(time.sleep, 6) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, future1, future2], - timeout=1.5, + timeout=5, return_when=futures.ALL_COMPLETED) self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, @@ -357,8 +357,8 @@ class ExecutorTest(unittest.TestCase): results = [] try: for i in self.executor.map(time.sleep, - [0, 0, 3], - timeout=1.5): + [0, 0, 6], + timeout=5): results.append(i) except futures.TimeoutError: pass -- cgit v0.12 From c8507bfe9cd581de1b324af92bf255a86a0437b0 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 30 May 2011 10:52:48 -0500 Subject: use better naming scheme --- Lib/test/test_grammar.py | 84 ++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 329b258..9953363 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -10,7 +10,7 @@ from sys import * class TokenTests(unittest.TestCase): - def testBackslash(self): + def test_backslash(self): # Backslash means line continuation: x = 1 \ + 1 @@ -20,7 +20,7 @@ class TokenTests(unittest.TestCase): x = 0 self.assertEqual(x, 0, 'backslash ending comment') - def testPlainIntegers(self): + def test_plain_integers(self): self.assertEqual(type(000), type(0)) self.assertEqual(0xff, 255) self.assertEqual(0o377, 255) @@ -56,7 +56,7 @@ class TokenTests(unittest.TestCase): else: self.fail('Weird maxsize value %r' % maxsize) - def testLongIntegers(self): + def test_long_integers(self): x = 0 x = 0xffffffffffffffff x = 0Xffffffffffffffff @@ -66,7 +66,7 @@ class TokenTests(unittest.TestCase): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testFloats(self): + def test_floats(self): x = 3.14 x = 314. x = 0.314 @@ -80,7 +80,7 @@ class TokenTests(unittest.TestCase): x = .3e14 x = 3.1e4 - def testStringLiterals(self): + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) x = '"'; y = "\""; self.assertTrue(len(x) == 1 and x == y and ord(x) == 34) @@ -120,7 +120,7 @@ the \'lazy\' dog.\n\ ' self.assertEqual(x, y) - def testEllipsis(self): + def test_ellipsis(self): x = ... self.assertTrue(x is Ellipsis) self.assertRaises(SyntaxError, eval, ".. .") @@ -136,11 +136,11 @@ class GrammarTests(unittest.TestCase): # expr_input: testlist NEWLINE # XXX Hard to test -- used only in calls to input() - def testEvalInput(self): + def test_eval_input(self): # testlist ENDMARKER x = eval('1, 0 or 1') - def testFuncdef(self): + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE ### decorators: decorator+ @@ -324,7 +324,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") - def testLambdef(self): + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 self.assertEqual(l1(), 0) @@ -346,7 +346,7 @@ class GrammarTests(unittest.TestCase): ### stmt: simple_stmt | compound_stmt # Tested below - def testSimpleStmt(self): + def test_simple_stmt(self): ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): @@ -357,7 +357,7 @@ class GrammarTests(unittest.TestCase): ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt # Tested below - def testExprStmt(self): + def test_expr_stmt(self): # (exprlist '=')* exprlist 1 1, 2, 3 @@ -370,7 +370,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "x + 1 = 1") check_syntax_error(self, "a + 1 = b + 2") - def testDelStmt(self): + def test_del_stmt(self): # 'del' exprlist abc = [1,2,3] x, y, z = abc @@ -379,18 +379,18 @@ class GrammarTests(unittest.TestCase): del abc del x, y, (z, xyz) - def testPassStmt(self): + def test_pass_stmt(self): # 'pass' pass # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt # Tested below - def testBreakStmt(self): + def test_break_stmt(self): # 'break' while 1: break - def testContinueStmt(self): + def test_continue_stmt(self): # 'continue' i = 1 while i: i = 0; continue @@ -442,7 +442,7 @@ class GrammarTests(unittest.TestCase): self.fail("continue then break in try/except in loop broken!") test_inner() - def testReturn(self): + def test_return(self): # 'return' [testlist] def g1(): return def g2(): return 1 @@ -450,17 +450,17 @@ class GrammarTests(unittest.TestCase): x = g2() check_syntax_error(self, "class foo:return 1") - def testYield(self): + def test_yield(self): check_syntax_error(self, "class foo:yield 1") - def testRaise(self): + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') except RuntimeError: pass try: raise KeyboardInterrupt except KeyboardInterrupt: pass - def testImport(self): + def test_import(self): # 'import' dotted_as_names import sys import time, sys @@ -473,13 +473,13 @@ class GrammarTests(unittest.TestCase): from sys import (path, argv) from sys import (path, argv,) - def testGlobal(self): + def test_global(self): # 'global' NAME (',' NAME)* global a global a, b global one, two, three, four, five, six, seven, eight, nine, ten - def testNonlocal(self): + def test_nonlocal(self): # 'nonlocal' NAME (',' NAME)* x = 0 y = 0 @@ -487,7 +487,7 @@ class GrammarTests(unittest.TestCase): nonlocal x nonlocal x, y - def testAssert(self): + def test_assert(self): # assertTruestmt: 'assert' test [',' test] assert 1 assert 1, 1 @@ -504,7 +504,7 @@ class GrammarTests(unittest.TestCase): ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef # Tested below - def testIf(self): + def test_if(self): # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] if 1: pass if 1: pass @@ -517,7 +517,7 @@ class GrammarTests(unittest.TestCase): elif 0: pass else: pass - def testWhile(self): + def test_while(self): # 'while' test ':' suite ['else' ':' suite] while 0: pass while 0: pass @@ -532,7 +532,7 @@ class GrammarTests(unittest.TestCase): x = 2 self.assertEqual(x, 2) - def testFor(self): + def test_for(self): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] for i in 1, 2, 3: pass for i, j, k in (): pass @@ -559,7 +559,7 @@ class GrammarTests(unittest.TestCase): result.append(x) self.assertEqual(result, [1, 2, 3]) - def testTry(self): + def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite ### except_clause: 'except' [expr ['as' expr]] @@ -582,7 +582,7 @@ class GrammarTests(unittest.TestCase): try: pass finally: pass - def testSuite(self): + def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT if 1: pass if 1: @@ -597,7 +597,7 @@ class GrammarTests(unittest.TestCase): pass # - def testTest(self): + def test_test(self): ### and_test ('or' and_test)* ### and_test: not_test ('and' not_test)* ### not_test: 'not' not_test | comparison @@ -608,7 +608,7 @@ class GrammarTests(unittest.TestCase): if not 1 and 1 and 1: pass if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - def testComparison(self): + def test_comparison(self): ### comparison: expr (comp_op expr)* ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' if 1: pass @@ -625,36 +625,36 @@ class GrammarTests(unittest.TestCase): if 1 not in (): pass if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass - def testBinaryMaskOps(self): + def test_binary_mask_ops(self): x = 1 & 1 x = 1 ^ 1 x = 1 | 1 - def testShiftOps(self): + def test_shift_ops(self): x = 1 << 1 x = 1 >> 1 x = 1 << 1 >> 1 - def testAdditiveOps(self): + def test_additive_ops(self): x = 1 x = 1 + 1 x = 1 - 1 - 1 x = 1 - 1 + 1 - 1 + 1 - def testMultiplicativeOps(self): + def test_multiplicative_ops(self): x = 1 * 1 x = 1 / 1 x = 1 % 1 x = 1 / 1 * 1 % 1 - def testUnaryOps(self): + def test_unary_ops(self): x = +1 x = -1 x = ~1 x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 x = -1*1/1 + 1*1 - ---1*1 - def testSelectors(self): + def test_selectors(self): ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME ### subscript: expr | [expr] ':' [expr] @@ -684,7 +684,7 @@ class GrammarTests(unittest.TestCase): L.sort(key=lambda x: x if isinstance(x, tuple) else ()) self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - def testAtoms(self): + def test_atoms(self): ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) @@ -719,7 +719,7 @@ class GrammarTests(unittest.TestCase): ### testlist: test (',' test)* [','] # These have been exercised enough above - def testClassdef(self): + def test_classdef(self): # 'class' NAME ['(' [testlist] ')'] ':' suite class B: pass class B2(): pass @@ -738,14 +738,14 @@ class GrammarTests(unittest.TestCase): @class_decorator class G: pass - def testDictcomps(self): + def test_dictcomps(self): # dictorsetmaker: ( (test ':' test (comp_for | # (',' test ':' test)* [','])) | # (test (comp_for | (',' test)* [','])) ) nums = [1, 2, 3] self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4}) - def testListcomps(self): + def test_listcomps(self): # list comprehension tests nums = [1, 2, 3, 4, 5] strs = ["Apple", "Banana", "Coconut"] @@ -808,7 +808,7 @@ class GrammarTests(unittest.TestCase): self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), ('Macdonalds', 'Cheeseburger')]) - def testGenexps(self): + def test_genexps(self): # generator expression tests g = ([x for x in range(10)] for x in range(1)) self.assertEqual(next(g), [x for x in range(10)]) @@ -843,7 +843,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "foo(x for x in range(10), 100)") check_syntax_error(self, "foo(100, x for x in range(10))") - def testComprehensionSpecials(self): + def test_comprehension_specials(self): # test for outmost iterable precomputation x = 10; g = (i for i in range(x)); x = 5 self.assertEqual(len(list(g)), 10) @@ -882,7 +882,7 @@ class GrammarTests(unittest.TestCase): with manager() as x, manager(): pass - def testIfElseExpr(self): + def test_if_else_expr(self): # Test ifelse expressions in various cases def _checkeval(msg, ret): "helper to check that evaluation of expressions is done correctly" -- cgit v0.12 From 758888d437c4c2d398ec322ff1596e1620f5f0dd Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 30 May 2011 11:12:38 -0500 Subject: don't restrict unexpected EOF errors to the first line (closes #12216) --- Lib/test/test_grammar.py | 7 +++++++ Misc/NEWS | 2 ++ Parser/parsetok.c | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 9953363..32dc15e 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -125,6 +125,13 @@ the \'lazy\' dog.\n\ self.assertTrue(x is Ellipsis) self.assertRaises(SyntaxError, eval, ".. .") + def test_eof_error(self): + samples = ("def foo(", "\ndef foo(", "def foo(\n") + for s in samples: + with self.assertRaises(SyntaxError) as cm: + compile(s, "", "exec") + self.assertIn("unexpected EOF", str(cm.exception)) + class GrammarTests(unittest.TestCase): # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE diff --git a/Misc/NEWS b/Misc/NEWS index fde085b..0b93945 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12216: Allow unexpected EOF errors to happen on any line of the file. + - Issue #12199: The TryExcept and TryFinally and AST nodes have been unified into a Try node. diff --git a/Parser/parsetok.c b/Parser/parsetok.c index eef650a..431a87c 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -232,7 +232,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, PyParser_Delete(ps); if (n == NULL) { - if (tok->lineno <= 1 && tok->done == E_EOF) + if (tok->done == E_EOF) err_ret->error = E_EOF; err_ret->lineno = tok->lineno; if (tok->buf != NULL) { -- cgit v0.12 From 371df0574034ad86b7dba9ba8e7d31a6fccde3db Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 18:55:32 +0200 Subject: Issue 12218: Removing wsgiref.egg-info --- Lib/wsgiref.egg-info | 8 -------- Misc/NEWS | 2 ++ 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 Lib/wsgiref.egg-info diff --git a/Lib/wsgiref.egg-info b/Lib/wsgiref.egg-info deleted file mode 100644 index c0b7893..0000000 --- a/Lib/wsgiref.egg-info +++ /dev/null @@ -1,8 +0,0 @@ -Metadata-Version: 1.0 -Name: wsgiref -Version: 0.1.2 -Summary: WSGI (PEP 333) Reference Library -Author: Phillip J. Eby -Author-email: web-sig@python.org -License: PSF or ZPL -Platform: UNKNOWN diff --git a/Misc/NEWS b/Misc/NEWS index 0b93945..785761c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -177,6 +177,8 @@ Core and Builtins Library ------- +- Issue #12218: Removed wsgiref.egg-info. + - Issue #12196: Add pipe2() to the os module. - Issue #985064: Make plistlib more resilient to faulty input plists. -- cgit v0.12 From 1cc9bbd7513c77599408bb36d893e2b1b1fdfce6 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 19:16:49 +0200 Subject: fixed the test_uninstall patch --- Lib/packaging/tests/test_uninstall.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/packaging/tests/test_uninstall.py b/Lib/packaging/tests/test_uninstall.py index d5a2521..45ce4ca 100644 --- a/Lib/packaging/tests/test_uninstall.py +++ b/Lib/packaging/tests/test_uninstall.py @@ -80,12 +80,12 @@ class UninstallTestCase(support.TempdirManager, if not dirname: dirname = self.make_dist(name, **kw) os.chdir(dirname) - old_out = sys.stdout + old_out = sys.stderr sys.stderr = StringIO() try: dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) finally: - sys.sterr = old_out + sys.stderr = old_out install_lib = self.get_path(dist, 'purelib') return dist, install_lib -- cgit v0.12 From ebbb3b797297506b91c2b9256bcd0fe92e8b47b8 Mon Sep 17 00:00:00 2001 From: Alexis Metaireau Date: Mon, 30 May 2011 21:57:20 +0200 Subject: Update the ACKS file with the packaging contributors --- Misc/ACKS | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Misc/ACKS b/Misc/ACKS index 350cc3c..0fc999d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -12,7 +12,9 @@ PS: In the standard Python distribution, this file is encoded in UTF-8 and the list is in rough alphabetical order by last names. David Abrahams +Rajiv Abraham Ron Adam +Ali Afshar Jim Ahlstrom Farhan Ahmad Matthew Ahrens @@ -58,6 +60,7 @@ Richard Barran Cesar Eduardo Barros Des Barry Ulf Bartelt +Pior Bastida Nick Bastin Jeff Bauer Mike Bayer @@ -122,6 +125,7 @@ Michael Broghton Daniel Brotsky Jean Brouwers Gary S. Brown +Titus Brown Oleg Broytmann Dave Brueck Stan Bubrouski @@ -135,6 +139,7 @@ Alastair Burt Tarn Weisner Burton Lee Busby Ralph Butler +Nicolas Cadou Jp Calderone Daniel Calvelo Tony Campbell @@ -153,6 +158,7 @@ Jeffrey Chang Mitch Chapman Greg Chapman Brad Chapman +Godefroid Chapelle David Chaum Nicolas Chauvat Jerry Chen @@ -176,6 +182,7 @@ Benjamin Collar Jeffery Collins Robert Collins Paul Colomiets +Christophe Combelles Denver Coneybeare Geremy Condra Juan José Conti @@ -206,6 +213,7 @@ Andrew Dalke Lars Damerow Evan Dandrea Eric Daniel +Pierre-Yves David Scott David Daniels Ben Darnell Jonathan Dasteel @@ -213,6 +221,7 @@ John DeGood Ned Deily Vincent Delft Arnaud Delobelle +Konrad Delong Erik Demaine Roger Dev Raghuram Devarakonda @@ -226,6 +235,7 @@ Daniel Diniz Humberto Diogenes Yves Dionne Daniel Dittmar +Josip Djolonga Jaromir Dolecek Ismail Donmez Marcos Donolo @@ -264,6 +274,7 @@ Jürgen A. Erhard Michael Ernst Ben Escoto Andy Eskilsson +André Espaze Stefan Esser Stephen D Evans Carey Evans @@ -277,8 +288,10 @@ Martijn Faassen Clovis Fabricio Andreas Faerber Bill Fancher +Andrew Francis Troy J. Farrell Mark Favas +Boris Feld Niels Ferguson Sebastian Fernandez Florian Festi @@ -328,6 +341,7 @@ Dinu Gherman Jonathan Giddy Johannes Gijsbers Michael Gilfix +Yannick Gingras Christoph Gohlke Tim Golden Chris Gonnerman @@ -351,6 +365,7 @@ Václav Haisman Bob Halley Jesse Hallio Jun Hamano +Alexandre Hamelin Mark Hammond Manus Hand Milton L. Hankins @@ -382,6 +397,7 @@ Bernhard Herzog Magnus L. Hetland Raymond Hettinger Kevan Heydon +Kelsey Hightower Jason Hildebrand Richie Hindle Konrad Hinsen @@ -409,6 +425,7 @@ Jan Hosang Ken Howard Brad Howes Chih-Hao Huang +Christian Hudon Lawrence Hudson Michael Hudson Jim Hugunin @@ -436,6 +453,7 @@ Bertrand Janin Geert Jansen Jack Jansen Bill Janssen +Julien Jehannet Drew Jenkins Flemming Kjær Jensen MunSic Jeong @@ -484,6 +502,7 @@ Reid Kleckner Bastian Kleineidam Bob Kline Matthias Klose +Jeremy Kloth Kim Knapp Lenny Kneler Pat Knight @@ -512,11 +531,13 @@ Tino Lange Andrew Langmead Detlef Lannert Soren Larsen +Amos Latteier Piers Lauder Ben Laurie Simon Law Chris Lawrence Brian Leair +Mathieu Leduc-Hamel James Lee John J. Lee Inyeol Lee @@ -532,6 +553,7 @@ Luke Kenneth Casson Leighton Marc-Andre Lemburg John Lenton Christopher Tur Lesniewski-Laas +Alain Leufroy Mark Levinson William Lewis Xuanji Li @@ -576,6 +598,7 @@ Owen Martin Sébastien Martini Roger Masse Nick Mathewson +Simon Mathieu Graham Matthews Dieter Maurer Arnaud Mazin @@ -595,7 +618,9 @@ Lucas Prado Melo Ezio Melotti Brian Merrell Luke Mewburn +Carl Meyer Mike Meyer +Alexis Métaireau Steven Miale Trent Mick Stan Mihai @@ -605,21 +630,26 @@ Chad Miller Jason V. Miller Jay T. Miller Roman Milner +Julien Miotte Andrii V. Mishkovskyi Dustin J. Mitchell Dom Mitchell +Zubin Mithra Doug Moen The Dragon De Monsyne Skip Montanaro Paul Moore Derek Morr James A Morrison +Derek McTavish Mounce Pablo Mouzo Mher Movsisyan Sjoerd Mullender Sape Mullender Michael Muller Neil Muller +Michael Mulich +Louis Munro R. David Murray Piotr Meyer John Nagle @@ -672,11 +702,14 @@ Peter Parente Alexandre Parenteau Dan Parisien Harri Pasanen +Gaël Pasgrimaud Randy Pausch Samuele Pedroni Marcel van der Peijl Steven Pemberton Santiago Peresón +George Peristerakis +Mathieu Perreault Mark Perrego Trevor Perrin Gabriel de Perthuis @@ -685,6 +718,7 @@ Benjamin Peterson Joe Peterson Chris Petrilli Bjorn Pettersen +Ronny Pfannschmidt Geoff Philbrick Gavrie Philipson Adrian Phillips @@ -731,6 +765,7 @@ Michael P. Reilly Bernhard Reiter Steven Reiz Roeland Rengelink +Antoine Reversat Tim Rice Francesco Ricciardi Jan Pieter Riegel @@ -745,11 +780,14 @@ Andy Robinson Mark Roddy Kevin Rodgers Giampaolo Rodola +Luis Rojas Mike Romberg Armin Ronacher Case Roole Timothy Roscoe +Erik Rose Jim Roskind +Brian Rosner Just van Rossum Hugo van Rossum Saskia van Rossum -- cgit v0.12 From e15dce3d18d333b91a567fb8d611ca74781e4f13 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 22:56:00 +0200 Subject: Close #12171: IncrementalEncoder.reset() of CJK codecs (multibytecodec) calls encreset() instead of decreset(). --- Doc/library/codecs.rst | 3 ++- Lib/test/test_multibytecodec.py | 9 +++++++-- Misc/NEWS | 5 ++++- Modules/cjkcodecs/multibytecodec.c | 14 ++++++++++---- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 4d5058e..90bd0dd 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -458,7 +458,8 @@ define in order to be compatible with the Python codec registry. .. method:: reset() - Reset the encoder to the initial state. + Reset the encoder to the initial state. The output is discarded: call + ``.encode('', final=True)`` to reset the encoder and to get the output. .. method:: IncrementalEncoder.getstate() diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 86c68dc..4448072 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -260,7 +260,8 @@ class TestStateful(unittest.TestCase): text = '\u4E16\u4E16' encoding = 'iso-2022-jp' expected = b'\x1b$B@$@$' - expected_reset = b'\x1b$B@$@$\x1b(B' + reset = b'\x1b(B' + expected_reset = expected + reset def test_encode(self): self.assertEqual(self.text.encode(self.encoding), self.expected_reset) @@ -271,6 +272,8 @@ class TestStateful(unittest.TestCase): encoder.encode(char) for char in self.text) self.assertEqual(output, self.expected) + self.assertEqual(encoder.encode('', final=True), self.reset) + self.assertEqual(encoder.encode('', final=True), b'') def test_incrementalencoder_final(self): encoder = codecs.getincrementalencoder(self.encoding)() @@ -279,12 +282,14 @@ class TestStateful(unittest.TestCase): encoder.encode(char, index == last_index) for index, char in enumerate(self.text)) self.assertEqual(output, self.expected_reset) + self.assertEqual(encoder.encode('', final=True), b'') class TestHZStateful(TestStateful): text = '\u804a\u804a' encoding = 'hz' expected = b'~{ADAD' - expected_reset = b'~{ADAD~}' + reset = b'~}' + expected_reset = expected + reset def test_main(): support.run_unittest(__name__) diff --git a/Misc/NEWS b/Misc/NEWS index 785761c..e5e5aa9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -177,9 +177,12 @@ Core and Builtins Library ------- +- Issue #12171: IncrementalEncoder.reset() of CJK codecs (multibytecodec) calls + encreset() instead of decreset(). + - Issue #12218: Removed wsgiref.egg-info. -- Issue #12196: Add pipe2() to the os module. +- Issue #12196: Add pipe2() to the os module. - Issue #985064: Make plistlib more resilient to faulty input plists. Patch by Mher Movsisyan. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index bb8176f..1b37845 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -901,11 +901,17 @@ mbiencoder_encode(MultibyteIncrementalEncoderObject *self, static PyObject * mbiencoder_reset(MultibyteIncrementalEncoderObject *self) { - if (self->codec->decreset != NULL && - self->codec->decreset(&self->state, self->codec->config) != 0) - return NULL; + /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */ + unsigned char buffer[4], *outbuf; + Py_ssize_t r; + if (self->codec->encreset != NULL) { + outbuf = buffer; + r = self->codec->encreset(&self->state, self->codec->config, + &outbuf, sizeof(buffer)); + if (r != 0) + return NULL; + } self->pendingsize = 0; - Py_RETURN_NONE; } -- cgit v0.12 From d976098e3bb53b9c52d55a303e1389a102ed8bae Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 22:58:13 +0200 Subject: Close #12089: Remove outdated and unused code from regrtest. --- Lib/test/regrtest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b5e5127..29e8648 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1023,10 +1023,6 @@ class saved_test_environment: def runtest_inner(test, verbose, quiet, huntrleaks=False, debug=False): support.unload(test) - if verbose: - capture_stdout = None - else: - capture_stdout = io.StringIO() test_time = 0.0 refleak = False # True if the test leaked references. -- cgit v0.12 From 2a12974bca2433eea0131d904a423ed8234dcf7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 23:02:52 +0200 Subject: Close #12028: Make threading._get_ident() public, rename it to threading.get_ident() and document it. This function was used by _thread.get_ident(). --- Doc/library/signal.rst | 7 +++---- Doc/library/threading.rst | 19 +++++++++++++++---- Lib/logging/__init__.py | 11 +++++------ Lib/reprlib.py | 2 +- Lib/test/lock_tests.py | 4 ++-- Lib/test/test_capi.py | 5 ++--- Lib/test/test_signal.py | 4 ++-- Lib/test/test_sys.py | 6 +++--- Lib/test/test_threaded_import.py | 2 +- Lib/test/test_threading.py | 4 ++-- Lib/threading.py | 21 ++++++++++----------- Misc/NEWS | 4 ++++ 12 files changed, 50 insertions(+), 39 deletions(-) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 68a5d2c..f2a37cc 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -187,10 +187,9 @@ The :mod:`signal` module defines the following functions: Send the signal *signum* to the thread *thread_id*, another thread in the same process as the caller. The signal is asynchronously directed to thread. - *thread_id* can be read from the :attr:`~threading.Thread.ident` attribute - of :attr:`threading.Thread`. For example, - ``threading.current_thread().ident`` gives the identifier of the current - thread. + Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` + attribute of :attr:`threading.Thread` to get a 'thread identifier' for + *thread_id*. If *signum* is 0, then no signal is sent, but error checking is still performed; this can be used to check if a thread is still running. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index df47045..504a2fb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -48,6 +48,17 @@ This module defines the following functions and objects: returned. +.. function:: get_ident() + + Return the 'thread identifier' of the current thread. This is a nonzero + integer. Its value has no direct meaning; it is intended as a magic cookie + to be used e.g. to index a dictionary of thread-specific data. Thread + identifiers may be recycled when a thread exits and another thread is + created. + + .. versionadded:: 3.3 + + .. function:: enumerate() Return a list of all :class:`Thread` objects currently alive. The list @@ -332,10 +343,10 @@ impossible to detect the termination of alien threads. .. attribute:: ident The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the - :func:`thread.get_ident()` function. Thread identifiers may be recycled - when a thread exits and another thread is created. The identifier is - available even after the thread has exited. + been started. This is a nonzero integer. See the :func:`get_ident()` + function. Thread identifiers may be recycled when a thread exits and + another thread is created. The identifier is available even after the + thread has exited. .. method:: is_alive() diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 3a8a639..1a4b241 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -41,10 +41,9 @@ except ImportError: #pragma: no cover codecs = None try: - import _thread as thread import threading except ImportError: #pragma: no cover - thread = None + threading = None __author__ = "Vinay Sajip " __status__ = "production" @@ -199,7 +198,7 @@ def _checkLevel(level): #the lock would already have been acquired - so we need an RLock. #The same argument applies to Loggers and Manager.loggerDict. # -if thread: +if threading: _lock = threading.RLock() else: #pragma: no cover _lock = None @@ -278,8 +277,8 @@ class LogRecord(object): self.created = ct self.msecs = (ct - int(ct)) * 1000 self.relativeCreated = (self.created - _startTime) * 1000 - if logThreads and thread: - self.thread = thread.get_ident() + if logThreads and threading: + self.thread = threading.get_ident() self.threadName = threading.current_thread().name else: # pragma: no cover self.thread = None @@ -773,7 +772,7 @@ class Handler(Filterer): """ Acquire a thread lock for serializing access to the underlying I/O. """ - if thread: + if threading: self.lock = threading.RLock() else: #pragma: no cover self.lock = None diff --git a/Lib/reprlib.py b/Lib/reprlib.py index f803360..092874a 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -5,7 +5,7 @@ __all__ = ["Repr", "repr", "recursive_repr"] import builtins from itertools import islice try: - from _thread import get_ident + from threading import get_ident except ImportError: from _dummy_thread import get_ident diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 126f97c..7bcc436 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -4,7 +4,7 @@ Various tests for synchronization primitives. import sys import time -from _thread import start_new_thread, get_ident, TIMEOUT_MAX +from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest @@ -31,7 +31,7 @@ class Bunch(object): self.finished = [] self._can_exit = not wait_before_exit def task(): - tid = get_ident() + tid = threading.get_ident() self.started.append(tid) try: f() diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 327ac66..7236474 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -190,18 +190,17 @@ def test_main(): idents = [] def callback(): - idents.append(_thread.get_ident()) + idents.append(threading.get_ident()) _testcapi._test_thread_state(callback) a = b = callback time.sleep(1) # Check our main thread is in the list exactly 3 times. - if idents.count(_thread.get_ident()) != 3: + if idents.count(threading.get_ident()) != 3: raise support.TestFailed( "Couldn't find main thread correctly in the list") if threading: - import _thread import time TestThreadState() t = threading.Thread(target=TestThreadState) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index cdd3f3e..4bf2d2d 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -557,7 +557,7 @@ class PendingSignalsTests(unittest.TestCase): def kill(self, signum): if self.has_pthread_kill: - tid = threading.current_thread().ident + tid = threading.get_ident() signal.pthread_kill(tid, signum) else: pid = os.getpid() @@ -589,7 +589,7 @@ class PendingSignalsTests(unittest.TestCase): 'need signal.pthread_kill()') def test_pthread_kill(self): signum = signal.SIGUSR1 - current = threading.current_thread().ident + current = threading.get_ident() old_handler = signal.signal(signum, self.handler) self.addCleanup(signal.signal, signum, old_handler) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 04bdfef..bc0f34c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -343,7 +343,7 @@ class SysModuleTest(unittest.TestCase): # Test sys._current_frames() in a WITH_THREADS build. @test.support.reap_threads def current_frames_with_threads(self): - import threading, _thread + import threading import traceback # Spawn a thread that blocks at a known place. Then the main @@ -357,7 +357,7 @@ class SysModuleTest(unittest.TestCase): g456() def g456(): - thread_info.append(_thread.get_ident()) + thread_info.append(threading.get_ident()) entered_g.set() leave_g.wait() @@ -373,7 +373,7 @@ class SysModuleTest(unittest.TestCase): d = sys._current_frames() - main_id = _thread.get_ident() + main_id = threading.get_ident() self.assertIn(main_id, d) self.assertIn(thread_id, d) diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py index 7791935..6919d21 100644 --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -30,7 +30,7 @@ def task(N, done, done_tasks, errors): except Exception as e: errors.append(e.with_traceback(None)) finally: - done_tasks.append(thread.get_ident()) + done_tasks.append(threading.get_ident()) finished = len(done_tasks) == N if finished: done.set() diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index c22d965..12e596e 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -173,7 +173,7 @@ class ThreadTests(BaseTestCase): exception = ctypes.py_object(AsyncExc) # First check it works when setting the exception from the same thread. - tid = _thread.get_ident() + tid = threading.get_ident() try: result = set_async_exc(ctypes.c_long(tid), exception) @@ -202,7 +202,7 @@ class ThreadTests(BaseTestCase): class Worker(threading.Thread): def run(self): - self.id = _thread.get_ident() + self.id = threading.get_ident() self.finished = False try: diff --git a/Lib/threading.py b/Lib/threading.py index fafe779..1f638b4 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -24,7 +24,7 @@ __all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread _allocate_lock = _thread.allocate_lock -_get_ident = _thread.get_ident +get_ident = _thread.get_ident ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -52,7 +52,7 @@ if __debug__: format = format % args # Issue #4188: calling current_thread() can incur an infinite # recursion if it has to create a DummyThread on the fly. - ident = _get_ident() + ident = get_ident() try: name = _active[ident].name except KeyError: @@ -110,7 +110,7 @@ class _RLock(_Verbose): self.__class__.__name__, owner, self._count) def acquire(self, blocking=True, timeout=-1): - me = _get_ident() + me = get_ident() if self._owner == me: self._count = self._count + 1 if __debug__: @@ -130,7 +130,7 @@ class _RLock(_Verbose): __enter__ = acquire def release(self): - if self._owner != _get_ident(): + if self._owner != get_ident(): raise RuntimeError("cannot release un-acquired lock") self._count = count = self._count - 1 if not count: @@ -166,7 +166,7 @@ class _RLock(_Verbose): return (count, owner) def _is_owned(self): - return self._owner == _get_ident() + return self._owner == get_ident() _PyRLock = _RLock @@ -714,7 +714,7 @@ class Thread(_Verbose): raise def _set_ident(self): - self._ident = _get_ident() + self._ident = get_ident() def _bootstrap_inner(self): try: @@ -787,7 +787,7 @@ class Thread(_Verbose): try: # We don't call self._delete() because it also # grabs _active_limbo_lock. - del _active[_get_ident()] + del _active[get_ident()] except: pass @@ -823,7 +823,7 @@ class Thread(_Verbose): try: with _active_limbo_lock: - del _active[_get_ident()] + del _active[get_ident()] # There must not be any python code between the previous line # and after the lock is released. Otherwise a tracing function # could try to acquire the lock again in the same thread, (in @@ -1006,9 +1006,8 @@ class _DummyThread(Thread): def current_thread(): try: - return _active[_get_ident()] + return _active[get_ident()] except KeyError: - ##print "current_thread(): no current thread for", _get_ident() return _DummyThread() currentThread = current_thread @@ -1062,7 +1061,7 @@ def _after_fork(): if thread is current: # There is only one active thread. We reset the ident to # its new value since it can have changed. - ident = _get_ident() + ident = get_ident() thread._ident = ident # Any condition variables hanging off of the active thread may # be in an invalid state, so we reinitialize them. diff --git a/Misc/NEWS b/Misc/NEWS index e5e5aa9..152acba 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -177,6 +177,10 @@ Core and Builtins Library ------- +- Issue #12028: Make threading._get_ident() public, rename it to + threading.get_ident() and document it. This function was used by + _thread.get_ident(). + - Issue #12171: IncrementalEncoder.reset() of CJK codecs (multibytecodec) calls encreset() instead of decreset(). -- cgit v0.12 From f2a94216de1750eedaea3f5983996a388a9866e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 23:21:30 +0200 Subject: Close #10616: mention bytes and bytearray in PyObject_AsCharBuffer() error message --- Objects/abstract.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 11e145f..f992573 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -237,7 +237,8 @@ PyObject_AsCharBuffer(PyObject *obj, pb = obj->ob_type->tp_as_buffer; if (pb == NULL || pb->bf_getbuffer == NULL) { PyErr_SetString(PyExc_TypeError, - "expected an object with the buffer interface"); + "expected bytes, bytearray " + "or buffer compatible object"); return -1; } if ((*pb->bf_getbuffer)(obj, &view, PyBUF_SIMPLE)) return -1; -- cgit v0.12 From 58ab76696536b63f588f10eeb165deae6c0e4624 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 23:23:50 +0200 Subject: Issue #12028: fix NEWS entry --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 152acba..6a32661 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -178,7 +178,7 @@ Library ------- - Issue #12028: Make threading._get_ident() public, rename it to - threading.get_ident() and document it. This function was used by + threading.get_ident() and document it. This function was already used using _thread.get_ident(). - Issue #12171: IncrementalEncoder.reset() of CJK codecs (multibytecodec) calls -- cgit v0.12 From f47fa58b62e6702f840625bff9fc8a07cef3408f Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Mon, 30 May 2011 23:26:51 +0200 Subject: better UI when the user does not have the perms to remove the project --- Lib/packaging/install.py | 27 +++++++++++++++++++++++++-- Lib/packaging/tests/test_uninstall.py | 28 +++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/Lib/packaging/install.py b/Lib/packaging/install.py index cd2bbb6..9f94532 100644 --- a/Lib/packaging/install.py +++ b/Lib/packaging/install.py @@ -376,7 +376,10 @@ def _remove_dist(dist, paths=sys.path): def remove(project_name, paths=sys.path, auto_confirm=True): - """Removes a single project from the installation""" + """Removes a single project from the installation. + + Returns True on success + """ dist = get_distribution(project_name, use_egg_info=True, paths=paths) if dist is None: raise PackagingError('Distribution "%s" not found' % project_name) @@ -384,13 +387,26 @@ def remove(project_name, paths=sys.path, auto_confirm=True): rmdirs = [] rmfiles = [] tmp = tempfile.mkdtemp(prefix=project_name + '-uninstall') + + def _move_file(source, target): + try: + os.rename(source, target) + except OSError as err: + return err + return None + + success = True + error = None try: for file_, md5, size in files: if os.path.isfile(file_): dirname, filename = os.path.split(file_) tmpfile = os.path.join(tmp, filename) try: - os.rename(file_, tmpfile) + error = _move_file(file_, tmpfile) + if error is not None: + success = False + break finally: if not os.path.isfile(file_): os.rename(tmpfile, file_) @@ -401,6 +417,11 @@ def remove(project_name, paths=sys.path, auto_confirm=True): finally: shutil.rmtree(tmp) + if not success: + logger.info('%r cannot be removed.', project_name) + logger.info('Error: %s' % str(error)) + return False + logger.info('Removing %r: ', project_name) for file_ in rmfiles: @@ -447,6 +468,8 @@ def remove(project_name, paths=sys.path, auto_confirm=True): logger.info('Success: removed %d files and %d dirs', file_count, dir_count) + return True + def install(project): logger.info('Getting information about %r...', project) diff --git a/Lib/packaging/tests/test_uninstall.py b/Lib/packaging/tests/test_uninstall.py index 45ce4ca..4b37286 100644 --- a/Lib/packaging/tests/test_uninstall.py +++ b/Lib/packaging/tests/test_uninstall.py @@ -2,6 +2,7 @@ import os import sys from io import StringIO +import stat from packaging.database import disable_cache, enable_cache from packaging.run import main @@ -82,10 +83,7 @@ class UninstallTestCase(support.TempdirManager, os.chdir(dirname) old_out = sys.stderr sys.stderr = StringIO() - try: - dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) - finally: - sys.stderr = old_out + dist = self.run_setup('install_dist', '--prefix=' + self.root_dir) install_lib = self.get_path(dist, 'purelib') return dist, install_lib @@ -99,10 +97,30 @@ class UninstallTestCase(support.TempdirManager, self.assertIsFile(install_lib, 'foo', '__init__.py') self.assertIsFile(install_lib, 'foo', 'sub', '__init__.py') self.assertIsFile(install_lib, 'Foo-0.1.dist-info', 'RECORD') - remove('Foo', paths=[install_lib]) + self.assertTrue(remove('Foo', paths=[install_lib])) self.assertIsNotFile(install_lib, 'foo', 'sub', '__init__.py') self.assertIsNotFile(install_lib, 'Foo-0.1.dist-info', 'RECORD') + @unittest.skipIf(sys.platform == 'win32', 'deactivated for now') + def test_remove_issue(self): + # makes sure if there are OSErrors (like permission denied) + # remove() stops and display a clean error + dist, install_lib = self.install_dist('Meh') + + # breaking os.rename + old = os.rename + + def _rename(source, target): + raise OSError() + + os.rename = _rename + try: + self.assertFalse(remove('Meh', paths=[install_lib])) + finally: + os.rename = old + + self.assertTrue(remove('Meh', paths=[install_lib])) + def test_suite(): return unittest.makeSuite(UninstallTestCase) -- cgit v0.12 From 4f71101eeda6fb0b55805ebcb12d11124ab3b567 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 23:46:00 +0200 Subject: Issue #12016: my_fgets() now always clears errors before calling fgets(). Fix the following case: sys.stdin.read() stopped with CTRL+d (end of file), raw_input() interrupted by CTRL+c. --- Misc/NEWS | 4 ++++ Parser/myreadline.c | 1 + 2 files changed, 5 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index e9bfef9..17ba290 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ What's New in Python 3.2.1 release candidate 2? Core and Builtins ----------------- +- Issue #12016: my_fgets() now always clears errors before calling fgets(). Fix + the following case: sys.stdin.read() stopped with CTRL+d (end of file), + raw_input() interrupted by CTRL+c. + - Issue #9670: Increase the default stack size for secondary threads on Mac OS X and FreeBSD to reduce the chances of a crash instead of a "maximum recursion depth" RuntimeError exception. diff --git a/Parser/myreadline.c b/Parser/myreadline.c index b12d052..fb4b805 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -40,6 +40,7 @@ my_fgets(char *buf, int len, FILE *fp) if (PyOS_InputHook != NULL) (void)(PyOS_InputHook)(); errno = 0; + clearerr(fp); p = fgets(buf, len, fp); if (p != NULL) return 0; /* No error */ -- cgit v0.12 From 19c5233303f2a5b2da3d7f7da6887daf46c13443 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 30 May 2011 23:50:04 +0200 Subject: Issue #1195: fix the issue number of the NEWS entry --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 17ba290..5b3cc5c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,7 +10,7 @@ What's New in Python 3.2.1 release candidate 2? Core and Builtins ----------------- -- Issue #12016: my_fgets() now always clears errors before calling fgets(). Fix +- Issue #1195: my_fgets() now always clears errors before calling fgets(). Fix the following case: sys.stdin.read() stopped with CTRL+d (end of file), raw_input() interrupted by CTRL+c. -- cgit v0.12 From ee49797c8dacb886d58804e59f6431ea6f842be2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 31 May 2011 00:01:24 +0200 Subject: Issue #12057: Add tests for ISO 2022 codecs iso2022_jp, iso2022_jp_2 and iso2022_kr --- Lib/test/cjkencodings/iso2022_jp-utf8.txt | 7 +++++ Lib/test/cjkencodings/iso2022_jp.txt | 7 +++++ Lib/test/cjkencodings/iso2022_kr-utf8.txt | 7 +++++ Lib/test/cjkencodings/iso2022_kr.txt | 7 +++++ Lib/test/test_codecencodings_iso2022.py | 46 +++++++++++++++++++++++++++++++ Lib/test/test_multibytecodec_support.py | 4 ++- Misc/NEWS | 6 ++++ 7 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Lib/test/cjkencodings/iso2022_jp-utf8.txt create mode 100644 Lib/test/cjkencodings/iso2022_jp.txt create mode 100644 Lib/test/cjkencodings/iso2022_kr-utf8.txt create mode 100644 Lib/test/cjkencodings/iso2022_kr.txt create mode 100644 Lib/test/test_codecencodings_iso2022.py diff --git a/Lib/test/cjkencodings/iso2022_jp-utf8.txt b/Lib/test/cjkencodings/iso2022_jp-utf8.txt new file mode 100644 index 0000000..7763250 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp-utf8.txt @@ -0,0 +1,7 @@ +Python の開発は、1990 年ごろから開始されています。 +開発者の Guido van Rossum は教育用のプログラミング言語「ABC」の開発に参加していましたが、ABC は実用上の目的にはあまり適していませんでした。 +このため、Guido はより実用的なプログラミング言語の開発を開始し、英国 BBS 放送のコメディ番組「モンティ パイソン」のファンである Guido はこの言語を「Python」と名づけました。 +このような背景から生まれた Python の言語設計は、「シンプル」で「習得が容易」という目標に重点が置かれています。 +多くのスクリプト系言語ではユーザの目先の利便性を優先して色々な機能を言語要素として取り入れる場合が多いのですが、Python ではそういった小細工が追加されることはあまりありません。 +言語自体の機能は最小限に押さえ、必要な機能は拡張モジュールとして追加する、というのが Python のポリシーです。 + diff --git a/Lib/test/cjkencodings/iso2022_jp.txt b/Lib/test/cjkencodings/iso2022_jp.txt new file mode 100644 index 0000000..fc398d6 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_jp.txt @@ -0,0 +1,7 @@ +Python $B$N3+H/$O!"(B1990 $BG/$4$m$+$i3+;O$5$l$F$$$^$9!#(B +$B3+H/e$NL\E*$K$O$"$^$jE,$7$F$$$^$;$s$G$7$?!#(B +$B$3$N$?$a!"(BGuido $B$O$h$j$E$1$^$7$?!#(B +$B$3$N$h$&$JGX7J$+$i@8$^$l$?(B Python $B$N8@8l@_7W$O!"!V%7%s%W%k!W$G!V=,F@$,MF0W!W$H$$$&L\I8$K=EE@$,CV$+$l$F$$$^$9!#(B +$BB?$/$N%9%/%j%W%H7O8@8l$G$O%f!<%6$NL\@h$NMxJX@-$rM%@h$7$F?'!9$J5!G=$r8@8lMWAG$H$7$Fl9g$,B?$$$N$G$9$,!"(BPython $B$G$O$=$&$$$C$?>.:Y9)$,DI2C$5$l$k$3$H$O$"$^$j$"$j$^$;$s!#(B +$B8@8l<+BN$N5!G=$O:G>.8B$K2!$5$(!"I,MW$J5!G=$O3HD%%b%8%e!<%k$H$7$FDI2C$9$k!"$H$$$&$N$,(B Python $B$N%]%j%7!<$G$9!#(B + diff --git a/Lib/test/cjkencodings/iso2022_kr-utf8.txt b/Lib/test/cjkencodings/iso2022_kr-utf8.txt new file mode 100644 index 0000000..d5c9d6e --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr-utf8.txt @@ -0,0 +1,7 @@ +◎ 파이썬(Python)은 배우기 쉽고, 강력한 프로그래밍 언어입니다. 파이썬은 +효율적인 고수준 데이터 구조와 간단하지만 효율적인 객체지향프로그래밍을 +지원합니다. 파이썬의 우아(優雅)한 문법과 동적 타이핑, 그리고 인터프리팅 +환경은 파이썬을 스크립팅과 여러 분야에서와 대부분의 플랫폼에서의 빠른 +애플리케이션 개발을 할 수 있는 이상적인 언어로 만들어줍니다. + +☆첫가끝: 날아라 쓩~ 큼! 금없이 전니다. 그런거 다. diff --git a/Lib/test/cjkencodings/iso2022_kr.txt b/Lib/test/cjkencodings/iso2022_kr.txt new file mode 100644 index 0000000..2cece21 --- /dev/null +++ b/Lib/test/cjkencodings/iso2022_kr.txt @@ -0,0 +1,7 @@ +$)C!] FD@L=c(Python)@: 9h?l1b =10m, 0-7BGQ GA7N1W7!9V >p>n@T4O4Y. FD@L=c@: +H?@2@{@N 0mF(iPd:)GQ 9.9}0z 5?@{ E8@LGN, 1W8.0m @NEMGA8.FC +H/0f@: FD@L=c@; =:E)83FC0z ?)7/ :P>_?!<-?M 4k:N:P@G GC7'F{?!<-@G :|8% +>VGC8.DI@Lp>n7N 885i>nA]4O4Y. + +!YC90!3!: 3/>F6s >1~ E-! 1]>x@L @|4O4Y. 1W710E 4Y. diff --git a/Lib/test/test_codecencodings_iso2022.py b/Lib/test/test_codecencodings_iso2022.py new file mode 100644 index 0000000..8c6e8a5 --- /dev/null +++ b/Lib/test/test_codecencodings_iso2022.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# +# Codec encoding tests for ISO 2022 encodings. + +from test import support +from test import test_multibytecodec_support +import unittest + +COMMON_CODEC_TESTS = ( + # invalid bytes + (b'ab\xFFcd', 'replace', 'ab\uFFFDcd'), + (b'ab\x1Bdef', 'replace', 'ab\x1Bdef'), + (b'ab\x1B$def', 'replace', 'ab\uFFFD'), + ) + +class Test_ISO2022_JP(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp' + tstring = test_multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + +class Test_ISO2022_JP2(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_jp_2' + tstring = test_multibytecodec_support.load_teststring('iso2022_jp') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'abdef'), + ) + +class Test_ISO2022_KR(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'iso2022_kr' + tstring = test_multibytecodec_support.load_teststring('iso2022_kr') + codectests = COMMON_CODEC_TESTS + ( + (b'ab\x1BNdef', 'replace', 'ab\x1BNdef'), + ) + + # iso2022_kr.txt cannot be used to test "chunk coding": the escape + # sequence is only written on the first line + def test_chunkcoding(self): + pass + +def test_main(): + support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py index a48fbe4..7735976 100644 --- a/Lib/test/test_multibytecodec_support.py +++ b/Lib/test/test_multibytecodec_support.py @@ -60,7 +60,9 @@ class TestBase: self.assertTrue(type(result) is str, type(result)) else: self.assertTrue(type(result) is bytes, type(result)) - self.assertEqual(result, expected) + self.assertEqual(result, expected, + '%a.decode(%r)=%a != %a' + % (source, self.encoding, result, expected)) else: self.assertRaises(UnicodeError, func, source, scheme) diff --git a/Misc/NEWS b/Misc/NEWS index 5b3cc5c..4ab11a5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -55,6 +55,12 @@ Build ensure "make install" creates symlinks in --prefix bin for the "-32" files in the framework bin directory like the installer does. +Tests +----- + +- Issue #12057: Add tests for ISO 2022 codecs (iso2022_jp, iso2022_jp_2, + iso2022_kr). + What's New in Python 3.2.1 release candidate 1? =============================================== -- cgit v0.12 From 15ac3110883c7f6e5cde5dd308baf178e666b985 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 30 May 2011 19:22:53 -0500 Subject: add tests for negative with conjugate --- Lib/test/test_abstract_numbers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2a396cd..253e6f0 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -14,6 +14,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) self.assertEqual(7, int(7).conjugate()) + self.assertEqual(-7, int(-7).conjugate()) self.assertEqual(7, int(7).numerator) self.assertEqual(1, int(7).denominator) @@ -24,6 +25,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) self.assertEqual(7.3, float(7.3).conjugate()) + self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): self.assertFalse(issubclass(complex, Real)) -- cgit v0.12 From 90fb9283fc342339ef6687d4927d3b74b2b73ecd Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Tue, 31 May 2011 08:31:50 +0200 Subject: promoted pysetup3 to a top-level script --- Makefile.pre.in | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index dff1252..f6ecacf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -888,6 +888,8 @@ bininstall: altbininstall (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) -rm -f $(DESTDIR)$(BINDIR)/2to3 (cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3) + -rm -f $(DESTDIR)$(BINDIR)/pysetup3 + (cd $(DESTDIR)$(BINDIR); $(LN) -s pysetup$(VERSION) pysetup3) # Install the manual page maninstall: diff --git a/setup.py b/setup.py index 9d46425..fa42191 100644 --- a/setup.py +++ b/setup.py @@ -1889,7 +1889,7 @@ def main(): # check the PyBuildScripts command above, and change the links # created by the bininstall target in Makefile.pre.in scripts = ["Tools/scripts/pydoc3", "Tools/scripts/idle3", - "Tools/scripts/2to3"] + "Tools/scripts/2to3", "Tools/scripts/pysetup3"] ) # --install-platlib -- cgit v0.12 From 441531f6e66e159b58a02663857a13c38719eb0b Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Tue, 31 May 2011 09:18:24 +0200 Subject: improved pysetup list UI --- Lib/packaging/run.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py index 03b80c6..6e4816d 100644 --- a/Lib/packaging/run.py +++ b/Lib/packaging/run.py @@ -337,13 +337,21 @@ def _run(dispatcher, args, **kw): def _list(dispatcher, args, **kw): opts = _parse_args(args[1:], '', ['all']) dists = get_distributions(use_egg_info=True) - if 'all' in opts: + if 'all' in opts or opts['args'] == []: results = dists else: results = [d for d in dists if d.name.lower() in opts['args']] + number = 0 for dist in results: print('%s %s at %s' % (dist.name, dist.metadata['version'], dist.path)) + number +=1 + + print('') + if number == 0: + print('Nothing seems to be installed.') + else: + print('Found %d projects installed.' % number) @action_help(search_usage) -- cgit v0.12 From 62ecb6aa0ae8dbfabe1f42cdda1eb6c1a184c3e7 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 31 May 2011 19:40:11 +1000 Subject: Tidy up the additional string module tests added at the Pycon sprints (closes #11505) --- Lib/test/test_pep292.py | 33 ++++++++++++++---------- Lib/test/test_string.py | 68 +++++++++++++++++++++++++++---------------------- Misc/ACKS | 1 + Misc/NEWS | 4 +-- 4 files changed, 61 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py index a1e52e9..6da8d2e 100644 --- a/Lib/test/test_pep292.py +++ b/Lib/test/test_pep292.py @@ -42,19 +42,6 @@ class TestTemplate(unittest.TestCase): s = Template('$who likes $$') eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $') - def test_invalid(self): - class MyPattern(Template): - pattern = r""" - (?: - (?P) | - (?P%(delim)s) | - @(?P%(id)s) | - @{(?P%(id)s)} - ) - """ - s = MyPattern('$') - self.assertRaises(ValueError, s.substitute, dict()) - def test_percents(self): eq = self.assertEqual s = Template('%(foo)s $foo ${foo}') @@ -172,6 +159,26 @@ class TestTemplate(unittest.TestCase): val = t.safe_substitute({'location': 'Cleveland'}) self.assertEqual(val, 'PyCon in Cleveland') + def test_invalid_with_no_lines(self): + # The error formatting for invalid templates + # has a special case for no data that the default + # pattern can't trigger (always has at least '$') + # So we craft a pattern that is always invalid + # with no leading data. + class MyTemplate(Template): + pattern = r""" + (?P) | + unreachable( + (?P) | + (?P) | + (?P) + ) + """ + s = MyTemplate('') + with self.assertRaises(ValueError) as err: + s.substitute({}) + self.assertIn('line 1, col 1', str(err.exception)) + def test_unicode_values(self): s = Template('$who likes $what') d = dict(who='t\xffm', what='f\xfe\fed') diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py index a352ee3..1615732 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string.py @@ -26,15 +26,38 @@ class ModuleTest(unittest.TestCase): self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') - def test_formatter(self): + def test_basic_formatter(self): fmt = string.Formatter() self.assertEqual(fmt.format("foo"), "foo") - self.assertEqual(fmt.format("foo{0}", "bar"), "foobar") self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6") + + def test_conversion_specifiers(self): + fmt = string.Formatter() self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-") + self.assertEqual(fmt.format("{0!s}", 'test'), 'test') + self.assertRaises(ValueError, fmt.format, "{0!h}", 'test') + + def test_name_lookup(self): + fmt = string.Formatter() + class AnyAttr: + def __getattr__(self, attr): + return attr + x = AnyAttr() + self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack') + with self.assertRaises(AttributeError): + fmt.format("{0.lumber}{0.jack}", '') + + def test_index_lookup(self): + fmt = string.Formatter() + lookup = ["eggs", "and", "spam"] + self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs') + with self.assertRaises(IndexError): + fmt.format("{0[2]}{0[0]}", []) + with self.assertRaises(KeyError): + fmt.format("{0[2]}{0[0]}", {}) - # override get_value ############################################ + def test_override_get_value(self): class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): string.Formatter.__init__(self) @@ -54,7 +77,7 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!') - # override format_field ######################################### + def test_override_format_field(self): class CallFormatter(string.Formatter): def format_field(self, value, format_spec): return format(value(), format_spec) @@ -63,18 +86,18 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*') - # override convert_field ######################################## + def test_override_convert_field(self): class XFormatter(string.Formatter): def convert_field(self, value, conversion): if conversion == 'x': return None - return super(XFormatter, self).convert_field(value, conversion) + return super().convert_field(value, conversion) fmt = XFormatter() self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None") - # override parse ################################################ + def test_override_parse(self): class BarFormatter(string.Formatter): # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) @@ -90,7 +113,7 @@ class ModuleTest(unittest.TestCase): fmt = BarFormatter() self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *') - # test all parameters used + def test_check_unused_args(self): class CheckAllUsedFormatter(string.Formatter): def check_unused_args(self, used_args, args, kwargs): # Track which arguments actually got used @@ -112,28 +135,13 @@ class ModuleTest(unittest.TestCase): self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100) self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100) - def test_vformat_assert(self): - cls = string.Formatter() - kwargs = { - "i": 100 - } - self.assertRaises(ValueError, cls._vformat, - cls.format, "{0}", kwargs, set(), -2) - - def test_convert_field(self): - cls = string.Formatter() - self.assertEqual(cls.format("{0!s}", 'foo'), 'foo') - self.assertRaises(ValueError, cls.format, "{0!h}", 'foo') - - def test_get_field(self): - cls = string.Formatter() - class MyClass: - name = 'lumberjack' - x = MyClass() - self.assertEqual(cls.format("{0.name}", x), 'lumberjack') - - lookup = ["eggs", "and", "spam"] - self.assertEqual(cls.format("{0[2]}", lookup), 'spam') + def test_vformat_recursion_limit(self): + fmt = string.Formatter() + args = () + kwargs = dict(i=100) + with self.assertRaises(ValueError) as err: + fmt._vformat("{i}", args, kwargs, set(), -1) + self.assertIn("recursion", str(err.exception)) def test_main(): diff --git a/Misc/ACKS b/Misc/ACKS index 0fc999d..93ec35f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -746,6 +746,7 @@ Eduardo Pérez Pierre Quentel Brian Quinlan Anders Qvist +Jérôme Radix Burton Radons Brodie Rao Antti Rasinen diff --git a/Misc/NEWS b/Misc/NEWS index ff7ed66..ce5a1d2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -852,8 +852,8 @@ Tests - Issue #11503: improve test coverage of posixpath.py. Patch by Evan Dandrea. -- Issue #11505: improves test coverage of string.py. Patch by Alicia - Arlen. +- Issue #11505: improves test coverage of string.py, increases granularity of + string.Formatter tests. Initial patch by Alicia Arlen. - Issue #11548: Improve test coverage of the shutil module. Patch by Evan Dandrea. -- cgit v0.12 From 5a5ce388ed3d67cd32f83a00ccf00ebbeaecb323 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Tue, 31 May 2011 12:09:34 +0200 Subject: make sure we check for write access before starting the install, and add correct exit code --- Lib/packaging/install.py | 48 ++++++++++++++++++++++++++++++------- Lib/packaging/run.py | 14 +++++++---- Lib/packaging/tests/test_install.py | 20 +++++++++++++--- Lib/packaging/tests/test_run.py | 25 ++++++++++++++++++- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/Lib/packaging/install.py b/Lib/packaging/install.py index 9f94532..4b82088 100644 --- a/Lib/packaging/install.py +++ b/Lib/packaging/install.py @@ -6,7 +6,6 @@ obtained from an index (e.g. PyPI), with dependencies. This is a higher-level module built on packaging.database and packaging.pypi. """ - import os import sys import stat @@ -14,7 +13,7 @@ import errno import shutil import logging import tempfile -from sysconfig import get_config_var +from sysconfig import get_config_var, get_path from packaging import logger from packaging.dist import Distribution @@ -28,6 +27,8 @@ from packaging.depgraph import generate_graph from packaging.errors import (PackagingError, InstallationException, InstallationConflict, CCompilerError) from packaging.pypi.errors import ProjectNotFound, ReleaseNotFound +from packaging import database + __all__ = ['install_dists', 'install_from_infos', 'get_infos', 'remove', 'install', 'install_local_project'] @@ -75,6 +76,7 @@ def _run_distutils_install(path): def _run_setuptools_install(path): cmd = '%s setup.py install --record=%s --single-version-externally-managed' record_file = os.path.join(path, 'RECORD') + os.system(cmd % (sys.executable, record_file)) if not os.path.exists(record_file): raise ValueError('failed to install') @@ -88,8 +90,10 @@ def _run_packaging_install(path): dist.parse_config_files() try: dist.run_command('install_dist') + name = dist.metadata['name'] + return database.get_distribution(name) is not None except (IOError, os.error, PackagingError, CCompilerError) as msg: - raise SystemExit("error: " + str(msg)) + raise ValueError("Failed to install, " + str(msg)) def _install_dist(dist, path): @@ -115,18 +119,20 @@ def install_local_project(path): If the source directory contains a setup.py install using distutils1. If a setup.cfg is found, install using the install_dist command. + Returns True on success, False on Failure. """ path = os.path.abspath(path) if os.path.isdir(path): logger.info('Installing from source directory: %s', path) - _run_install_from_dir(path) + return _run_install_from_dir(path) elif _is_archive_file(path): logger.info('Installing from archive: %s', path) _unpacked_dir = tempfile.mkdtemp() shutil.unpack_archive(path, _unpacked_dir) - _run_install_from_archive(_unpacked_dir) + return _run_install_from_archive(_unpacked_dir) else: logger.warning('No projects to install.') + return False def _run_install_from_archive(source_dir): @@ -152,7 +158,13 @@ def _run_install_from_dir(source_dir): func = install_methods[install_method] try: func = install_methods[install_method] - return func(source_dir) + try: + func(source_dir) + return True + except ValueError as err: + # failed to install + logger.info(str(err)) + return False finally: os.chdir(old_dir) @@ -472,16 +484,34 @@ def remove(project_name, paths=sys.path, auto_confirm=True): def install(project): + """Installs a project. + + Returns True on success, False on failure + """ + logger.info('Checking the installation location...') + purelib_path = get_path('purelib') + # trying to write a file there + try: + with tempfile.NamedTemporaryFile(suffix=project, + dir=purelib_path) as testfile: + testfile.write(b'test') + except OSError: + # was unable to write a file + logger.info('Unable to write in "%s". Do you have the permissions ?' + % purelib_path) + return False + + logger.info('Getting information about %r...', project) try: info = get_infos(project) except InstallationException: logger.info('Cound not find %r', project) - return + return False if info['install'] == []: logger.info('Nothing to install') - return + return False install_path = get_config_var('base') try: @@ -493,6 +523,8 @@ def install(project): projects = ['%s %s' % (p.name, p.version) for p in e.args[0]] logger.info('%r conflicts with %s', project, ','.join(projects)) + return True + def _main(**attrs): if 'script_args' not in attrs: diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py index 6e4816d..1a246c5 100644 --- a/Lib/packaging/run.py +++ b/Lib/packaging/run.py @@ -225,16 +225,22 @@ def _install(dispatcher, args, **kw): if 'setup.py' in listing or 'setup.cfg' in listing: args.insert(1, os.getcwd()) else: - logger.warning('no project to install') - return + logger.warning('No project to install.') + return 1 target = args[1] # installing from a source dir or archive file? if os.path.isdir(target) or _is_archive_file(target): - install_local_project(target) + if install_local_project(target): + return 0 + else: + return 1 else: # download from PyPI - install(target) + if install(target): + return 0 + else: + return 1 @action_help(metadata_usage) diff --git a/Lib/packaging/tests/test_install.py b/Lib/packaging/tests/test_install.py index c0924bf..c50a45e 100644 --- a/Lib/packaging/tests/test_install.py +++ b/Lib/packaging/tests/test_install.py @@ -1,11 +1,10 @@ """Tests for the packaging.install module.""" - import os from tempfile import mkstemp + from packaging import install from packaging.pypi.xmlrpc import Client from packaging.metadata import Metadata - from packaging.tests.support import (LoggingCatcher, TempdirManager, unittest, fake_dec) try: @@ -340,7 +339,7 @@ class TestInstall(LoggingCatcher, TempdirManager, unittest.TestCase): self.assertTrue(os.path.exists(f)) dist._unlink_installed_files() finally: - install.install_dist = old_install_dist + install._install_dist = old_install_dist install.uninstall = old_uninstall def test_install_from_infos_install_succes(self): @@ -357,6 +356,21 @@ class TestInstall(LoggingCatcher, TempdirManager, unittest.TestCase): finally: install._install_dist = old_install_dist + def test_install_permission_denied(self): + # if we don't have the access to the installation + # path, we should abort immediatly + project = os.path.join(os.path.dirname(__file__), 'package.tgz') + install_path = self.mkdtemp() + old_get_path = install.get_path + install.get_path = lambda path: install_path + old_mod = os.stat(install_path).st_mode + os.chmod(install_path, 0) + try: + self.assertFalse(install.install(project)) + finally: + os.chmod(install_path, old_mod) + install.get_path = old_get_path + def test_suite(): suite = unittest.TestSuite() diff --git a/Lib/packaging/tests/test_run.py b/Lib/packaging/tests/test_run.py index 01fa5aa..cb576b7 100644 --- a/Lib/packaging/tests/test_run.py +++ b/Lib/packaging/tests/test_run.py @@ -3,8 +3,12 @@ import os import sys import shutil +from tempfile import mkstemp +from io import StringIO +from packaging import install from packaging.tests import unittest, support, TESTFN +from packaging.run import main # setup script that uses __file__ setup_using___file__ = """\ @@ -25,7 +29,8 @@ setup() """ -class CoreTestCase(unittest.TestCase): +class CoreTestCase(support.TempdirManager, support.LoggingCatcher, + unittest.TestCase): def setUp(self): super(CoreTestCase, self).setUp() @@ -54,6 +59,24 @@ class CoreTestCase(unittest.TestCase): # TODO restore the tests removed six months ago and port them to pysetup + def test_install(self): + # making sure install returns 0 or 1 exit codes + project = os.path.join(os.path.dirname(__file__), 'package.tgz') + install_path = self.mkdtemp() + old_get_path = install.get_path + install.get_path = lambda path: install_path + old_mod = os.stat(install_path).st_mode + os.chmod(install_path, 0) + old_stderr = sys.stderr + sys.stderr = StringIO() + try: + self.assertFalse(install.install(project)) + self.assertEqual(main(['install', 'blabla']), 1) + finally: + sys.stderr = old_stderr + os.chmod(install_path, old_mod) + install.get_path = old_get_path + def test_suite(): return unittest.makeSuite(CoreTestCase) -- cgit v0.12 From e2655974cf631f21cd074904697163a5c6405f43 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Tue, 31 May 2011 12:15:42 +0200 Subject: code cleanup in packaging.run --- Lib/packaging/run.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py index 1a246c5..2d22bfd 100644 --- a/Lib/packaging/run.py +++ b/Lib/packaging/run.py @@ -351,7 +351,7 @@ def _list(dispatcher, args, **kw): number = 0 for dist in results: print('%s %s at %s' % (dist.name, dist.metadata['version'], dist.path)) - number +=1 + number += 1 print('') if number == 0: @@ -367,8 +367,9 @@ def _search(dispatcher, args, **kw): It is able to search for a specific index (specified with --index), using the simple or xmlrpc index types (with --type xmlrpc / --type simple) """ - opts = _parse_args(args[1:], '', ['simple', 'xmlrpc']) + #opts = _parse_args(args[1:], '', ['simple', 'xmlrpc']) # 1. what kind of index is requested ? (xmlrpc / simple) + raise NotImplementedError() actions = [ @@ -416,12 +417,9 @@ class Dispatcher: raise PackagingArgError(msg) self._set_logger() - - # for display options we return immediately - option_order = self.parser.get_option_order() - self.args = args + # for display options we return immediately if self.help or self.action is None: self._show_help(self.parser, display_options_=False) @@ -592,8 +590,6 @@ class Dispatcher: if isinstance(command, str): command = get_command_class(command) - name = command.get_command_name() - desc = getattr(command, 'description', '(no description available)') print('Description: %s' % desc) print('') -- cgit v0.12 From 9bcf8bfb243f66b235a2758573051390a30b6cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 14:08:26 +0200 Subject: Fix error message to use the Python name instead of the C name --- Modules/_threadmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index d91c99a..ea038de 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1091,7 +1091,7 @@ thread_PyThread_exit_thread(PyObject *self) PyDoc_STRVAR(exit_doc, "exit()\n\ -(PyThread_exit_thread() is an obsolete synonym)\n\ +(exit_thread() is an obsolete synonym)\n\ \n\ This is synonymous to ``raise SystemExit''. It will cause the current\n\ thread to exit silently unless the exception is caught."); -- cgit v0.12 From b4fefc8fa2722afa79fb8f519e74ab0b558ec19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 14:33:32 +0200 Subject: Fix test name in debug message --- Lib/packaging/tests/test_command_build_ext.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/packaging/tests/test_command_build_ext.py b/Lib/packaging/tests/test_command_build_ext.py index a7856d2..9729559 100644 --- a/Lib/packaging/tests/test_command_build_ext.py +++ b/Lib/packaging/tests/test_command_build_ext.py @@ -370,8 +370,8 @@ def test_suite(): src = _get_source_filename() if not os.path.exists(src): if verbose: - print ('test_build_ext: Cannot find source code (test' - ' must run in python build dir)') + print('test_command_build_ext: Cannot find source code (test' + ' must run in python build dir)') return unittest.TestSuite() else: return unittest.makeSuite(BuildExtTestCase) -- cgit v0.12 From a94bdee2e479cdfa6b98c7d823380a88dcb8c9fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 15:05:38 +0200 Subject: Fix nonsensical name. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code used “long” to refer to a long option (e.g. --quiet), which was probably changed by 2to3 and not caught by the human operator, and then changed to “integer” by me to avoid shadowing without seeing the real obvious fix. --- Lib/packaging/fancy_getopt.py | 56 +++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Lib/packaging/fancy_getopt.py b/Lib/packaging/fancy_getopt.py index 0490864..ebe376e 100644 --- a/Lib/packaging/fancy_getopt.py +++ b/Lib/packaging/fancy_getopt.py @@ -142,20 +142,20 @@ class FancyGetopt: for option in self.option_table: if len(option) == 3: - integer, short, help = option + longopt, short, help = option repeat = 0 elif len(option) == 4: - integer, short, help, repeat = option + longopt, short, help, repeat = option else: # the option table is part of the code, so simply # assert that it is correct raise ValueError("invalid option tuple: %r" % option) # Type- and value-check the option names - if not isinstance(integer, str) or len(integer) < 2: + if not isinstance(longopt, str) or len(longopt) < 2: raise PackagingGetoptError( ("invalid long option '%s': " - "must be a string of length >= 2") % integer) + "must be a string of length >= 2") % longopt) if (not ((short is None) or (isinstance(short, str) and len(short) == 1))): @@ -163,55 +163,55 @@ class FancyGetopt: ("invalid short option '%s': " "must be a single character or None") % short) - self.repeat[integer] = repeat - self.long_opts.append(integer) + self.repeat[longopt] = repeat + self.long_opts.append(longopt) - if integer[-1] == '=': # option takes an argument? + if longopt[-1] == '=': # option takes an argument? if short: short = short + ':' - integer = integer[0:-1] - self.takes_arg[integer] = 1 + longopt = longopt[0:-1] + self.takes_arg[longopt] = 1 else: # Is option is a "negative alias" for some other option (eg. # "quiet" == "!verbose")? - alias_to = self.negative_alias.get(integer) + alias_to = self.negative_alias.get(longopt) if alias_to is not None: if self.takes_arg[alias_to]: raise PackagingGetoptError( ("invalid negative alias '%s': " "aliased option '%s' takes a value") % \ - (integer, alias_to)) + (longopt, alias_to)) - self.long_opts[-1] = integer # XXX redundant?! - self.takes_arg[integer] = 0 + self.long_opts[-1] = longopt # XXX redundant?! + self.takes_arg[longopt] = 0 else: - self.takes_arg[integer] = 0 + self.takes_arg[longopt] = 0 # If this is an alias option, make sure its "takes arg" flag is # the same as the option it's aliased to. - alias_to = self.alias.get(integer) + alias_to = self.alias.get(longopt) if alias_to is not None: - if self.takes_arg[integer] != self.takes_arg[alias_to]: + if self.takes_arg[longopt] != self.takes_arg[alias_to]: raise PackagingGetoptError( ("invalid alias '%s': inconsistent with " "aliased option '%s' (one of them takes a value, " - "the other doesn't") % (integer, alias_to)) + "the other doesn't") % (longopt, alias_to)) # Now enforce some bondage on the long option name, so we can # later translate it to an attribute name on some object. Have # to do this a bit late to make sure we've removed any trailing # '='. - if not longopt_re.match(integer): + if not longopt_re.match(longopt): raise PackagingGetoptError( ("invalid long option name '%s' " + - "(must be letters, numbers, hyphens only") % integer) + "(must be letters, numbers, hyphens only") % longopt) - self.attr_name[integer] = integer.replace('-', '_') + self.attr_name[longopt] = longopt.replace('-', '_') if short: self.short_opts.append(short) - self.short2long[short[0]] = integer + self.short2long[short[0]] = longopt def getopt(self, args=None, object=None): """Parse command-line options in args. Store as attributes on object. @@ -297,10 +297,10 @@ class FancyGetopt: # First pass: determine maximum length of long option names max_opt = 0 for option in self.option_table: - integer = option[0] + longopt = option[0] short = option[1] - l = len(integer) - if integer[-1] == '=': + l = len(longopt) + if longopt[-1] == '=': l = l - 1 if short is not None: l = l + 5 # " (-x)" where short == 'x' @@ -340,20 +340,20 @@ class FancyGetopt: lines = ['Option summary:'] for option in self.option_table: - integer, short, help = option[:3] + longopt, short, help = option[:3] text = textwrap.wrap(help, text_width) # Case 1: no short option at all (makes life easy) if short is None: if text: - lines.append(" --%-*s %s" % (max_opt, integer, text[0])) + lines.append(" --%-*s %s" % (max_opt, longopt, text[0])) else: - lines.append(" --%-*s " % (max_opt, integer)) + lines.append(" --%-*s " % (max_opt, longopt)) # Case 2: we have a short option, so we have to include it # just after the long option else: - opt_names = "%s (-%s)" % (integer, short) + opt_names = "%s (-%s)" % (longopt, short) if text: lines.append(" --%-*s %s" % (max_opt, opt_names, text[0])) -- cgit v0.12 From 9deedf696eeb9a015118d1d0c76bf8318f2190d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 18:04:32 +0200 Subject: Re-apply distutils2 changes lost before the merge of packaging. wrap_text was removed in favor of standard textwrap but the removal of the function was lost in a bad merge; a change in sdist mysteriously disappeared. --- Lib/packaging/command/sdist.py | 36 +----------------- Lib/packaging/fancy_getopt.py | 63 ------------------------------- Lib/packaging/tests/test_command_sdist.py | 9 ++--- 3 files changed, 6 insertions(+), 102 deletions(-) diff --git a/Lib/packaging/command/sdist.py b/Lib/packaging/command/sdist.py index a28019b..a19203f 100644 --- a/Lib/packaging/command/sdist.py +++ b/Lib/packaging/command/sdist.py @@ -1,10 +1,9 @@ """Create a source distribution.""" import os -import sys import re +import sys from io import StringIO -from glob import glob from shutil import get_archive_formats, rmtree from packaging import logger @@ -203,45 +202,14 @@ class sdist(Command): def add_defaults(self): """Add all the default files to self.filelist: - - README or README.txt - - test/test*.py - all pure Python modules mentioned in setup script - all files pointed by package_data (build_py) - all files defined in data_files. - all files defined as scripts. - all C sources listed as part of extensions or C libraries in the setup script (doesn't catch C headers!) - Warns if (README or README.txt) or setup.py are missing; everything - else is optional. + Everything is optional. """ - standards = [('README', 'README.txt')] - for fn in standards: - if isinstance(fn, tuple): - alts = fn - got_it = False - for fn in alts: - if os.path.exists(fn): - got_it = True - self.filelist.append(fn) - break - - if not got_it: - logger.warning( - '%s: standard file not found: should have one of %s', - self.get_command_name(), ', '.join(alts)) - else: - if os.path.exists(fn): - self.filelist.append(fn) - else: - logger.warning('%s: standard file %r not found', - self.get_command_name(), fn) - - optional = ['test/test*.py', 'setup.cfg'] - for pattern in optional: - files = [f for f in glob(pattern) if os.path.isfile(f)] - if files: - self.filelist.extend(files) - for cmd_name in get_command_names(): try: cmd_obj = self.get_finalized_command(cmd_name) diff --git a/Lib/packaging/fancy_getopt.py b/Lib/packaging/fancy_getopt.py index ebe376e..61dd5fc 100644 --- a/Lib/packaging/fancy_getopt.py +++ b/Lib/packaging/fancy_getopt.py @@ -13,7 +13,6 @@ It is used under the hood by the command classes. Do not use directly. import getopt import re import sys -import string import textwrap from packaging.errors import PackagingGetoptError, PackagingArgError @@ -378,68 +377,6 @@ def fancy_getopt(options, negative_opt, object, args): return parser.getopt(args, object) -WS_TRANS = str.maketrans(string.whitespace, ' ' * len(string.whitespace)) - - -def wrap_text(text, width): - """Split *text* into lines of no more than *width* characters each. - - *text* is a str and *width* an int. Returns a list of str. - """ - - if text is None: - return [] - if len(text) <= width: - return [text] - - text = text.expandtabs() - text = text.translate(WS_TRANS) - - chunks = re.split(r'( +|-+)', text) - chunks = [_f for _f in chunks if _f] # ' - ' results in empty strings - lines = [] - - while chunks: - - cur_line = [] # list of chunks (to-be-joined) - cur_len = 0 # length of current line - - while chunks: - l = len(chunks[0]) - if cur_len + l <= width: # can squeeze (at least) this chunk in - cur_line.append(chunks[0]) - del chunks[0] - cur_len = cur_len + l - else: # this line is full - # drop last chunk if all space - if cur_line and cur_line[-1][0] == ' ': - del cur_line[-1] - break - - if chunks: # any chunks left to process? - - # if the current line is still empty, then we had a single - # chunk that's too big too fit on a line -- so we break - # down and break it up at the line width - if cur_len == 0: - cur_line.append(chunks[0][0:width]) - chunks[0] = chunks[0][width:] - - # all-whitespace chunks at the end of a line can be discarded - # (and we know from the re.split above that if a chunk has - # *any* whitespace, it is *all* whitespace) - if chunks[0][0] == ' ': - del chunks[0] - - # and store this line in the list-of-all-lines -- as a single - # string, of course! - lines.append(''.join(cur_line)) - - # while chunks - - return lines - - class OptionDummy: """Dummy class just used as a place to hold command-line option values as instance attributes.""" diff --git a/Lib/packaging/tests/test_command_sdist.py b/Lib/packaging/tests/test_command_sdist.py index a086e62..41b2a24 100644 --- a/Lib/packaging/tests/test_command_sdist.py +++ b/Lib/packaging/tests/test_command_sdist.py @@ -33,7 +33,6 @@ setup(name='fake') MANIFEST = """\ # file GENERATED by packaging, do NOT edit -README inroot.txt data%(sep)sdata.dt scripts%(sep)sscript.py @@ -129,7 +128,7 @@ class SDistTestCase(support.TempdirManager, content = zip_file.namelist() # making sure everything has been pruned correctly - self.assertEqual(len(content), 3) + self.assertEqual(len(content), 2) @requires_zlib @unittest.skipIf(find_executable('tar') is None or @@ -214,7 +213,7 @@ class SDistTestCase(support.TempdirManager, # Making sure everything was added. This includes 9 code and data # files in addition to PKG-INFO. - self.assertEqual(len(content), 10) + self.assertEqual(len(content), 9) # Checking the MANIFEST with open(join(self.tmp_dir, 'MANIFEST')) as fp: @@ -331,7 +330,7 @@ class SDistTestCase(support.TempdirManager, with open(cmd.manifest) as f: manifest = [line.strip() for line in f.read().split('\n') if line.strip() != ''] - self.assertEqual(len(manifest), 4) + self.assertEqual(len(manifest), 3) # Adding a file self.write_file((self.tmp_dir, 'somecode', 'doc2.txt'), '#') @@ -348,7 +347,7 @@ class SDistTestCase(support.TempdirManager, if line.strip() != ''] # Do we have the new file in MANIFEST? - self.assertEqual(len(manifest2), 5) + self.assertEqual(len(manifest2), 4) self.assertIn('doc2.txt', manifest2[-1]) @requires_zlib -- cgit v0.12 From 5864b9fd3c5a227f73845b166e75fcbaf352a661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 21:50:38 +0200 Subject: Fix markup: arguments in a class directive are __init__ arguments, not base classes --- Doc/distutils/apiref.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index dc5bcf2..1fb6f9e 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -1814,7 +1814,7 @@ Subclasses of :class:`Command` must define the following methods. .. module:: distutils.command.bdist_msi :synopsis: Build a binary distribution as a Windows MSI file -.. class:: bdist_msi(Command) +.. class:: bdist_msi Builds a `Windows Installer`_ (.msi) binary package. @@ -1893,9 +1893,9 @@ Subclasses of :class:`Command` must define the following methods. :synopsis: Build the .py/.pyc files of a package -.. class:: build_py(Command) +.. class:: build_py -.. class:: build_py_2to3(build_py) +.. class:: build_py_2to3 Alternative implementation of build_py which also runs the 2to3 conversion library on each .py file that is going to be -- cgit v0.12 From 208226802572c8eeda17b3a0d7f214b57c87f7ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 31 May 2011 22:31:09 +0200 Subject: Issue #8407: test_signal doesn't check signal delivery order Tthe signal delivery order is not portable or reliable. --- Lib/test/test_signal.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 4bf2d2d..0631390 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -229,12 +229,13 @@ class WakeupSignalTests(unittest.TestCase): def handler(self, signum, frame): pass - def check_signum(self, *signals, **kw): + def check_signum(self, *signals): data = os.read(self.read, len(signals)+1) raised = struct.unpack('%uB' % len(data), data) - if kw.get('unordered', False): - raised = set(raised) - signals = set(signals) + # We don't care of the signal delivery order (it's not portable or + # reliable) + raised = set(raised) + signals = set(signals) self.assertEqual(raised, signals) def test_wakeup_fd_early(self): @@ -291,7 +292,7 @@ class WakeupSignalTests(unittest.TestCase): # Unblocking the 2 signals calls the C signal handler twice signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2)) - self.check_signum(signum1, signum2, unordered=True) + self.check_signum(signum1, signum2) def setUp(self): import fcntl -- cgit v0.12 From 1256a6864e4dc1728a6d7034d3f3dd322f87b02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Tue, 31 May 2011 22:38:41 +0200 Subject: Re-add line that was mistakenly removed alongside RPM support. This reveals the need for bdist* tests. --- Lib/packaging/command/bdist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/packaging/command/bdist.py b/Lib/packaging/command/bdist.py index 4338a97..e8c023d 100644 --- a/Lib/packaging/command/bdist.py +++ b/Lib/packaging/command/bdist.py @@ -128,6 +128,7 @@ class bdist(Command): for i in range(len(self.formats)): cmd_name = commands[i] sub_cmd = self.get_reinitialized_command(cmd_name) + sub_cmd.format = self.formats[i] # passing the owner and group names for tar archiving if cmd_name == 'bdist_dumb': -- cgit v0.12 From bc93a116eb8f73bae881503feb17d966d61680ed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 Jun 2011 00:01:24 +0200 Subject: Close #12229: Remove an unused argument of _bufferedreader_peek_unlocked(), io.BufferedReader._peek_unlocked(). Patch written by John O'Connor. --- Modules/_io/bufferedio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 63ae1cb..cdaa36e 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -589,7 +589,7 @@ _bufferedreader_reset_buf(buffered *self); static void _bufferedwriter_reset_buf(buffered *self); static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t); +_bufferedreader_peek_unlocked(buffered *self); static PyObject * _bufferedreader_read_all(buffered *self); static PyObject * @@ -797,7 +797,7 @@ buffered_peek(buffered *self, PyObject *args) goto end; Py_CLEAR(res); } - res = _bufferedreader_peek_unlocked(self, n); + res = _bufferedreader_peek_unlocked(self); end: LEAVE_BUFFERED(self) @@ -1611,7 +1611,7 @@ error: } static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t n) +_bufferedreader_peek_unlocked(buffered *self) { Py_ssize_t have, r; -- cgit v0.12 From 023f3a7cf6124498598ec82e03baa1219783deed Mon Sep 17 00:00:00 2001 From: Ralf Schmitt Date: Tue, 31 May 2011 17:10:03 -0500 Subject: disable ASDLGEN if hg won't work, or if python is not installed. This change makes configure check for - the existence of a hg repository - the hg executable itself - the python executable Running $(srcdir)/Parser/asdl_c.py (i.e. ASDLGEN) will fail if any of the above prerequisites is missing, so we now disable it instead. closes #12225 --- Makefile.pre.in | 2 +- configure.in | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index f6ecacf..3cf8ec1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -274,7 +274,7 @@ AST_ASDL= $(srcdir)/Parser/Python.asdl ASDLGEN_FILES= $(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py # XXX Note that a build now requires Python exist before the build starts -ASDLGEN= $(srcdir)/Parser/asdl_c.py +ASDLGEN= @DISABLE_ASDLGEN@ $(srcdir)/Parser/asdl_c.py ########################################################################## # Python diff --git a/configure.in b/configure.in index b5f155d..49932de 100644 --- a/configure.in +++ b/configure.in @@ -811,7 +811,13 @@ fi AC_SUBST(HGVERSION) AC_SUBST(HGTAG) AC_SUBST(HGBRANCH) + +if test -e $srcdir/.hg/00changelog.i +then AC_CHECK_PROG(HAS_HG, hg, found, not-found) +else +HAS_HG=no-repository +fi if test $HAS_HG = found then HGVERSION="hg id -i \$(srcdir)" @@ -823,6 +829,15 @@ else HGBRANCH="" fi +AC_SUBST(DISABLE_ASDLGEN) +DISABLE_ASDLGEN="" +AC_CHECK_PROG(HAS_PYTHON, python, found, not-found) +if test $HAS_HG != found -o $HAS_PYTHON != found +then + DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #" +fi + + case $MACHDEP in bsdos*|hp*|HP*) # install -d does not work on BSDI or HP-UX -- cgit v0.12 From 0bb299165344129dcdc0891233f3799ca258d4a3 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 17:11:26 -0500 Subject: add news note --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index ce5a1d2..7c82dbe 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12225: Still allow Python to build if Python is not in its hg repo or + mercurial is not installed. + - Issue #1195: my_fgets() now always clears errors before calling fgets(). Fix the following case: sys.stdin.read() stopped with CTRL+d (end of file), raw_input() interrupted by CTRL+c. -- cgit v0.12 From 87b9bc3893bac402bd773a83ee6734507f978607 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 Jun 2011 00:57:47 +0200 Subject: Close #12085: Fix an attribute error in subprocess.Popen destructor if the constructor has failed, e.g. because of an undeclared keyword argument. Patch written by Oleg Oshmyan. --- Lib/subprocess.py | 5 ++++- Lib/test/test_subprocess.py | 10 ++++++++++ Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 28dd691..4bcf159 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -768,7 +768,10 @@ class Popen(object): self.wait() def __del__(self, _maxsize=sys.maxsize, _active=_active): - if not self._child_created: + # If __init__ hasn't had a chance to execute (e.g. if it + # was passed an undeclared keyword argument), we don't + # have a _child_created attribute at all. + if not getattr(self, '_child_created', False): # We didn't get to successfully create a child process. return # In case the child hasn't been waited on, check if it's done. diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ad89864..9d66659 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -121,6 +121,16 @@ class ProcessTestCase(BaseTestCase): env=newenv) self.assertEqual(rc, 1) + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + def test_stdin_none(self): # .stdin is None when not redirected p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], diff --git a/Misc/ACKS b/Misc/ACKS index d0e32e1..aa077fc 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -650,6 +650,7 @@ Piet van Oostrum Jason Orendorff Douglas Orr Michele Orrù +Oleg Oshmyan Denis S. Otkidach Michael Otteneder R. M. Oudkerk diff --git a/Misc/NEWS b/Misc/NEWS index 4ab11a5..3703ac5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,10 @@ Core and Builtins Library ------- +- Issue #12085: Fix an attribute error in subprocess.Popen destructor if the + constructor has failed, e.g. because of an undeclared keyword argument. Patch + written by Oleg Oshmyan. + - Issue #985064: Make plistlib more resilient to faulty input plists. Patch by Mher Movsisyan. -- cgit v0.12 From 87cdb81646f6628922c5781dbc15a4dbffb8e923 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 18:26:08 -0500 Subject: run autoreconf --- configure | 570 +++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 302 insertions(+), 268 deletions(-) diff --git a/configure b/configure index 89a5d70..a3f2a3e 100755 --- a/configure +++ b/configure @@ -1,14 +1,14 @@ #! /bin/sh # From configure.in Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.67 for python 3.3. +# Generated by GNU Autoconf 2.65 for python 3.3. # # Report bugs to . # # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, -# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software -# Foundation, Inc. +# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, +# Inc. # # # This configure script is free software; the Free Software Foundation @@ -320,7 +320,7 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" } # as_fn_mkdir_p @@ -360,19 +360,19 @@ else fi # as_fn_arith -# as_fn_error STATUS ERROR [LINENO LOG_FD] -# ---------------------------------------- +# as_fn_error ERROR [LINENO LOG_FD] +# --------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with STATUS, using 1 if that was 0. +# script with status $?, using 1 if that was 0. as_fn_error () { - as_status=$1; test $as_status -eq 0 && as_status=1 - if test "$4"; then - as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + as_status=$?; test $as_status -eq 0 && as_status=1 + if test "$3"; then + as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 fi - $as_echo "$as_me: error: $2" >&2 + $as_echo "$as_me: error: $1" >&2 as_fn_exit $as_status } # as_fn_error @@ -534,7 +534,7 @@ test -n "$DJDIR" || exec 7<&0 &1 # Name of the host. -# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, +# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, # so uname gets run too. ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` @@ -636,6 +636,8 @@ LN INSTALL_DATA INSTALL_SCRIPT INSTALL_PROGRAM +HAS_PYTHON +DISABLE_ASDLGEN HAS_HG HGBRANCH HGTAG @@ -832,9 +834,8 @@ do fi case $ac_option in - *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; - *=) ac_optarg= ;; - *) ac_optarg=yes ;; + *=*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *) ac_optarg=yes ;; esac # Accept the important Cygnus configure options, so we can diagnose typos. @@ -879,7 +880,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: $ac_useropt" + as_fn_error "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -905,7 +906,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid feature name: $ac_useropt" + as_fn_error "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1109,7 +1110,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: $ac_useropt" + as_fn_error "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1125,7 +1126,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error $? "invalid package name: $ac_useropt" + as_fn_error "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1155,8 +1156,8 @@ do | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; - -*) as_fn_error $? "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information" + -*) as_fn_error "unrecognized option: \`$ac_option' +Try \`$0 --help' for more information." ;; *=*) @@ -1164,7 +1165,7 @@ Try \`$0 --help' for more information" # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; + as_fn_error "invalid variable name: \`$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; @@ -1182,13 +1183,13 @@ done if test -n "$ac_prev"; then ac_option=--`echo $ac_prev | sed 's/_/-/g'` - as_fn_error $? "missing argument to $ac_option" + as_fn_error "missing argument to $ac_option" fi if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; - fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; + fatal) as_fn_error "unrecognized options: $ac_unrecognized_opts" ;; *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi @@ -1211,7 +1212,7 @@ do [\\/$]* | ?:[\\/]* ) continue;; NONE | '' ) case $ac_var in *prefix ) continue;; esac;; esac - as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" + as_fn_error "expected an absolute directory name for --$ac_var: $ac_val" done # There might be people who depend on the old broken behavior: `$host' @@ -1225,8 +1226,8 @@ target=$target_alias if test "x$host_alias" != x; then if test "x$build_alias" = x; then cross_compiling=maybe - $as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host. - If a cross compiler is detected then cross compile mode will be used" >&2 + $as_echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used." >&2 elif test "x$build_alias" != "x$host_alias"; then cross_compiling=yes fi @@ -1241,9 +1242,9 @@ test "$silent" = yes && exec 6>/dev/null ac_pwd=`pwd` && test -n "$ac_pwd" && ac_ls_di=`ls -di .` && ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || - as_fn_error $? "working directory cannot be determined" + as_fn_error "working directory cannot be determined" test "X$ac_ls_di" = "X$ac_pwd_ls_di" || - as_fn_error $? "pwd does not report name of working directory" + as_fn_error "pwd does not report name of working directory" # Find the source files, if location was not specified. @@ -1282,11 +1283,11 @@ else fi if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." - as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" + as_fn_error "cannot find sources ($ac_unique_file) in $srcdir" fi ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" ac_abs_confdir=`( - cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" + cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error "$ac_msg" pwd)` # When building in place, set srcdir=. if test "$ac_abs_confdir" = "$ac_pwd"; then @@ -1326,7 +1327,7 @@ Configuration: --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking ...' messages + -q, --quiet, --silent do not print \`checking...' messages --cache-file=FILE cache test results in FILE [disabled] -C, --config-cache alias for \`--cache-file=config.cache' -n, --no-create do not create output files @@ -1511,9 +1512,9 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF python configure 3.3 -generated by GNU Autoconf 2.67 +generated by GNU Autoconf 2.65 -Copyright (C) 2010 Free Software Foundation, Inc. +Copyright (C) 2009 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -1629,7 +1630,7 @@ $as_echo "$ac_try_echo"; } >&5 mv -f conftest.er1 conftest.err fi $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } > conftest.i && { + test $ac_status = 0; } >/dev/null && { test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || test ! -s conftest.err }; then : @@ -1653,10 +1654,10 @@ fi ac_fn_c_check_header_mongrel () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if eval "test \"\${$3+set}\"" = set; then : + if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 fi eval ac_res=\$$3 @@ -1692,7 +1693,7 @@ if ac_fn_c_try_cpp "$LINENO"; then : else ac_header_preproc=no fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 $as_echo "$ac_header_preproc" >&6; } @@ -1715,15 +1716,17 @@ $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} $as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 $as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} -( $as_echo "## -------------------------------------- ## +( cat <<\_ASBOX +## -------------------------------------- ## ## Report this to http://bugs.python.org/ ## -## -------------------------------------- ##" +## -------------------------------------- ## +_ASBOX ) | sed "s/^/$as_me: WARNING: /" >&2 ;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else eval "$3=\$ac_header_compiler" @@ -1787,7 +1790,7 @@ ac_fn_c_check_header_compile () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -1818,7 +1821,7 @@ ac_fn_c_check_type () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else eval "$3=no" @@ -1872,7 +1875,7 @@ ac_fn_c_find_uintX_t () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for uint$2_t" >&5 $as_echo_n "checking for uint$2_t... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else eval "$3=no" @@ -1902,7 +1905,8 @@ if ac_fn_c_try_compile "$LINENO"; then : esac fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - if eval test \"x\$"$3"\" = x"no"; then : + eval as_val=\$$3 + if test "x$as_val" = x""no; then : else break @@ -1925,7 +1929,7 @@ ac_fn_c_find_intX_t () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for int$2_t" >&5 $as_echo_n "checking for int$2_t... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else eval "$3=no" @@ -1976,7 +1980,8 @@ fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - if eval test \"x\$"$3"\" = x"no"; then : + eval as_val=\$$3 + if test "x$as_val" = x""no; then : else break @@ -2176,7 +2181,7 @@ ac_fn_c_check_func () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -2244,7 +2249,7 @@ ac_fn_c_check_member () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2.$3" >&5 $as_echo_n "checking for $2.$3... " >&6; } -if eval "test \"\${$4+set}\"" = set; then : +if { as_var=$4; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -2292,18 +2297,15 @@ $as_echo "$ac_res" >&6; } } # ac_fn_c_check_member -# ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES -# --------------------------------------------- -# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR -# accordingly. +# ac_fn_c_check_decl LINENO SYMBOL VAR +# ------------------------------------ +# Tests whether SYMBOL is declared, setting cache variable VAR accordingly. ac_fn_c_check_decl () { as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - as_decl_name=`echo $2|sed 's/ *(.*//'` - as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'` - { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5 -$as_echo_n "checking whether $as_decl_name is declared... " >&6; } -if eval "test \"\${$3+set}\"" = set; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $2 is declared" >&5 +$as_echo_n "checking whether $2 is declared... " >&6; } +if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -2312,12 +2314,8 @@ $4 int main () { -#ifndef $as_decl_name -#ifdef __cplusplus - (void) $as_decl_use; -#else - (void) $as_decl_name; -#endif +#ifndef $2 + (void) $2; #endif ; @@ -2342,7 +2340,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by python $as_me 3.3, which was -generated by GNU Autoconf 2.67. Invocation command line was +generated by GNU Autoconf 2.65. Invocation command line was $ $0 $@ @@ -2452,9 +2450,11 @@ trap 'exit_status=$? { echo - $as_echo "## ---------------- ## + cat <<\_ASBOX +## ---------------- ## ## Cache variables. ## -## ---------------- ##" +## ---------------- ## +_ASBOX echo # The following way of writing the cache mishandles newlines in values, ( @@ -2488,9 +2488,11 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; ) echo - $as_echo "## ----------------- ## + cat <<\_ASBOX +## ----------------- ## ## Output variables. ## -## ----------------- ##" +## ----------------- ## +_ASBOX echo for ac_var in $ac_subst_vars do @@ -2503,9 +2505,11 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; echo if test -n "$ac_subst_files"; then - $as_echo "## ------------------- ## + cat <<\_ASBOX +## ------------------- ## ## File substitutions. ## -## ------------------- ##" +## ------------------- ## +_ASBOX echo for ac_var in $ac_subst_files do @@ -2519,9 +2523,11 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; fi if test -s confdefs.h; then - $as_echo "## ----------- ## + cat <<\_ASBOX +## ----------- ## ## confdefs.h. ## -## ----------- ##" +## ----------- ## +_ASBOX echo cat confdefs.h echo @@ -2576,12 +2582,7 @@ _ACEOF ac_site_file1=NONE ac_site_file2=NONE if test -n "$CONFIG_SITE"; then - # We do not want a PATH search for config.site. - case $CONFIG_SITE in #(( - -*) ac_site_file1=./$CONFIG_SITE;; - */*) ac_site_file1=$CONFIG_SITE;; - *) ac_site_file1=./$CONFIG_SITE;; - esac + ac_site_file1=$CONFIG_SITE elif test "x$prefix" != xNONE; then ac_site_file1=$prefix/share/config.site ac_site_file2=$prefix/etc/config.site @@ -2596,11 +2597,7 @@ do { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 $as_echo "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 - . "$ac_site_file" \ - || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "failed to load site script $ac_site_file -See \`config.log' for more details" "$LINENO" 5 ; } + . "$ac_site_file" fi done @@ -2676,7 +2673,7 @@ if $ac_cache_corrupted; then $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 $as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 + as_fn_error "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## ## Main body of script. ## @@ -2777,7 +2774,7 @@ if test "${enable_universalsdk+set}" = set; then : UNIVERSALSDK=$enableval if test ! -d "${UNIVERSALSDK}" then - as_fn_error $? "--enable-universalsdk specifies non-existing SDK: ${UNIVERSALSDK}" "$LINENO" 5 + as_fn_error "--enable-universalsdk specifies non-existing SDK: ${UNIVERSALSDK}" "$LINENO" 5 fi ;; esac @@ -3169,7 +3166,7 @@ $as_echo "$without_gcc" >&6; } # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then - as_fn_error $? "cached CC is different -- throw away $cache_file + as_fn_error "cached CC is different -- throw away $cache_file (it is also a good idea to do 'make clean' before compiling)" "$LINENO" 5 fi @@ -3479,8 +3476,8 @@ fi test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "no acceptable C compiler found in \$PATH -See \`config.log' for more details" "$LINENO" 5 ; } +as_fn_error "no acceptable C compiler found in \$PATH +See \`config.log' for more details." "$LINENO" 5; } # Provide some information about the compiler. $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 @@ -3594,8 +3591,9 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "C compiler cannot create executables -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "C compiler cannot create executables +See \`config.log' for more details." "$LINENO" 5; }; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } @@ -3637,8 +3635,8 @@ done else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details" "$LINENO" 5 ; } +as_fn_error "cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details." "$LINENO" 5; } fi rm -f conftest conftest$ac_cv_exeext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 @@ -3695,9 +3693,9 @@ $as_echo "$ac_try_echo"; } >&5 else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot run C compiled programs. +as_fn_error "cannot run C compiled programs. If you meant to cross compile, use \`--host'. -See \`config.log' for more details" "$LINENO" 5 ; } +See \`config.log' for more details." "$LINENO" 5; } fi fi fi @@ -3748,8 +3746,8 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "cannot compute suffix of object files: cannot compile -See \`config.log' for more details" "$LINENO" 5 ; } +as_fn_error "cannot compute suffix of object files: cannot compile +See \`config.log' for more details." "$LINENO" 5; } fi rm -f conftest.$ac_cv_objext conftest.$ac_ext fi @@ -4232,7 +4230,7 @@ else # Broken: fails on valid input. continue fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. @@ -4248,11 +4246,11 @@ else ac_preproc_ok=: break fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext if $ac_preproc_ok; then : break fi @@ -4291,7 +4289,7 @@ else # Broken: fails on valid input. continue fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. @@ -4307,18 +4305,18 @@ else ac_preproc_ok=: break fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.i conftest.err conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext if $ac_preproc_ok; then : else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details" "$LINENO" 5 ; } +as_fn_error "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details." "$LINENO" 5; } fi ac_ext=c @@ -4379,7 +4377,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_GREP"; then - as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_GREP=$GREP @@ -4445,7 +4443,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_EGREP"; then - as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_EGREP=$EGREP @@ -4577,7 +4575,8 @@ do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default " -if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : +eval as_val=\$$as_ac_Header + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF @@ -5142,6 +5141,9 @@ fi + +if test -e $srcdir/.hg/00changelog.i +then # Extract the first word of "hg", so it can be a program name with args. set dummy hg; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -5180,6 +5182,9 @@ $as_echo "no" >&6; } fi +else +HAS_HG=no-repository +fi if test $HAS_HG = found then HGVERSION="hg id -i \$(srcdir)" @@ -5191,6 +5196,52 @@ else HGBRANCH="" fi + +DISABLE_ASDLGEN="" +# Extract the first word of "python", so it can be a program name with args. +set dummy python; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_HAS_PYTHON+set}" = set; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$HAS_PYTHON"; then + ac_cv_prog_HAS_PYTHON="$HAS_PYTHON" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_HAS_PYTHON="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_HAS_PYTHON" && ac_cv_prog_HAS_PYTHON="not-found" +fi +fi +HAS_PYTHON=$ac_cv_prog_HAS_PYTHON +if test -n "$HAS_PYTHON"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_PYTHON" >&5 +$as_echo "$HAS_PYTHON" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +if test $HAS_HG != found -o $HAS_PYTHON != found +then + DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #" +fi + + case $MACHDEP in bsdos*|hp*|HP*) # install -d does not work on BSDI or HP-UX @@ -5201,22 +5252,16 @@ bsdos*|hp*|HP*) esac ac_aux_dir= for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do - if test -f "$ac_dir/install-sh"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/install-sh -c" - break - elif test -f "$ac_dir/install.sh"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/install.sh -c" - break - elif test -f "$ac_dir/shtool"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/shtool install -c" - break - fi + for ac_t in install-sh install.sh shtool; do + if test -f "$ac_dir/$ac_t"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/$ac_t -c" + break 2 + fi + done done if test -z "$ac_aux_dir"; then - as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 + as_fn_error "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 fi # These three variables are undocumented and unsupported, @@ -5554,7 +5599,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" else - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 fi @@ -6042,7 +6087,8 @@ bluetooth/bluetooth.h linux/tipc.h spawn.h util.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : +eval as_val=\$$as_ac_Header + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF @@ -6056,7 +6102,7 @@ for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h; do as_ac_Header=`$as_echo "ac_cv_header_dirent_$ac_hdr" | $as_tr_sh` { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_hdr that defines DIR" >&5 $as_echo_n "checking for $ac_hdr that defines DIR... " >&6; } -if eval "test \"\${$as_ac_Header+set}\"" = set; then : +if { as_var=$as_ac_Header; eval "test \"\${$as_var+set}\" = set"; }; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -6083,7 +6129,8 @@ fi eval ac_res=\$$as_ac_Header { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } -if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : +eval as_val=\$$as_ac_Header + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_hdr" | $as_tr_cpp` 1 _ACEOF @@ -6608,8 +6655,9 @@ else if test "$ac_cv_type_int" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (int) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (int) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_int=0 fi @@ -6641,8 +6689,9 @@ else if test "$ac_cv_type_long" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (long) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (long) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_long=0 fi @@ -6674,8 +6723,9 @@ else if test "$ac_cv_type_void_p" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (void *) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (void *) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_void_p=0 fi @@ -6707,8 +6757,9 @@ else if test "$ac_cv_type_short" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (short) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (short) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_short=0 fi @@ -6740,8 +6791,9 @@ else if test "$ac_cv_type_float" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (float) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (float) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_float=0 fi @@ -6773,8 +6825,9 @@ else if test "$ac_cv_type_double" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (double) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (double) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_double=0 fi @@ -6806,8 +6859,9 @@ else if test "$ac_cv_type_fpos_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (fpos_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (fpos_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_fpos_t=0 fi @@ -6839,8 +6893,9 @@ else if test "$ac_cv_type_size_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (size_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (size_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_size_t=0 fi @@ -6872,8 +6927,9 @@ else if test "$ac_cv_type_pid_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (pid_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (pid_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_pid_t=0 fi @@ -6932,8 +6988,9 @@ else if test "$ac_cv_type_long_long" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (long long) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (long long) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_long_long=0 fi @@ -6993,8 +7050,9 @@ else if test "$ac_cv_type_long_double" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (long double) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (long double) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_long_double=0 fi @@ -7055,8 +7113,9 @@ else if test "$ac_cv_type__Bool" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (_Bool) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (_Bool) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof__Bool=0 fi @@ -7103,8 +7162,9 @@ else if test "$ac_cv_type_uintptr_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (uintptr_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (uintptr_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_uintptr_t=0 fi @@ -7144,8 +7204,9 @@ else if test "$ac_cv_type_off_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (off_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (off_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_off_t=0 fi @@ -7206,8 +7267,9 @@ else if test "$ac_cv_type_time_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (time_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (time_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_time_t=0 fi @@ -7278,8 +7340,9 @@ else if test "$ac_cv_type_pthread_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (pthread_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (pthread_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_pthread_t=0 fi @@ -7366,7 +7429,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error "Unexpected output of 'arch' on OSX" "$LINENO" 5 ;; esac else @@ -7378,7 +7441,7 @@ fi MACOSX_DEFAULT_ARCH="ppc64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error "Unexpected output of 'arch' on OSX" "$LINENO" 5 ;; esac @@ -7404,7 +7467,7 @@ $as_echo "#define WITH_NEXT_FRAMEWORK 1" >>confdefs.h $as_echo "yes" >&6; } if test $enable_shared = "yes" then - as_fn_error $? "Specifying both --enable-shared and --enable-framework is not supported, use only --enable-framework instead" "$LINENO" 5 + as_fn_error "Specifying both --enable-shared and --enable-framework is not supported, use only --enable-framework instead" "$LINENO" 5 fi else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 @@ -8244,12 +8307,12 @@ if test "${with_dbmliborder+set}" = set; then : withval=$with_dbmliborder; if test x$with_dbmliborder = xyes then -as_fn_error $? "proper usage is --with-dbmliborder=db1:db2:..." "$LINENO" 5 +as_fn_error "proper usage is --with-dbmliborder=db1:db2:..." "$LINENO" 5 else for db in `echo $with_dbmliborder | sed 's/:/ /g'`; do if test x$db != xndbm && test x$db != xgdbm && test x$db != xbdb then - as_fn_error $? "proper usage is --with-dbmliborder=db1:db2:..." "$LINENO" 5 + as_fn_error "proper usage is --with-dbmliborder=db1:db2:..." "$LINENO" 5 fi done fi @@ -9222,7 +9285,7 @@ if test "x$ac_cv_header_valgrind_valgrind_h" = x""yes; then : $as_echo "#define WITH_VALGRIND 1" >>confdefs.h else - as_fn_error $? "Valgrind support requested but headers not available" "$LINENO" 5 + as_fn_error "Valgrind support requested but headers not available" "$LINENO" 5 fi @@ -9321,7 +9384,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" -if eval test \"x\$"$as_ac_var"\" = x"yes"; then : +eval as_val=\$$as_ac_var + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF @@ -10344,7 +10408,8 @@ for ac_func in fseek64 fseeko fstatvfs ftell64 ftello statvfs do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" -if eval test \"x\$"$as_ac_var"\" = x"yes"; then : +eval as_val=\$$as_ac_var + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF @@ -10353,44 +10418,25 @@ fi done -ac_fn_c_check_func "$LINENO" "dup2" "ac_cv_func_dup2" -if test "x$ac_cv_func_dup2" = x""yes; then : - $as_echo "#define HAVE_DUP2 1" >>confdefs.h - -else - case " $LIBOBJS " in - *" dup2.$ac_objext "* ) ;; - *) LIBOBJS="$LIBOBJS dup2.$ac_objext" - ;; -esac - -fi - -ac_fn_c_check_func "$LINENO" "getcwd" "ac_cv_func_getcwd" -if test "x$ac_cv_func_getcwd" = x""yes; then : - $as_echo "#define HAVE_GETCWD 1" >>confdefs.h - -else - case " $LIBOBJS " in - *" getcwd.$ac_objext "* ) ;; - *) LIBOBJS="$LIBOBJS getcwd.$ac_objext" - ;; -esac - -fi - -ac_fn_c_check_func "$LINENO" "strdup" "ac_cv_func_strdup" -if test "x$ac_cv_func_strdup" = x""yes; then : - $as_echo "#define HAVE_STRDUP 1" >>confdefs.h +for ac_func in dup2 getcwd strdup +do : + as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` +ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" +eval as_val=\$$as_ac_var + if test "x$as_val" = x""yes; then : + cat >>confdefs.h <<_ACEOF +#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 +_ACEOF else case " $LIBOBJS " in - *" strdup.$ac_objext "* ) ;; - *) LIBOBJS="$LIBOBJS strdup.$ac_objext" + *" $ac_func.$ac_objext "* ) ;; + *) LIBOBJS="$LIBOBJS $ac_func.$ac_objext" ;; esac fi +done for ac_func in getpgrp @@ -11603,7 +11649,7 @@ elif test "$withval" != yes then LIBM=$withval { $as_echo "$as_me:${as_lineno-$LINENO}: result: set LIBM=\"$withval\"" >&5 $as_echo "set LIBM=\"$withval\"" >&6; } -else as_fn_error $? "proper usage is --with-libm=STRING" "$LINENO" 5 +else as_fn_error "proper usage is --with-libm=STRING" "$LINENO" 5 fi else { $as_echo "$as_me:${as_lineno-$LINENO}: result: default LIBM=\"$LIBM\"" >&5 @@ -11627,7 +11673,7 @@ elif test "$withval" != yes then LIBC=$withval { $as_echo "$as_me:${as_lineno-$LINENO}: result: set LIBC=\"$withval\"" >&5 $as_echo "set LIBC=\"$withval\"" >&6; } -else as_fn_error $? "proper usage is --with-libc=STRING" "$LINENO" 5 +else as_fn_error "proper usage is --with-libc=STRING" "$LINENO" 5 fi else { $as_echo "$as_me:${as_lineno-$LINENO}: result: default LIBC=\"$LIBC\"" >&5 @@ -11877,7 +11923,8 @@ for ac_func in acosh asinh atanh copysign erf erfc expm1 finite gamma do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" -if eval test \"x\$"$as_ac_var"\" = x"yes"; then : +eval as_val=\$$as_ac_var + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF @@ -11889,7 +11936,8 @@ for ac_func in hypot lgamma log1p log2 round tgamma do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" -if eval test \"x\$"$as_ac_var"\" = x"yes"; then : +eval as_val=\$$as_ac_var + if test "x$as_val" = x""yes; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF @@ -12152,7 +12200,7 @@ no) 15|30) ;; *) - as_fn_error $? "bad value $enable_big_digits for --enable-big-digits; value should be 15 or 30" "$LINENO" 5 ;; + as_fn_error "bad value $enable_big_digits for --enable-big-digits; value should be 15 or 30" "$LINENO" 5 ;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_big_digits" >&5 $as_echo "$enable_big_digits" >&6; } @@ -12203,8 +12251,9 @@ else if test "$ac_cv_type_wchar_t" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error 77 "cannot compute sizeof (wchar_t) -See \`config.log' for more details" "$LINENO" 5 ; } +{ as_fn_set_status 77 +as_fn_error "cannot compute sizeof (wchar_t) +See \`config.log' for more details." "$LINENO" 5; }; } else ac_cv_sizeof_wchar_t=0 fi @@ -12573,8 +12622,8 @@ $as_echo "#define AC_APPLE_UNIVERSAL_BUILD 1" >>confdefs.h ;; #( *) - as_fn_error $? "unknown endianness - presetting ac_cv_c_bigendian=no (or yes) will help" "$LINENO" 5 ;; + as_fn_error "unknown endianness + presetting ac_cv_c_bigendian=no (or yes) will help" "$LINENO" 5 ;; esac @@ -12835,7 +12884,7 @@ else have_readline=no fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext if test $have_readline = yes then cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -13009,7 +13058,7 @@ else have_readline=no fi -rm -f conftest.err conftest.i conftest.$ac_ext +rm -f conftest.err conftest.$ac_ext if test $have_readline = yes then cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -13931,7 +13980,6 @@ DEFS=-DHAVE_CONFIG_H ac_libobjs= ac_ltlibobjs= -U= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' @@ -14094,19 +14142,19 @@ export LANGUAGE (unset CDPATH) >/dev/null 2>&1 && unset CDPATH -# as_fn_error STATUS ERROR [LINENO LOG_FD] -# ---------------------------------------- +# as_fn_error ERROR [LINENO LOG_FD] +# --------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with STATUS, using 1 if that was 0. +# script with status $?, using 1 if that was 0. as_fn_error () { - as_status=$1; test $as_status -eq 0 && as_status=1 - if test "$4"; then - as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 + as_status=$?; test $as_status -eq 0 && as_status=1 + if test "$3"; then + as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 fi - $as_echo "$as_me: error: $2" >&2 + $as_echo "$as_me: error: $1" >&2 as_fn_exit $as_status } # as_fn_error @@ -14302,7 +14350,7 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" } # as_fn_mkdir_p @@ -14356,7 +14404,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # values after options handling. ac_log=" This file was extended by python $as_me 3.3, which was -generated by GNU Autoconf 2.67. Invocation command line was +generated by GNU Autoconf 2.65. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -14418,10 +14466,10 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ python config.status 3.3 -configured by $0, generated by GNU Autoconf 2.67, +configured by $0, generated by GNU Autoconf 2.65, with options \\"\$ac_cs_config\\" -Copyright (C) 2010 Free Software Foundation, Inc. +Copyright (C) 2009 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." @@ -14437,16 +14485,11 @@ ac_need_defaults=: while test $# != 0 do case $1 in - --*=?*) + --*=*) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` ac_shift=: ;; - --*=) - ac_option=`expr "X$1" : 'X\([^=]*\)='` - ac_optarg= - ac_shift=: - ;; *) ac_option=$1 ac_optarg=$2 @@ -14468,7 +14511,6 @@ do $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; - '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; @@ -14481,7 +14523,7 @@ do ac_need_defaults=false;; --he | --h) # Conflict between --help and --header - as_fn_error $? "ambiguous option: \`$1' + as_fn_error "ambiguous option: \`$1' Try \`$0 --help' for more information.";; --help | --hel | -h ) $as_echo "$ac_cs_usage"; exit ;; @@ -14490,7 +14532,7 @@ Try \`$0 --help' for more information.";; ac_cs_silent=: ;; # This is an error. - -*) as_fn_error $? "unrecognized option: \`$1' + -*) as_fn_error "unrecognized option: \`$1' Try \`$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" @@ -14549,7 +14591,7 @@ do "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Modules/ld_so_aix") CONFIG_FILES="$CONFIG_FILES Modules/ld_so_aix" ;; - *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5 ;; + *) as_fn_error "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac done @@ -14586,7 +14628,7 @@ $debug || { tmp=./conf$$-$RANDOM (umask 077 && mkdir "$tmp") -} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 +} || as_fn_error "cannot create a temporary directory in ." "$LINENO" 5 # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. @@ -14603,7 +14645,7 @@ if test "x$ac_cr" = x; then fi ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then - ac_cs_awk_cr='\\r' + ac_cs_awk_cr='\r' else ac_cs_awk_cr=$ac_cr fi @@ -14617,18 +14659,18 @@ _ACEOF echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && echo "_ACEOF" } >conf$$subs.sh || - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 -ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` + as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 +ac_delim_num=`echo "$ac_subst_vars" | grep -c '$'` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then - as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -14717,28 +14759,20 @@ if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then else cat fi < "$tmp/subs1.awk" > "$tmp/subs.awk" \ - || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 + || as_fn_error "could not setup config files machinery" "$LINENO" 5 _ACEOF -# VPATH may cause trouble with some makes, so we remove sole $(srcdir), -# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and +# VPATH may cause trouble with some makes, so we remove $(srcdir), +# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and # trailing colons and then remove the whole line if VPATH becomes empty # (actually we leave an empty line to preserve line numbers). if test "x$srcdir" = x.; then - ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ -h -s/// -s/^/:/ -s/[ ]*$/:/ -s/:\$(srcdir):/:/g -s/:\${srcdir}:/:/g -s/:@srcdir@:/:/g -s/^:*// + ac_vpsub='/^[ ]*VPATH[ ]*=/{ +s/:*\$(srcdir):*/:/ +s/:*\${srcdir}:*/:/ +s/:*@srcdir@:*/:/ +s/^\([^=]*=[ ]*\):*/\1/ s/:*$// -x -s/\(=[ ]*\).*/\1/ -G -s/\n// s/^[^=]*=[ ]*$// }' fi @@ -14766,7 +14800,7 @@ for ac_last_try in false false :; do if test -z "$ac_t"; then break elif $ac_last_try; then - as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 + as_fn_error "could not make $CONFIG_HEADERS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -14851,7 +14885,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 - as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 + as_fn_error "could not setup config headers machinery" "$LINENO" 5 fi # test -n "$CONFIG_HEADERS" @@ -14864,7 +14898,7 @@ do esac case $ac_mode$ac_tag in :[FHL]*:*);; - :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5 ;; + :L* | :C*:*) as_fn_error "invalid tag \`$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac @@ -14892,7 +14926,7 @@ do [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || - as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5 ;; + as_fn_error "cannot find input file: \`$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" @@ -14919,7 +14953,7 @@ $as_echo "$as_me: creating $ac_file" >&6;} case $ac_tag in *:-:* | *:-) cat >"$tmp/stdin" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; + || as_fn_error "could not create $ac_file" "$LINENO" 5 ;; esac ;; esac @@ -15050,22 +15084,22 @@ s&@INSTALL@&$ac_INSTALL&;t t $ac_datarootdir_hack " eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$tmp/subs.awk" >$tmp/out \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + || as_fn_error "could not create $ac_file" "$LINENO" 5 test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && { ac_out=`sed -n '/\${datarootdir}/p' "$tmp/out"`; test -n "$ac_out"; } && { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' "$tmp/out"`; test -z "$ac_out"; } && { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined" >&5 +which seems to be undefined. Please make sure it is defined." >&5 $as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined" >&2;} +which seems to be undefined. Please make sure it is defined." >&2;} rm -f "$tmp/stdin" case $ac_file in -) cat "$tmp/out" && rm -f "$tmp/out";; *) rm -f "$ac_file" && mv "$tmp/out" "$ac_file";; esac \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + || as_fn_error "could not create $ac_file" "$LINENO" 5 ;; :H) # @@ -15076,19 +15110,19 @@ which seems to be undefined. Please make sure it is defined" >&2;} $as_echo "/* $configure_input */" \ && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" } >"$tmp/config.h" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + || as_fn_error "could not create $ac_file" "$LINENO" 5 if diff "$ac_file" "$tmp/config.h" >/dev/null 2>&1; then { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 $as_echo "$as_me: $ac_file is unchanged" >&6;} else rm -f "$ac_file" mv "$tmp/config.h" "$ac_file" \ - || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + || as_fn_error "could not create $ac_file" "$LINENO" 5 fi else $as_echo "/* $configure_input */" \ && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" \ - || as_fn_error $? "could not create -" "$LINENO" 5 + || as_fn_error "could not create -" "$LINENO" 5 fi ;; @@ -15108,7 +15142,7 @@ _ACEOF ac_clean_files=$ac_clean_files_save test $ac_write_fail = 0 || - as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 + as_fn_error "write failure creating $CONFIG_STATUS" "$LINENO" 5 # configure is writing to config.log, and then calls config.status. @@ -15129,7 +15163,7 @@ if test "$no_create" != yes; then exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. - $ac_cs_success || as_fn_exit 1 + $ac_cs_success || as_fn_exit $? fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 -- cgit v0.12 From a22c98de8507b18ed0c0933f81e6b0ab63467d11 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 18:59:49 -0500 Subject: simply use the Python version for pyexpat.__version__ #12221 --- Misc/NEWS | 10 ++++++++++ Modules/pyexpat.c | 26 +++++--------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3982b2b..ed70507 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,16 @@ Python News +++++++++++ +What's New in Python 3.1.4? +=========================== + +*Release date: 2011-05-XX* + +Extension Modules +----------------- + +- Issue #12221: Replace pyexpat.__version__ with the Python version. + What's New in Python 3.1.4 release candidate 1? =============================================== diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index cc62274..7c610a8 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1730,26 +1730,6 @@ static struct PyMethodDef pyexpat_methods[] = { PyDoc_STRVAR(pyexpat_module_documentation, "Python wrapper for Expat parser."); -/* Return a Python string that represents the version number without the - * extra cruft added by revision control, even if the right options were - * given to the "cvs export" command to make it not include the extra - * cruft. - */ -static PyObject * -get_version_string(void) -{ - static char *rcsid = "$Revision$"; - char *rev = rcsid; - int i = 0; - - while (!isdigit(Py_CHARMASK(*rev))) - ++rev; - while (rev[i] != ' ' && rev[i] != '\0') - ++i; - - return PyUnicode_FromStringAndSize(rev, i); -} - /* Initialization function for the module */ #ifndef MODULE_NAME @@ -1790,6 +1770,7 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; + PyObject *version; PyObject *sys_modules; static struct PyExpat_CAPI capi; PyObject* capi_object; @@ -1822,7 +1803,10 @@ MODULE_INITFUNC(void) Py_INCREF(&Xmlparsetype); PyModule_AddObject(m, "XMLParserType", (PyObject *) &Xmlparsetype); - PyModule_AddObject(m, "__version__", get_version_string()); + version = PyUnicode_FromString(PY_VERSION); + if (!version) + return; + PyModule_AddObject(m, "__version__", version); PyModule_AddStringConstant(m, "EXPAT_VERSION", (char *) XML_ExpatVersion()); { -- cgit v0.12 From 8c6f88efa2371addacc2acf5cc4634974160406c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 20:52:17 -0500 Subject: remove __version__s dependent on subversion keyword expansion (closes #12221) --- Lib/pickle.py | 2 - Lib/pydoc.py | 1 - Lib/tarfile.py | 2 - Lib/tkinter/__init__.py | 2 - Lib/xml/parsers/expat.py | 2 - Misc/NEWS | 4 ++ Modules/pyexpat.c | 5 -- Objects/typeslots.inc | 2 +- Objects/typeslots.py | 2 +- Tools/unittestgui/unittestgui.py | 1 - configure | 128 ++++++++++++++++++++------------------- configure.in | 45 +++++++------- setup.py | 2 - 13 files changed, 94 insertions(+), 104 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index aca8fd1..fdeadee 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -23,8 +23,6 @@ Misc variables: """ -__version__ = "$Revision$" # Code version - from types import FunctionType, BuiltinFunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 548e71c..89d00a3 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -37,7 +37,6 @@ __all__ = ['help'] __author__ = "Ka-Ping Yee " __date__ = "26 February 2001" -__version__ = "$Revision$" __credits__ = """Guido van Rossum, for an excellent programming language. Tommy Burnette, the original creator of manpy. Paul Prescod, for all his work on onlinehelp. diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 6b663f4..8129a80 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -29,8 +29,6 @@ """Read from and write to tar format archives. """ -__version__ = "$Revision$" - version = "0.9.0" __author__ = "Lars Gust\u00e4bel (lars@gustaebel.de)" __date__ = "$Date$" diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index e54c53f..ee87147 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -30,8 +30,6 @@ button.pack(side=BOTTOM) tk.mainloop() """ -__version__ = "$Revision$" - import sys if sys.platform == "win32": # Attempt to configure Tcl/Tk without requiring PATH diff --git a/Lib/xml/parsers/expat.py b/Lib/xml/parsers/expat.py index a805b82..bcbe9fb 100644 --- a/Lib/xml/parsers/expat.py +++ b/Lib/xml/parsers/expat.py @@ -1,6 +1,4 @@ """Interface to the Expat non-validating XML parser.""" -__version__ = '$Revision$' - import sys from pyexpat import * diff --git a/Misc/NEWS b/Misc/NEWS index b3d21f8..63a9ded 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -184,6 +184,10 @@ Core and Builtins Library ------- +- Issue #12221: Remove __version__ attributes from pyexpat, pickle, tarfile, + pydoc, tkinter, and xml.parsers.expat. This were useless version constants + left over from the Mercurial transition + - Issue #12085: Fix an attribute error in subprocess.Popen destructor if the constructor has failed, e.g. because of an undeclared keyword argument. Patch written by Oleg Oshmyan. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 1cf699b..d923eeb 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1662,7 +1662,6 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; - PyObject *version; PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; @@ -1699,10 +1698,6 @@ MODULE_INITFUNC(void) Py_INCREF(&Xmlparsetype); PyModule_AddObject(m, "XMLParserType", (PyObject *) &Xmlparsetype); - version = PyUnicode_FromString(PY_VERSION); - if (!version) - return; - PyModule_AddObject(m, "__version__", version); PyModule_AddStringConstant(m, "EXPAT_VERSION", (char *) XML_ExpatVersion()); { diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index 5186dcf..caa1e03 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -1,4 +1,4 @@ -/* Generated by typeslots.py $Revision$ */ +/* Generated by typeslots.py */ 0, 0, offsetof(PyHeapTypeObject, as_mapping.mp_ass_subscript), diff --git a/Objects/typeslots.py b/Objects/typeslots.py index 2e00c80..b24c7f4 100644 --- a/Objects/typeslots.py +++ b/Objects/typeslots.py @@ -3,7 +3,7 @@ import sys, re -print("/* Generated by typeslots.py $Revision$ */") +print("/* Generated by typeslots.py */") res = {} for line in sys.stdin: m = re.match("#define Py_([a-z_]+) ([0-9]+)", line) diff --git a/Tools/unittestgui/unittestgui.py b/Tools/unittestgui/unittestgui.py index b526646..09a20e2 100644 --- a/Tools/unittestgui/unittestgui.py +++ b/Tools/unittestgui/unittestgui.py @@ -28,7 +28,6 @@ SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. """ __author__ = "Steve Purcell (stephen_purcell@yahoo.com)" -__version__ = "$Revision: 1.7 $"[11:-2] import sys import traceback diff --git a/configure b/configure index a3f2a3e..f9cccf2 100755 --- a/configure +++ b/configure @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Revision. +# From configure.in HGVERSION. # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.65 for python 3.3. # @@ -638,10 +638,6 @@ INSTALL_SCRIPT INSTALL_PROGRAM HAS_PYTHON DISABLE_ASDLGEN -HAS_HG -HGBRANCH -HGTAG -HGVERSION ARFLAGS AR RANLIB @@ -692,6 +688,10 @@ UNIVERSALSDK CONFIG_ARGS SOVERSION VERSION +HAS_HG +HGBRANCH +HGTAG +HGVERSION target_alias host_alias build_alias @@ -2687,6 +2687,66 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu + + + + +if test -e $srcdir/.hg/00changelog.i +then +# Extract the first word of "hg", so it can be a program name with args. +set dummy hg; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if test "${ac_cv_prog_HAS_HG+set}" = set; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$HAS_HG"; then + ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then + ac_cv_prog_HAS_HG="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" +fi +fi +HAS_HG=$ac_cv_prog_HAS_HG +if test -n "$HAS_HG"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 +$as_echo "$HAS_HG" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +else +HAS_HG=no-repository +fi +if test $HAS_HG = found +then + HGVERSION="hg id -i \$(srcdir)" + HGTAG="hg id -t \$(srcdir)" + HGBRANCH="hg id -b \$(srcdir)" +else + HGVERSION="" + HGTAG="" + HGBRANCH="" +fi + + + ac_config_headers="$ac_config_headers pyconfig.h" @@ -5139,64 +5199,6 @@ then fi - - - -if test -e $srcdir/.hg/00changelog.i -then -# Extract the first word of "hg", so it can be a program name with args. -set dummy hg; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_HAS_HG+set}" = set; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$HAS_HG"; then - ac_cv_prog_HAS_HG="$HAS_HG" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then - ac_cv_prog_HAS_HG="found" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - - test -z "$ac_cv_prog_HAS_HG" && ac_cv_prog_HAS_HG="not-found" -fi -fi -HAS_HG=$ac_cv_prog_HAS_HG -if test -n "$HAS_HG"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_HG" >&5 -$as_echo "$HAS_HG" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -else -HAS_HG=no-repository -fi -if test $HAS_HG = found -then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" -else - HGVERSION="" - HGTAG="" - HGBRANCH="" -fi - - DISABLE_ASDLGEN="" # Extract the first word of "python", so it can be a program name with args. set dummy python; ac_word=$2 diff --git a/configure.in b/configure.in index 49932de..71f488d 100644 --- a/configure.in +++ b/configure.in @@ -7,8 +7,30 @@ m4_define(PYTHON_VERSION, 3.3) AC_PREREQ(2.65) -AC_REVISION($Revision$) AC_INIT(python, PYTHON_VERSION, http://bugs.python.org/) + +AC_SUBST(HGVERSION) +AC_SUBST(HGTAG) +AC_SUBST(HGBRANCH) + +if test -e $srcdir/.hg/00changelog.i +then +AC_CHECK_PROG(HAS_HG, hg, found, not-found) +else +HAS_HG=no-repository +fi +if test $HAS_HG = found +then + HGVERSION="hg id -i \$(srcdir)" + HGTAG="hg id -t \$(srcdir)" + HGBRANCH="hg id -b \$(srcdir)" +else + HGVERSION="" + HGTAG="" + HGBRANCH="" +fi + +AC_REVISION(HGVERSION) AC_CONFIG_SRCDIR([Include/object.h]) AC_CONFIG_HEADER(pyconfig.h) @@ -808,27 +830,6 @@ then ARFLAGS="rc" fi -AC_SUBST(HGVERSION) -AC_SUBST(HGTAG) -AC_SUBST(HGBRANCH) - -if test -e $srcdir/.hg/00changelog.i -then -AC_CHECK_PROG(HAS_HG, hg, found, not-found) -else -HAS_HG=no-repository -fi -if test $HAS_HG = found -then - HGVERSION="hg id -i \$(srcdir)" - HGTAG="hg id -t \$(srcdir)" - HGBRANCH="hg id -b \$(srcdir)" -else - HGVERSION="" - HGTAG="" - HGBRANCH="" -fi - AC_SUBST(DISABLE_ASDLGEN) DISABLE_ASDLGEN="" AC_CHECK_PROG(HAS_PYTHON, python, found, not-found) diff --git a/setup.py b/setup.py index fa42191..896d604 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,6 @@ # Autodetecting setup.py script for building the Python extensions # -__version__ = "$Revision$" - import sys, os, imp, re, optparse from glob import glob import sysconfig -- cgit v0.12 From 2e3a38a77468661f19d228977aa8fa2616d14a6a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 21:27:41 -0500 Subject: demote this to a note --- Doc/c-api/weakref.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 6b053c8..6cb3e33 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -55,7 +55,7 @@ as much as it can. Return the referenced object from a weak reference, *ref*. If the referent is no longer live, returns :const:`Py_None`. - .. warning:: + .. note:: This function returns a **borrowed reference** to the referenced object. This means that you should always call :c:func:`Py_INCREF` on the object -- cgit v0.12 From 5c2b09e856555e34128113ac99758a776f66ce3c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 21:31:37 -0500 Subject: be extra careful with a borrowed reference when the GIL could be released (closes #8578) --- Modules/_sqlite/connection.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index e4969e3..7d12d5e 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -200,11 +200,13 @@ void pysqlite_do_all_statements(pysqlite_Connection* self, int action, int reset weakref = PyList_GetItem(self->statements, i); statement = PyWeakref_GetObject(weakref); if (statement != Py_None) { + Py_INCREF(statement); if (action == ACTION_RESET) { (void)pysqlite_statement_reset((pysqlite_Statement*)statement); } else { (void)pysqlite_statement_finalize((pysqlite_Statement*)statement); } + Py_DECREF(statement); } } -- cgit v0.12 From 52e61449e39b6778d907c65fb2931a2d37c7eef2 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 31 May 2011 21:38:15 -0500 Subject: return NULL on error --- Modules/pyexpat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 7c610a8..4e806e2 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1805,7 +1805,7 @@ MODULE_INITFUNC(void) version = PyUnicode_FromString(PY_VERSION); if (!version) - return; + return NULL; PyModule_AddObject(m, "__version__", version); PyModule_AddStringConstant(m, "EXPAT_VERSION", (char *) XML_ExpatVersion()); -- cgit v0.12 From fce9233e93cb217a31d8c5b1776b4ec226595dc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 Jun 2011 12:28:04 +0200 Subject: test.support: add requires_mac_ver() function Add also linux_version() to __all__. --- Lib/test/support.py | 22 +++++++++++++++++++++- Lib/test/test_math.py | 7 ++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Lib/test/support.py b/Lib/test/support.py index 6724e9b..1bf9ca5 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -37,7 +37,8 @@ __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "find_unused_port", "bind_port", + "is_resource_enabled", "requires", "linux_version", "requires_mac_ver", + "find_unused_port", "bind_port", "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd", "findfile", "sortdict", "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource", @@ -299,6 +300,25 @@ def linux_version(): except ValueError: return 0, 0, 0 +def requires_mac_ver(*min_version): + """Raise SkipTest if the OS is Mac OS X and the OS X version if less than + min_version. + + For example, support.requires_linux_version(10, 5) raises SkipTest if the + version is less than 10.5. + """ + if sys.platform != 'darwin': + return + version_txt = platform.mac_ver()[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + return + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest("Mac OS X %s or higher required, not %s" + % (min_version_txt, version_txt)) + HOST = 'localhost' def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 5b914d5..636ff2c 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2,6 +2,7 @@ # XXXX Should not do tests around zero only from test.support import run_unittest, verbose, requires_IEEE_754 +from test import support import unittest import math import os @@ -669,10 +670,10 @@ class MathTests(unittest.TestCase): self.assertTrue(math.isnan(math.log2(NAN))) @requires_IEEE_754 - @unittest.skipIf(sys.platform == 'darwin' - and platform.mac_ver()[0].startswith('10.4.'), - 'Mac OS X Tiger log2() is not accurate enough') def testLog2Exact(self): + # log2() is not accurate enough on Mac OS X Tiger (10.4) + support.requires_mac_ver(10, 5) + # Check that we get exact equality for log2 of powers of 2. actual = [math.log2(math.ldexp(1.0, n)) for n in range(-1074, 1024)] expected = [float(n) for n in range(-1074, 1024)] -- cgit v0.12 From 88701e27e90c0b70e2b22b06ad8bbfa231cf99dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 Jun 2011 13:13:04 +0200 Subject: Close #12230: Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file descriptor of a pipe closed in the parent process is valid in the child process according to fstat(), but the mode of the file descriptor is invalid, and read or write raise an error. Add also requires_mac_ver() decorator to test.support. --- Lib/test/support.py | 30 +++++++++++++++++++++++++++++- Lib/test/test_subprocess.py | 5 +++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Lib/test/support.py b/Lib/test/support.py index 0cc8c31..2aedf24 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -32,7 +32,8 @@ __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "find_unused_port", "bind_port", + "is_resource_enabled", "requires", "requires_mac_ver", + "find_unused_port", "bind_port", "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", "findfile", "sortdict", "check_syntax_error", "open_urlresource", "check_warnings", "CleanImport", "EnvironmentVarGuard", @@ -288,6 +289,33 @@ def requires(resource, msg=None): msg = "Use of the `%s' resource not enabled" % resource raise ResourceDenied(msg) +def requires_mac_ver(*min_version): + """Decorator raising SkipTest if the OS is Mac OS X and the OS X + version if less than min_version. + + For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version + is lesser than 10.5. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if sys.platform == 'darwin': + version_txt = platform.mac_ver()[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest( + "Mac OS X %s or higher required, not %s" + % (min_version_txt, version_txt)) + return func(*args, **kw) + wrapper.min_version = min_version + return wrapper + return decorator + HOST = 'localhost' def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 9d66659..e6e8b3a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1185,6 +1185,11 @@ class POSIXProcessTestCase(BaseTestCase): "Some fds were left open") self.assertIn(1, remaining_fds, "Subprocess failed") + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) def test_pass_fds(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") -- cgit v0.12 From 301f3f093c463b8eba8f55d6cb3eecc0893d5cfa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 Jun 2011 13:49:12 +0200 Subject: test_faulthandler: use a longer sleep (2.5 sec instead of 1.3 sec) in _check_dump_tracebacks_later() to avoid failures if the system is slow. --- Lib/test/test_faulthandler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index dbc1917..0ddfc7f 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -29,13 +29,14 @@ else: except (ValueError, resource_error): pass -def expected_traceback(lineno1, lineno2, header, count=1): +def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header regex += ' File "", line %s in func\n' % lineno1 regex += ' File "", line %s in ' % lineno2 - if count != 1: - regex = (regex + '\n') * (count - 1) + regex - return '^' + regex + '$' + if 1 < min_count: + return '^' + (regex + '\n') * (min_count - 1) + regex + else: + return '^' + regex + '$' @contextmanager def temporary_filename(): @@ -371,7 +372,7 @@ def func(timeout, repeat, cancel, file, loops): faulthandler.dump_tracebacks_later(timeout, repeat=repeat, file=file) if cancel: faulthandler.cancel_dump_tracebacks_later() - time.sleep(timeout * 2.5) + time.sleep(timeout * 5) faulthandler.cancel_dump_tracebacks_later() timeout = {timeout} @@ -402,7 +403,7 @@ if file is not None: if repeat: count *= 2 header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+:\n' % timeout_str - regex = expected_traceback(9, 20, header, count=count) + regex = expected_traceback(9, 20, header, min_count=count) self.assertRegex(trace, regex) else: self.assertEqual(trace, '') -- cgit v0.12 From 25d5737f1583fc3844fd27f37cacb673937f7af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 1 Jun 2011 14:41:11 +0200 Subject: Fix typos in class names --- Lib/packaging/pypi/simple.py | 4 ++-- Lib/packaging/pypi/xmlrpc.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/packaging/pypi/simple.py b/Lib/packaging/pypi/simple.py index 983d477..c492179 100644 --- a/Lib/packaging/pypi/simple.py +++ b/Lib/packaging/pypi/simple.py @@ -1,6 +1,6 @@ """Spider using the screen-scraping "simple" PyPI API. -This module contains the class SimpleIndexCrawler, a simple spider that +This module contains the class Crawler, a simple spider that can be used to find and retrieve distributions from a project index (like the Python Package Index), using its so-called simple API (see reference implementation available at http://pypi.python.org/simple/). @@ -178,7 +178,7 @@ class Crawler(BaseClient): def get_releases(self, requirements, prefer_final=None, force_update=False): - """Search for releases and return a ReleaseList object containing + """Search for releases and return a ReleasesList object containing the results. """ predicate = get_version_predicate(requirements) diff --git a/Lib/packaging/pypi/xmlrpc.py b/Lib/packaging/pypi/xmlrpc.py index 7a9f6cc..befdf6d 100644 --- a/Lib/packaging/pypi/xmlrpc.py +++ b/Lib/packaging/pypi/xmlrpc.py @@ -31,11 +31,11 @@ class Client(BaseClient): If no server_url is specified, use the default PyPI XML-RPC URL, defined in the DEFAULT_XMLRPC_INDEX_URL constant:: - >>> client = XMLRPCClient() + >>> client = Client() >>> client.server_url == DEFAULT_XMLRPC_INDEX_URL True - >>> client = XMLRPCClient("http://someurl/") + >>> client = Client("http://someurl/") >>> client.server_url 'http://someurl/' """ @@ -69,7 +69,7 @@ class Client(BaseClient): informations (eg. make a new XML-RPC call). :: - >>> client = XMLRPCClient() + >>> client = Client() >>> client.get_releases('Foo') ['1.1', '1.2', '1.3'] @@ -189,7 +189,7 @@ class Client(BaseClient): If no server proxy is defined yet, creates a new one:: - >>> client = XmlRpcClient() + >>> client = Client() >>> client.proxy() -- cgit v0.12 From 25987d0c4be3cc8e332879d830ef112f930e01b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 1 Jun 2011 15:20:44 +0200 Subject: The compiler class for EMX was removed --- Lib/packaging/compiler/__init__.py | 5 +---- Lib/packaging/compiler/ccompiler.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/packaging/compiler/__init__.py b/Lib/packaging/compiler/__init__.py index 34315aa..e267e9f 100644 --- a/Lib/packaging/compiler/__init__.py +++ b/Lib/packaging/compiler/__init__.py @@ -83,19 +83,16 @@ def customize_compiler(compiler): # patterns. Order is important; platform mappings are preferred over # OS names. _default_compilers = ( - # Platform string mappings # on a cygwin built python we can use gcc like an ordinary UNIXish # compiler ('cygwin.*', 'unix'), - ('os2emx', 'emx'), # OS name mappings ('posix', 'unix'), ('nt', 'msvc'), - - ) +) def get_default_compiler(osname=None, platform=None): """ Determine the default compiler to use for the given platform. diff --git a/Lib/packaging/compiler/ccompiler.py b/Lib/packaging/compiler/ccompiler.py index ef806a2..d274327 100644 --- a/Lib/packaging/compiler/ccompiler.py +++ b/Lib/packaging/compiler/ccompiler.py @@ -352,7 +352,7 @@ class CCompiler: return macros, objects, extra, pp_opts, build def _get_cc_args(self, pp_opts, debug, before): - # works for unixccompiler, emxccompiler, cygwinccompiler + # works for unixccompiler and cygwinccompiler cc_args = pp_opts + ['-c'] if debug: cc_args[:0] = ['-g'] -- cgit v0.12 From c0725162d581f902ebe60121851fb869973a8d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 1 Jun 2011 19:41:21 +0200 Subject: Remove duplicate entry for Josip --- Misc/ACKS | 1 - 1 file changed, 1 deletion(-) diff --git a/Misc/ACKS b/Misc/ACKS index e4aa789..08fc572 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -255,7 +255,6 @@ Virgil Dupras Andy Dustman Gary Duzan Eugene Dvurechenski -Josip Dzolonga Maxim Dzumanenko Walter Dörwald Hans Eckardt -- cgit v0.12 From a003af1ce9d008e03371b3d16c4d6361961c2e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 1 Jun 2011 20:30:52 +0200 Subject: Issue #12196: Add a note on os.pipe2() in the "Whats' new in Python 3.3" document. --- Doc/whatsnew/3.3.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 2b9bd11..a2d3af1 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -106,6 +106,11 @@ connection when done:: os -- +* The :mod:`os` module has a new :func:`~os.pipe2` function that makes it + possible to create a pipe with :data:`~os.O_CLOEXEC` or + :data:`~os.O_NONBLOCK` flags set atomically. This is especially useful to + avoid race conditions in multi-threaded programs. + * The :mod:`os` module has a new :func:`~os.sendfile` function which provides an efficent "zero-copy" way for copying data from one file (or socket) descriptor to another. The phrase "zero-copy" refers to the fact that all of -- cgit v0.12 From 3a9f58f6b3938823328374f34a3b52a167fed871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 1 Jun 2011 20:42:49 +0200 Subject: Add documentation for the packaging module. This updates the user guide to refer to Packaging instead of Distutils. Some files still require an update. --- Doc/contents.rst | 2 +- Doc/distutils/index.rst | 13 + Doc/distutils/install.rst | 1005 ++++++++++++++++++++++++++ Doc/install/index.rst | 1005 -------------------------- Doc/install/install.rst | 1029 +++++++++++++++++++++++++++ Doc/install/pysetup-config.rst | 44 ++ Doc/install/pysetup-servers.rst | 61 ++ Doc/install/pysetup.rst | 163 +++++ Doc/library/depgraph-output.png | Bin 0 -> 24719 bytes Doc/library/distutils.rst | 5 + Doc/library/packaging-misc.rst | 27 + Doc/library/packaging.command.rst | 111 +++ Doc/library/packaging.compiler.rst | 672 +++++++++++++++++ Doc/library/packaging.database.rst | 324 +++++++++ Doc/library/packaging.depgraph.rst | 199 ++++++ Doc/library/packaging.dist.rst | 102 +++ Doc/library/packaging.fancy_getopt.rst | 75 ++ Doc/library/packaging.install.rst | 112 +++ Doc/library/packaging.metadata.rst | 122 ++++ Doc/library/packaging.pypi.dist.rst | 114 +++ Doc/library/packaging.pypi.rst | 53 ++ Doc/library/packaging.pypi.simple.rst | 157 ++++ Doc/library/packaging.pypi.xmlrpc.rst | 143 ++++ Doc/library/packaging.rst | 78 ++ Doc/library/packaging.tests.pypi_server.rst | 105 +++ Doc/library/packaging.util.rst | 186 +++++ Doc/library/packaging.version.rst | 104 +++ Doc/library/python.rst | 1 + Doc/packaging/builtdist.rst | 307 ++++++++ Doc/packaging/commandhooks.rst | 31 + Doc/packaging/commandref.rst | 349 +++++++++ Doc/packaging/configfile.rst | 125 ++++ Doc/packaging/examples.rst | 334 +++++++++ Doc/packaging/extending.rst | 95 +++ Doc/packaging/index.rst | 45 ++ Doc/packaging/introduction.rst | 193 +++++ Doc/packaging/packageindex.rst | 104 +++ Doc/packaging/setupcfg.rst | 648 +++++++++++++++++ Doc/packaging/setupscript.rst | 689 ++++++++++++++++++ Doc/packaging/sourcedist.rst | 273 +++++++ Doc/packaging/tutorial.rst | 112 +++ Doc/packaging/uploading.rst | 80 +++ Doc/tools/sphinxext/indexcontent.html | 4 +- 43 files changed, 8393 insertions(+), 1008 deletions(-) create mode 100644 Doc/distutils/install.rst delete mode 100644 Doc/install/index.rst create mode 100644 Doc/install/install.rst create mode 100644 Doc/install/pysetup-config.rst create mode 100644 Doc/install/pysetup-servers.rst create mode 100644 Doc/install/pysetup.rst create mode 100644 Doc/library/depgraph-output.png create mode 100644 Doc/library/packaging-misc.rst create mode 100644 Doc/library/packaging.command.rst create mode 100644 Doc/library/packaging.compiler.rst create mode 100644 Doc/library/packaging.database.rst create mode 100644 Doc/library/packaging.depgraph.rst create mode 100644 Doc/library/packaging.dist.rst create mode 100644 Doc/library/packaging.fancy_getopt.rst create mode 100644 Doc/library/packaging.install.rst create mode 100644 Doc/library/packaging.metadata.rst create mode 100644 Doc/library/packaging.pypi.dist.rst create mode 100644 Doc/library/packaging.pypi.rst create mode 100644 Doc/library/packaging.pypi.simple.rst create mode 100644 Doc/library/packaging.pypi.xmlrpc.rst create mode 100644 Doc/library/packaging.rst create mode 100644 Doc/library/packaging.tests.pypi_server.rst create mode 100644 Doc/library/packaging.util.rst create mode 100644 Doc/library/packaging.version.rst create mode 100644 Doc/packaging/builtdist.rst create mode 100644 Doc/packaging/commandhooks.rst create mode 100644 Doc/packaging/commandref.rst create mode 100644 Doc/packaging/configfile.rst create mode 100644 Doc/packaging/examples.rst create mode 100644 Doc/packaging/extending.rst create mode 100644 Doc/packaging/index.rst create mode 100644 Doc/packaging/introduction.rst create mode 100644 Doc/packaging/packageindex.rst create mode 100644 Doc/packaging/setupcfg.rst create mode 100644 Doc/packaging/setupscript.rst create mode 100644 Doc/packaging/sourcedist.rst create mode 100644 Doc/packaging/tutorial.rst create mode 100644 Doc/packaging/uploading.rst diff --git a/Doc/contents.rst b/Doc/contents.rst index e938fcd..e9d1771 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,7 @@ library/index.rst extending/index.rst c-api/index.rst - distutils/index.rst + packaging/index.rst install/index.rst documenting/index.rst howto/index.rst diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index ace8280..5fa25a6 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -29,3 +29,16 @@ very little overhead for build/release/install mechanics. extending.rst commandref.rst apiref.rst + +Another document describes how to install modules and extensions packaged +following the above guidelines: + +.. toctree:: + + install.rst + + +.. seealso:: + + :ref:`packaging-index` and :ref:`packaging-install-index` + Documentation of Packaging, the new version of Distutils. diff --git a/Doc/distutils/install.rst b/Doc/distutils/install.rst new file mode 100644 index 0000000..31c1d7f --- /dev/null +++ b/Doc/distutils/install.rst @@ -0,0 +1,1005 @@ +.. highlightlang:: none + +.. _install-index: + +***************************** + Installing Python Modules +***************************** + +:Author: Greg Ward +:Release: |version| +:Date: |today| + +.. TODO: Fill in XXX comments + +.. The audience for this document includes people who don't know anything + about Python and aren't about to learn the language just in order to + install and maintain it for their users, i.e. system administrators. + Thus, I have to be sure to explain the basics at some point: + sys.path and PYTHONPATH at least. Should probably give pointers to + other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. + + Finally, it might be useful to include all the material from my "Care + and Feeding of a Python Installation" talk in here somewhere. Yow! + +.. topic:: Abstract + + This document describes the Python Distribution Utilities ("Distutils") from the + end-user's point-of-view, describing how to extend the capabilities of a + standard Python installation by building and installing third-party Python + modules and extensions. + + +.. _inst-intro: + +Introduction +============ + +Although Python's extensive standard library covers many programming needs, +there often comes a time when you need to add some new functionality to your +Python installation in the form of third-party modules. This might be necessary +to support your own programming, or to support an application that you want to +use and that happens to be written in Python. + +In the past, there has been little support for adding third-party modules to an +existing Python installation. With the introduction of the Python Distribution +Utilities (Distutils for short) in Python 2.0, this changed. + +This document is aimed primarily at the people who need to install third-party +Python modules: end-users and system administrators who just need to get some +Python application running, and existing Python programmers who want to add some +new goodies to their toolbox. You don't need to know Python to read this +document; there will be some brief forays into using Python's interactive mode +to explore your installation, but that's it. If you're looking for information +on how to distribute your own Python modules so that others may use them, see +the :ref:`distutils-index` manual. + + +.. _inst-trivial-install: + +Best case: trivial installation +------------------------------- + +In the best case, someone will have prepared a special version of the module +distribution you want to install that is targeted specifically at your platform +and is installed just like any other software on your platform. For example, +the module developer might make an executable installer available for Windows +users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, +Mandrake, and many others), a Debian package for users of Debian-based Linux +systems, and so forth. + +In that case, you would download the installer appropriate to your platform and +do the obvious thing with it: run it if it's an executable installer, ``rpm +--install`` it if it's an RPM, etc. You don't need to run Python or a setup +script, you don't need to compile anything---you might not even need to read any +instructions (although it's always a good idea to do so anyways). + +Of course, things will not always be that easy. You might be interested in a +module distribution that doesn't have an easy-to-use installer for your +platform. In that case, you'll have to start with the source distribution +released by the module's author/maintainer. Installing from a source +distribution is not too hard, as long as the modules are packaged in the +standard way. The bulk of this document is about building and installing +modules from standard source distributions. + + +.. _inst-new-standard: + +The new standard: Distutils +--------------------------- + +If you download a module source distribution, you can tell pretty quickly if it +was packaged and distributed in the standard way, i.e. using the Distutils. +First, the distribution's name and version number will be featured prominently +in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or +:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named +directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the +distribution will contain a setup script :file:`setup.py`, and a file named +:file:`README.txt` or possibly just :file:`README`, which should explain that +building and installing the module distribution is a simple matter of running :: + + python setup.py install + +If all these things are true, then you already know how to build and install the +modules you've just downloaded: Run the command above. Unless you need to +install things in a non-standard way or customize the build process, you don't +really need this manual. Or rather, the above command is everything you need to +get out of this manual. + + +.. _inst-standard-install: + +Standard Build and Install +========================== + +As described in section :ref:`inst-new-standard`, building and installing a module +distribution using the Distutils is usually one simple command:: + + python setup.py install + +On Unix, you'd run this command from a shell prompt; on Windows, you have to +open a command prompt window ("DOS box") and do it there; on Mac OS X, you open +a :command:`Terminal` window to get a shell prompt. + + +.. _inst-platform-variations: + +Platform variations +------------------- + +You should always run the setup command from the distribution root directory, +i.e. the top-level subdirectory that the module source distribution unpacks +into. For example, if you've just downloaded a module source distribution +:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: + + gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 + cd foo-1.0 + python setup.py install + +On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the +archive file to :file:`C:\\Temp`, then it would unpack into +:file:`C:\\Temp\\foo-1.0`; you can use either a archive manipulator with a +graphical user interface (such as WinZip) or a command-line tool (such as +:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a +command prompt window ("DOS box"), and run:: + + cd c:\Temp\foo-1.0 + python setup.py install + + +.. _inst-splitting-up: + +Splitting the job up +-------------------- + +Running ``setup.py install`` builds and installs all modules in one run. If you +prefer to work incrementally---especially useful if you want to customize the +build process, or if things are going wrong---you can use the setup script to do +one thing at a time. This is particularly helpful when the build and install +will be done by different users---for example, you might want to build a module +distribution and hand it off to a system administrator for installation (or do +it yourself, with super-user privileges). + +For example, you can build everything in one step, and then install everything +in a second step, by invoking the setup script twice:: + + python setup.py build + python setup.py install + +If you do this, you will notice that running the :command:`install` command +first runs the :command:`build` command, which---in this case---quickly notices +that it has nothing to do, since everything in the :file:`build` directory is +up-to-date. + +You may not need this ability to break things down often if all you do is +install modules downloaded off the 'net, but it's very handy for more advanced +tasks. If you get into distributing your own Python modules and extensions, +you'll run lots of individual Distutils commands on their own. + + +.. _inst-how-build-works: + +How building works +------------------ + +As implied above, the :command:`build` command is responsible for putting the +files to install into a *build directory*. By default, this is :file:`build` +under the distribution root; if you're excessively concerned with speed, or want +to keep the source tree pristine, you can change the build directory with the +:option:`--build-base` option. For example:: + + python setup.py build --build-base=/tmp/pybuild/foo-1.0 + +(Or you could do this permanently with a directive in your system or personal +Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this +isn't necessary. + +The default layout for the build tree is as follows:: + + --- build/ --- lib/ + or + --- build/ --- lib./ + temp./ + +where ```` expands to a brief description of the current OS/hardware +platform and Python version. The first form, with just a :file:`lib` directory, +is used for "pure module distributions"---that is, module distributions that +include only pure Python modules. If a module distribution contains any +extensions (modules written in C/C++), then the second form, with two ```` +directories, is used. In that case, the :file:`temp.{plat}` directory holds +temporary files generated by the compile/link process that don't actually get +installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory +contains all Python modules (pure Python and extensions) that will be installed. + +In the future, more directories will be added to handle Python scripts, +documentation, binary executables, and whatever else is needed to handle the job +of installing Python modules and applications. + + +.. _inst-how-install-works: + +How installation works +---------------------- + +After the :command:`build` command runs (whether you run it explicitly, or the +:command:`install` command does it for you), the work of the :command:`install` +command is relatively simple: all it has to do is copy everything under +:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation +directory. + +If you don't choose an installation directory---i.e., if you just run ``setup.py +install``\ ---then the :command:`install` command installs to the standard +location for third-party Python modules. This location varies by platform and +by how you built/installed Python itself. On Unix (and Mac OS X, which is also +Unix-based), it also depends on whether the module distribution being installed +is pure Python or contains extensions ("non-pure"): + ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Platform | Standard installation location | Default value | Notes | ++=================+=====================================================+==================================================+=======+ +| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ + +Notes: + +(1) + Most Linux distributions include Python as a standard part of the system, so + :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on + Linux. If you build Python yourself on Linux (or any Unix-like system), the + default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. + +(2) + The default installation directory on Windows was :file:`C:\\Program + Files\\Python` under Python 1.6a1, 1.5.2, and earlier. + +:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python +is installed to, and where it finds its libraries at run-time. They are always +the same under Windows, and very often the same under Unix and Mac OS X. You +can find out what your Python installation uses for :file:`{prefix}` and +:file:`{exec-prefix}` by running Python in interactive mode and typing a few +simple commands. Under Unix, just type ``python`` at the shell prompt. Under +Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> +Python (command line)`. Once the interpreter is started, you type Python code +at the prompt. For example, on my Linux system, I type the three Python +statements shown below, and get the output as shown, to find out my +:file:`{prefix}` and :file:`{exec-prefix}`:: + + Python 2.4 (#26, Aug 7 2004, 17:19:02) + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.prefix + '/usr' + >>> sys.exec_prefix + '/usr' + +If you don't want to install modules to the standard location, or if you don't +have permission to write there, then you need to read about alternate +installations in section :ref:`inst-alt-install`. If you want to customize your +installation directories more heavily, see section :ref:`inst-custom-install` on +custom installations. + + +.. _inst-alt-install: + +Alternate Installation +====================== + +Often, it is necessary or desirable to install modules to a location other than +the standard location for third-party Python modules. For example, on a Unix +system you might not have permission to write to the standard third-party module +directory. Or you might wish to try out a module before making it a standard +part of your local Python installation. This is especially true when upgrading +a distribution already present: you want to make sure your existing base of +scripts still works with the new version before actually upgrading. + +The Distutils :command:`install` command is designed to make installing module +distributions to an alternate location simple and painless. The basic idea is +that you supply a base directory for the installation, and the +:command:`install` command picks a set of directories (called an *installation +scheme*) under this base directory in which to install files. The details +differ across platforms, so read whichever of the following sections applies to +you. + + +.. _inst-alt-install-prefix: + +Alternate installation: the home scheme +--------------------------------------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the idea of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +This scheme can be used by anyone, regardless of the operating system they +are installing for. + +Installing a new module distribution is as simple as :: + + python setup.py install --home=

+ +where you can supply any directory you like for the :option:`--home` option. On +Unix, lazy typists can just type a tilde (``~``); the :command:`install` command +will expand this to your home directory:: + + python setup.py install --home=~ + +The :option:`--home` option defines the installation base directory. Files are +installed to the following directories under the installation base as follows: + ++------------------------------+---------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+===========================+=============================+ +| pure module distribution | :file:`{home}/lib/python` | :option:`--install-purelib` | ++------------------------------+---------------------------+-----------------------------+ +| non-pure module distribution | :file:`{home}/lib/python` | :option:`--install-platlib` | ++------------------------------+---------------------------+-----------------------------+ +| scripts | :file:`{home}/bin` | :option:`--install-scripts` | ++------------------------------+---------------------------+-----------------------------+ +| data | :file:`{home}/share` | :option:`--install-data` | ++------------------------------+---------------------------+-----------------------------+ + + +.. _inst-alt-install-home: + +Alternate installation: Unix (the prefix scheme) +------------------------------------------------ + +The "prefix scheme" is useful when you wish to use one Python installation to +perform the build/install (i.e., to run the setup script), but install modules +into the third-party module directory of a different Python installation (or +something that looks like a different Python installation). If this sounds a +trifle unusual, it is---that's why the "home scheme" comes first. However, +there are at least two known cases where the prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. This can be done with :: + + /usr/bin/python setup.py install --prefix=/usr/local + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could +be done with :: + + /usr/local/bin/python setup.py install --prefix=/mnt/@server/export + +In either case, the :option:`--prefix` option defines the installation base, and +the :option:`--exec-prefix` option defines the platform-specific installation +base, which is used for platform-specific files. (Currently, this just means +non-pure module distributions, but could be expanded to C libraries, binary +executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to +:option:`--prefix`. Files are installed as follows: + ++------------------------------+-----------------------------------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+=====================================================+=============================+ +| pure module distribution | :file:`{prefix}/lib/python{X.Y}/site-packages` | :option:`--install-purelib` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| non-pure module distribution | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :option:`--install-platlib` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| scripts | :file:`{prefix}/bin` | :option:`--install-scripts` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| data | :file:`{prefix}/share` | :option:`--install-data` | ++------------------------------+-----------------------------------------------------+-----------------------------+ + +There is no requirement that :option:`--prefix` or :option:`--exec-prefix` +actually point to an alternate Python installation; if the directories listed +above do not already exist, they are created at installation time. + +Incidentally, the real reason the prefix scheme is important is simply that a +standard Unix installation uses the prefix scheme, but with :option:`--prefix` +and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and +``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, +but every time you run ``python setup.py install`` without any other options, +you're using it. + +Note that installing extensions to an alternate Python installation has no +effect on how those extensions are built: in particular, the Python header files +(:file:`Python.h` and friends) installed with the Python interpreter used to run +the setup script will be used in compiling extensions. It is your +responsibility to ensure that the interpreter used to run extensions installed +in this way is compatible with the interpreter used to build them. The best way +to do this is to ensure that the two interpreters are the same version of Python +(possibly different builds, or possibly copies of the same build). (Of course, +if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an +alternate Python installation, this is immaterial.) + + +.. _inst-alt-install-windows: + +Alternate installation: Windows (the prefix scheme) +--------------------------------------------------- + +Windows has no concept of a user's home directory, and since the standard Python +installation under Windows is simpler than under Unix, the :option:`--prefix` +option has traditionally been used to install additional packages in separate +locations on Windows. :: + + python setup.py install --prefix="\Temp\Python" + +to install modules to the :file:`\\Temp\\Python` directory on the current drive. + +The installation base is defined by the :option:`--prefix` option; the +:option:`--exec-prefix` option is not supported under Windows. Files are +installed as follows: + ++------------------------------+---------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+===========================+=============================+ +| pure module distribution | :file:`{prefix}` | :option:`--install-purelib` | ++------------------------------+---------------------------+-----------------------------+ +| non-pure module distribution | :file:`{prefix}` | :option:`--install-platlib` | ++------------------------------+---------------------------+-----------------------------+ +| scripts | :file:`{prefix}\\Scripts` | :option:`--install-scripts` | ++------------------------------+---------------------------+-----------------------------+ +| data | :file:`{prefix}\\Data` | :option:`--install-data` | ++------------------------------+---------------------------+-----------------------------+ + + +.. _inst-custom-install: + +Custom Installation +=================== + +Sometimes, the alternate installation schemes described in section +:ref:`inst-alt-install` just don't do what you want. You might want to tweak just +one or two directories while keeping everything under the same base directory, +or you might want to completely redefine the installation scheme. In either +case, you're creating a *custom installation scheme*. + +You probably noticed the column of "override options" in the tables describing +the alternate installation schemes above. Those options are how you define a +custom installation scheme. These override options can be relative, absolute, +or explicitly defined in terms of one of the installation base directories. +(There are two installation base directories, and they are normally the same--- +they only differ when you use the Unix "prefix scheme" and supply different +:option:`--prefix` and :option:`--exec-prefix` options.) + +For example, say you're installing a module distribution to your home directory +under Unix---but you want scripts to go in :file:`~/scripts` rather than +:file:`~/bin`. As you might expect, you can override this directory with the +:option:`--install-scripts` option; in this case, it makes most sense to supply +a relative path, which will be interpreted relative to the installation base +directory (your home directory, in this case):: + + python setup.py install --home=~ --install-scripts=scripts + +Another Unix example: suppose your Python installation was built and installed +with a prefix of :file:`/usr/local/python`, so under a standard installation +scripts will wind up in :file:`/usr/local/python/bin`. If you want them in +:file:`/usr/local/bin` instead, you would supply this absolute directory for the +:option:`--install-scripts` option:: + + python setup.py install --install-scripts=/usr/local/bin + +(This performs an installation using the "prefix scheme," where the prefix is +whatever your Python interpreter was installed with--- :file:`/usr/local/python` +in this case.) + +If you maintain Python on Windows, you might want third-party modules to live in +a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` +itself. This is almost as easy as customizing the script installation directory +---you just have to remember that there are two types of modules to worry about, +pure modules and non-pure modules (i.e., modules from a non-pure distribution). +For example:: + + python setup.py install --install-purelib=Site --install-platlib=Site + +The specified installation directories are relative to :file:`{prefix}`. Of +course, you also have to ensure that these directories are in Python's module +search path, such as by putting a :file:`.pth` file in :file:`{prefix}`. See +section :ref:`inst-search-path` to find out how to modify Python's search path. + +If you want to define an entire installation scheme, you just have to supply all +of the installation directory options. The recommended way to do this is to +supply relative paths; for example, if you want to maintain all Python +module-related files under :file:`python` in your home directory, and you want a +separate directory for each platform that you use your home directory from, you +might define the following installation scheme:: + + python setup.py install --home=~ \ + --install-purelib=python/lib \ + --install-platlib=python/lib.$PLAT \ + --install-scripts=python/scripts + --install-data=python/data + +or, equivalently, :: + + python setup.py install --home=~/python \ + --install-purelib=lib \ + --install-platlib='lib.$PLAT' \ + --install-scripts=scripts + --install-data=data + +``$PLAT`` is not (necessarily) an environment variable---it will be expanded by +the Distutils as it parses your command line options, just as it does when +parsing your configuration file(s). + +Obviously, specifying the entire installation scheme every time you install a +new module distribution would be very tedious. Thus, you can put these options +into your Distutils config file (see section :ref:`inst-config-files`):: + + [install] + install-base=$HOME + install-purelib=python/lib + install-platlib=python/lib.$PLAT + install-scripts=python/scripts + install-data=python/data + +or, equivalently, :: + + [install] + install-base=$HOME/python + install-purelib=lib + install-platlib=lib.$PLAT + install-scripts=scripts + install-data=data + +Note that these two are *not* equivalent if you supply a different installation +base directory when you run the setup script. For example, :: + + python setup.py install --install-base=/tmp + +would install pure modules to :file:`{/tmp/python/lib}` in the first case, and +to :file:`{/tmp/lib}` in the second case. (For the second case, you probably +want to supply an installation base of :file:`/tmp/python`.) + +You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample +configuration file input. These are Distutils configuration variables, which +bear a strong resemblance to environment variables. In fact, you can use +environment variables in config files on platforms that have such a notion but +the Distutils additionally define a few extra variables that may not be in your +environment, such as ``$PLAT``. (And of course, on systems that don't have +environment variables, such as Mac OS 9, the configuration variables supplied by +the Distutils are the only ones you can use.) See section :ref:`inst-config-files` +for details. + +.. XXX need some Windows examples---when would custom installation schemes be + needed on those platforms? + + +.. XXX I'm not sure where this section should go. + +.. _inst-search-path: + +Modifying Python's Search Path +------------------------------ + +When the Python interpreter executes an :keyword:`import` statement, it searches +for both Python code and extension modules along a search path. A default value +for the path is configured into the Python binary when the interpreter is built. +You can determine the path by importing the :mod:`sys` module and printing the +value of ``sys.path``. :: + + $ python + Python 2.2 (#11, Oct 3 2002, 13:31:27) + [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.path + ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', + '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', + '/usr/local/lib/python2.3/site-packages'] + >>> + +The null string in ``sys.path`` represents the current working directory. + +The expected convention for locally installed packages is to put them in the +:file:`{...}/site-packages/` directory, but you may want to install Python +modules into some arbitrary directory. For example, your site may have a +convention of keeping all software related to the web server under :file:`/www`. +Add-on Python modules might then belong in :file:`/www/python`, and in order to +import them, this directory must be added to ``sys.path``. There are several +different ways to add the directory. + +The most convenient way is to add a path configuration file to a directory +that's already on Python's path, usually to the :file:`.../site-packages/` +directory. Path configuration files have an extension of :file:`.pth`, and each +line must contain a single path that will be appended to ``sys.path``. (Because +the new paths are appended to ``sys.path``, modules in the added directories +will not override standard modules. This means you can't use this mechanism for +installing fixed versions of standard modules.) + +Paths can be absolute or relative, in which case they're relative to the +directory containing the :file:`.pth` file. See the documentation of +the :mod:`site` module for more information. + +A slightly less convenient way is to edit the :file:`site.py` file in Python's +standard library, and modify ``sys.path``. :file:`site.py` is automatically +imported when the Python interpreter is executed, unless the :option:`-S` switch +is supplied to suppress this behaviour. So you could simply edit +:file:`site.py` and add two lines to it:: + + import sys + sys.path.append('/www/python/') + +However, if you reinstall the same major version of Python (perhaps when +upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by +the stock version. You'd have to remember that it was modified and save a copy +before doing the installation. + +There are two environment variables that can modify ``sys.path``. +:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python +installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, +the search path will be set to ``['', '/www/python/lib/pythonX.Y/', +'/www/python/lib/pythonX.Y/plat-linux2', ...]``. + +The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be +added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is +set to ``/www/python:/opt/py``, the search path will begin with +``['/www/python', '/opt/py']``. (Note that directories must exist in order to +be added to ``sys.path``; the :mod:`site` module removes paths that don't +exist.) + +Finally, ``sys.path`` is just a regular Python list, so any Python application +can modify it by adding or removing entries. + + +.. _inst-config-files: + +Distutils Configuration Files +============================= + +As mentioned above, you can use Distutils configuration files to record personal +or site preferences for any Distutils options. That is, any option to any +command can be stored in one of two or three (depending on your platform) +configuration files, which will be consulted before the command-line is parsed. +This means that configuration files will override default values, and the +command-line will in turn override configuration files. Furthermore, if +multiple configuration files apply, values from "earlier" files are overridden +by "later" files. + + +.. _inst-config-filenames: + +Location and names of config files +---------------------------------- + +The names and locations of the configuration files vary slightly across +platforms. On Unix and Mac OS X, the three configuration files (in the order +they are processed) are: + ++--------------+----------------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+==========================================================+=======+ +| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | ++--------------+----------------------------------------------------------+-------+ +| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | ++--------------+----------------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+----------------------------------------------------------+-------+ + +And on Windows, the configuration files are: + ++--------------+-------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+=================================================+=======+ +| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | ++--------------+-------------------------------------------------+-------+ +| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | ++--------------+-------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+-------------------------------------------------+-------+ + +On all platforms, the "personal" file can be temporarily disabled by +passing the `--no-user-cfg` option. + +Notes: + +(1) + Strictly speaking, the system-wide configuration file lives in the directory + where the Distutils are installed; under Python 1.6 and later on Unix, this is + as shown. For Python 1.5.2, the Distutils will normally be installed to + :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system + configuration file should be put there under Python 1.5.2. + +(2) + On Unix, if the :envvar:`HOME` environment variable is not defined, the user's + home directory will be determined with the :func:`getpwuid` function from the + standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` + function used by Distutils. + +(3) + I.e., in the current directory (usually the location of the setup script). + +(4) + (See also note (1).) Under Python 1.6 and later, Python's default "installation + prefix" is :file:`C:\\Python`, so the system configuration file is normally + :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the + default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not + part of the standard library---so the system configuration file would be + :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python + 1.5.2 installation under Windows. + +(5) + On Windows, if the :envvar:`HOME` environment variable is not defined, + :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will + be tried. This is done by the :func:`os.path.expanduser` function used + by Distutils. + + +.. _inst-config-syntax: + +Syntax of config files +---------------------- + +The Distutils configuration files all have the same syntax. The config files +are grouped into sections. There is one section for each Distutils command, +plus a ``global`` section for global options that affect every command. Each +section consists of one option per line, specified as ``option=value``. + +For example, the following is a complete config file that just forces all +commands to run quietly by default:: + + [global] + verbose=0 + +If this is installed as the system config file, it will affect all processing of +any Python module distribution by any user on the current system. If it is +installed as your personal config file (on systems that support them), it will +affect only module distributions processed by you. And if it is used as the +:file:`setup.cfg` for a particular module distribution, it affects only that +distribution. + +You could override the default "build base" directory and make the +:command:`build\*` commands always forcibly rebuild all files with the +following:: + + [build] + build-base=blib + force=1 + +which corresponds to the command-line arguments :: + + python setup.py build --build-base=blib --force + +except that including the :command:`build` command on the command-line means +that command will be run. Including a particular command in config files has no +such implication; it only means that if the command is run, the options in the +config file will apply. (Or if other commands that derive values from it are +run, they will use the values in the config file.) + +You can find out the complete list of options for any command using the +:option:`--help` option, e.g.:: + + python setup.py build --help + +and you can find out the complete list of global options by using +:option:`--help` without a command:: + + python setup.py --help + +See also the "Reference" section of the "Distributing Python Modules" manual. + + +.. _inst-building-ext: + +Building Extensions: Tips and Tricks +==================================== + +Whenever possible, the Distutils try to use the configuration information made +available by the Python interpreter used to run the :file:`setup.py` script. +For example, the same compiler and linker flags used to compile Python will also +be used for compiling extensions. Usually this will work well, but in +complicated situations this might be inappropriate. This section discusses how +to override the usual Distutils behaviour. + + +.. _inst-tweak-flags: + +Tweaking compiler/linker flags +------------------------------ + +Compiling a Python extension written in C or C++ will sometimes require +specifying custom flags for the compiler and linker in order to use a particular +library or produce a special kind of object code. This is especially true if the +extension hasn't been tested on your platform, or if you're trying to +cross-compile Python. + +In the most general case, the extension author might have foreseen that +compiling the extensions would be complicated, and provided a :file:`Setup` file +for you to edit. This will likely only be done if the module distribution +contains many separate extension modules, or if they often require elaborate +sets of compiler flags in order to work. + +A :file:`Setup` file, if present, is parsed in order to get a list of extensions +to build. Each line in a :file:`Setup` describes a single module. Lines have +the following structure:: + + module ... [sourcefile ...] [cpparg ...] [library ...] + + +Let's examine each of the fields in turn. + +* *module* is the name of the extension module to be built, and should be a + valid Python identifier. You can't just change this in order to rename a module + (edits to the source code would also be needed), so this should be left alone. + +* *sourcefile* is anything that's likely to be a source code file, at least + judging by the filename. Filenames ending in :file:`.c` are assumed to be + written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are + assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed + to be in Objective C. + +* *cpparg* is an argument for the C preprocessor, and is anything starting with + :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. + +* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or + :option:`-L`. + +If a particular platform requires a special library on your platform, you can +add it by editing the :file:`Setup` file and running ``python setup.py build``. +For example, if the module defined by the line :: + + foo foomodule.c + +must be linked with the math library :file:`libm.a` on your platform, simply add +:option:`-lm` to the line:: + + foo foomodule.c -lm + +Arbitrary switches intended for the compiler or the linker can be supplied with +the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: + + foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm + +The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be +appended to the proper command line, so in the above example the compiler will +be passed the :option:`-o32` option, and the linker will be passed +:option:`-shared`. If a compiler option requires an argument, you'll have to +supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` +the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. + +Compiler flags can also be supplied through setting the :envvar:`CFLAGS` +environment variable. If set, the contents of :envvar:`CFLAGS` will be added to +the compiler flags specified in the :file:`Setup` file. + + +.. _inst-non-ms-compilers: + +Using non-Microsoft compilers on Windows +---------------------------------------- + +.. sectionauthor:: Rene Liebscher + + + +Borland/CodeGear C++ +^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the necessary steps to use Distutils with the Borland +C++ compiler version 5.5. First you have to know that Borland's object file +format (OMF) is different from the format used by the Python version you can +download from the Python or ActiveState Web site. (Python is built with +Microsoft Visual C++, which uses COFF as the object file format.) For this +reason you have to convert Python's library :file:`python25.lib` into the +Borland format. You can do this as follows: + +.. Should we mention that users have to create cfg-files for the compiler? +.. see also http://community.borland.com/article/0,1410,21205,00.html + +:: + + coff2omf python25.lib python25_bcpp.lib + +The :file:`coff2omf` program comes with the Borland compiler. The file +:file:`python25.lib` is in the :file:`Libs` directory of your Python +installation. If your extension uses other libraries (zlib, ...) you have to +convert them too. + +The converted files have to reside in the same directories as the normal +libraries. + +How does Distutils manage to use these libraries with their changed names? If +the extension needs a library (eg. :file:`foo`) Distutils checks first if it +finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then +uses this library. In the case it doesn't find such a special library it uses +the default name (:file:`foo.lib`.) [#]_ + +To let Distutils compile your extension with Borland C++ you now have to type:: + + python setup.py build --compiler=bcpp + +If you want to use the Borland C++ compiler as the default, you could specify +this in your personal or system-wide configuration file for Distutils (see +section :ref:`inst-config-files`.) + + +.. seealso:: + + `C++Builder Compiler `_ + Information about the free C++ compiler from Borland, including links to the + download pages. + + `Creating Python Extensions Using Borland's Free Compiler `_ + Document describing how to use Borland's free command-line C++ compiler to build + Python. + + +GNU C / Cygwin / MinGW +^^^^^^^^^^^^^^^^^^^^^^ + +This section describes the necessary steps to use Distutils with the GNU C/C++ +compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter +that was built with Cygwin, everything should work without any of these +following steps. + +Not all extensions can be built with MinGW or Cygwin, but many can. Extensions +most likely to not work are those that use C++ or depend on Microsoft Visual C +extensions. + +To let Distutils compile your extension with Cygwin you have to type:: + + python setup.py build --compiler=cygwin + +and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: + + python setup.py build --compiler=mingw32 + +If you want to use any of these options/compilers as default, you should +consider writing it in your personal or system-wide configuration file for +Distutils (see section :ref:`inst-config-files`.) + +Older Versions of Python and MinGW +"""""""""""""""""""""""""""""""""" +The following instructions only apply if you're using a version of Python +inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with +binutils-2.13.90-20030111-1). + +These compilers require some special libraries. This task is more complex than +for Borland's C++, because there is no program to convert the library. First +you have to create a list of symbols which the Python DLL exports. (You can find +a good program for this task at +http://www.emmestech.com/software/pexports-0.43/download_pexports.html). + +.. I don't understand what the next line means. --amk +.. (inclusive the references on data structures.) + +:: + + pexports python25.dll >python25.def + +The location of an installed :file:`python25.dll` will depend on the +installation options and the version and language of Windows. In a "just for +me" installation, it will appear in the root of the installation directory. In +a shared installation, it will be located in the system directory. + +Then you can create from these information an import library for gcc. :: + + /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a + +The resulting library has to be placed in the same directory as +:file:`python25.lib`. (Should be the :file:`libs` directory under your Python +installation directory.) + +If your extension uses other libraries (zlib,...) you might have to convert +them too. The converted files have to reside in the same directories as the +normal libraries do. + + +.. seealso:: + + `Building Python modules on MS Windows platform with MinGW `_ + Information about building the required libraries for the MinGW environment. + + +.. rubric:: Footnotes + +.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries + of the same name. + +.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for more + information + +.. [#] Then you have no POSIX emulation available, but you also don't need + :file:`cygwin1.dll`. diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index 31c1d7f..0000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1005 +0,0 @@ -.. highlightlang:: none - -.. _install-index: - -***************************** - Installing Python Modules -***************************** - -:Author: Greg Ward -:Release: |version| -:Date: |today| - -.. TODO: Fill in XXX comments - -.. The audience for this document includes people who don't know anything - about Python and aren't about to learn the language just in order to - install and maintain it for their users, i.e. system administrators. - Thus, I have to be sure to explain the basics at some point: - sys.path and PYTHONPATH at least. Should probably give pointers to - other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. - - Finally, it might be useful to include all the material from my "Care - and Feeding of a Python Installation" talk in here somewhere. Yow! - -.. topic:: Abstract - - This document describes the Python Distribution Utilities ("Distutils") from the - end-user's point-of-view, describing how to extend the capabilities of a - standard Python installation by building and installing third-party Python - modules and extensions. - - -.. _inst-intro: - -Introduction -============ - -Although Python's extensive standard library covers many programming needs, -there often comes a time when you need to add some new functionality to your -Python installation in the form of third-party modules. This might be necessary -to support your own programming, or to support an application that you want to -use and that happens to be written in Python. - -In the past, there has been little support for adding third-party modules to an -existing Python installation. With the introduction of the Python Distribution -Utilities (Distutils for short) in Python 2.0, this changed. - -This document is aimed primarily at the people who need to install third-party -Python modules: end-users and system administrators who just need to get some -Python application running, and existing Python programmers who want to add some -new goodies to their toolbox. You don't need to know Python to read this -document; there will be some brief forays into using Python's interactive mode -to explore your installation, but that's it. If you're looking for information -on how to distribute your own Python modules so that others may use them, see -the :ref:`distutils-index` manual. - - -.. _inst-trivial-install: - -Best case: trivial installation -------------------------------- - -In the best case, someone will have prepared a special version of the module -distribution you want to install that is targeted specifically at your platform -and is installed just like any other software on your platform. For example, -the module developer might make an executable installer available for Windows -users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, -Mandrake, and many others), a Debian package for users of Debian-based Linux -systems, and so forth. - -In that case, you would download the installer appropriate to your platform and -do the obvious thing with it: run it if it's an executable installer, ``rpm ---install`` it if it's an RPM, etc. You don't need to run Python or a setup -script, you don't need to compile anything---you might not even need to read any -instructions (although it's always a good idea to do so anyways). - -Of course, things will not always be that easy. You might be interested in a -module distribution that doesn't have an easy-to-use installer for your -platform. In that case, you'll have to start with the source distribution -released by the module's author/maintainer. Installing from a source -distribution is not too hard, as long as the modules are packaged in the -standard way. The bulk of this document is about building and installing -modules from standard source distributions. - - -.. _inst-new-standard: - -The new standard: Distutils ---------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly-named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running :: - - python setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command:: - - python setup.py install - -On Unix, you'd run this command from a shell prompt; on Windows, you have to -open a command prompt window ("DOS box") and do it there; on Mac OS X, you open -a :command:`Terminal` window to get a shell prompt. - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either a archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window ("DOS box"), and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`--build-base` option. For example:: - - python setup.py build --build-base=/tmp/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and Mac OS X, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and Mac OS X. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`:: - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - - -.. _inst-alt-install-prefix: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -The :option:`--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -+------------------------------+---------------------------+-----------------------------+ -| Type of file | Installation Directory | Override option | -+==============================+===========================+=============================+ -| pure module distribution | :file:`{home}/lib/python` | :option:`--install-purelib` | -+------------------------------+---------------------------+-----------------------------+ -| non-pure module distribution | :file:`{home}/lib/python` | :option:`--install-platlib` | -+------------------------------+---------------------------+-----------------------------+ -| scripts | :file:`{home}/bin` | :option:`--install-scripts` | -+------------------------------+---------------------------+-----------------------------+ -| data | :file:`{home}/share` | :option:`--install-data` | -+------------------------------+---------------------------+-----------------------------+ - - -.. _inst-alt-install-home: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the "home scheme" comes first. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`--prefix` option defines the installation base, and -the :option:`--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to -:option:`--prefix`. Files are installed as follows: - -+------------------------------+-----------------------------------------------------+-----------------------------+ -| Type of file | Installation Directory | Override option | -+==============================+=====================================================+=============================+ -| pure module distribution | :file:`{prefix}/lib/python{X.Y}/site-packages` | :option:`--install-purelib` | -+------------------------------+-----------------------------------------------------+-----------------------------+ -| non-pure module distribution | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :option:`--install-platlib` | -+------------------------------+-----------------------------------------------------+-----------------------------+ -| scripts | :file:`{prefix}/bin` | :option:`--install-scripts` | -+------------------------------+-----------------------------------------------------+-----------------------------+ -| data | :file:`{prefix}/share` | :option:`--install-data` | -+------------------------------+-----------------------------------------------------+-----------------------------+ - -There is no requirement that :option:`--prefix` or :option:`--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`--prefix` -and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`--prefix` and :option:`--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`--prefix` option; the -:option:`--exec-prefix` option is not supported under Windows. Files are -installed as follows: - -+------------------------------+---------------------------+-----------------------------+ -| Type of file | Installation Directory | Override option | -+==============================+===========================+=============================+ -| pure module distribution | :file:`{prefix}` | :option:`--install-purelib` | -+------------------------------+---------------------------+-----------------------------+ -| non-pure module distribution | :file:`{prefix}` | :option:`--install-platlib` | -+------------------------------+---------------------------+-----------------------------+ -| scripts | :file:`{prefix}\\Scripts` | :option:`--install-scripts` | -+------------------------------+---------------------------+-----------------------------+ -| data | :file:`{prefix}\\Data` | :option:`--install-data` | -+------------------------------+---------------------------+-----------------------------+ - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -You probably noticed the column of "override options" in the tables describing -the alternate installation schemes above. Those options are how you define a -custom installation scheme. These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the same--- -they only differ when you use the Unix "prefix scheme" and supply different -:option:`--prefix` and :option:`--exec-prefix` options.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme," where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation directory ----you just have to remember that there are two types of modules to worry about, -pure modules and non-pure modules (i.e., modules from a non-pure distribution). -For example:: - - python setup.py install --install-purelib=Site --install-platlib=Site - -The specified installation directories are relative to :file:`{prefix}`. Of -course, you also have to ensure that these directories are in Python's module -search path, such as by putting a :file:`.pth` file in :file:`{prefix}`. See -section :ref:`inst-search-path` to find out how to modify Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`):: - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, :: - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`{/tmp/python/lib}` in the first case, and -to :file:`{/tmp/lib}` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX I'm not sure where this section should go. - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it:: - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same major version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and Mac OS X, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the `--no-user-cfg` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default:: - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following:: - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or - :option:`-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`-o32` option, and the linker will be passed -:option:`-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState Web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -http://www.emmestech.com/software/pexports-0.43/download_pexports.html). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for more - information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/install/install.rst b/Doc/install/install.rst new file mode 100644 index 0000000..8067544 --- /dev/null +++ b/Doc/install/install.rst @@ -0,0 +1,1029 @@ +.. highlightlang:: none + +==================================== +Installing Python projects: overwiew +==================================== + +.. _packaging_packaging-intro: + +Introduction +============ + +Although Python's extensive standard library covers many programming needs, +there often comes a time when you need to add new functionality to your Python +installation in the form of third-party modules. This might be necessary to +support your own programming, or to support an application that you want to use +and that happens to be written in Python. + +In the past, there was little support for adding third-party modules to an +existing Python installation. With the introduction of the Python Distribution +Utilities (Distutils for short) in Python 2.0, this changed. However, not all +problems were solved; end-users had to rely on ``easy_install`` or +``pip`` to download third-party modules from PyPI, uninstall distributions or do +other maintenance operations. Packaging is a more complete replacement for +Distutils, in the standard library, with a backport named Distutils2 available +for older Python versions. + +This document is aimed primarily at people who need to install third-party +Python modules: end-users and system administrators who just need to get some +Python application running, and existing Python programmers who want to add +new goodies to their toolbox. You don't need to know Python to read this +document; there will be some brief forays into using Python's interactive mode +to explore your installation, but that's it. If you're looking for information +on how to distribute your own Python modules so that others may use them, see +the :ref:`packaging-index` manual. + + +.. _packaging-trivial-install: + +Best case: trivial installation +------------------------------- + +In the best case, someone will have prepared a special version of the module +distribution you want to install that is targeted specifically at your platform +and can be installed just like any other software on your platform. For example, +the module's developer might make an executable installer available for Windows +users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE, +Mandrake, and many others), a Debian package for users of Debian and derivative +systems, and so forth. + +In that case, you would use the standard system tools to download and install +the specific installer for your platform and its dependencies. + +Of course, things will not always be that easy. You might be interested in a +module whose distribution doesn't have an easy-to-use installer for your +platform. In that case, you'll have to start with the source distribution +released by the module's author/maintainer. Installing from a source +distribution is not too hard, as long as the modules are packaged in the +standard way. The bulk of this document addresses the building and installing +of modules from standard source distributions. + + +.. _packaging-distutils: + +The Python standard: Distutils +------------------------------ + +If you download a source distribution of a module, it will be obvious whether +it was packaged and distributed using Distutils. First, the distribution's name +and version number will be featured prominently in the name of the downloaded +archive, e.g. :file:`foo-1.0.tar.gz` or :file:`widget-0.9.7.zip`. Next, the +archive will unpack into a similarly-named directory: :file:`foo-1.0` or +:file:`widget-0.9.7`. Additionally, the distribution may contain a +:file:`setup.cfg` file and a file named :file:`README.txt` ---or possibly just +:file:`README`--- explaining that building and installing the module +distribution is a simple matter of issuing the following command at your shell's +prompt:: + + python setup.py install + +Third-party projects have extended Distutils to work around its limitations or +add functionality. After some years of near-inactivity in Distutils, a new +maintainer has started to standardize good ideas in PEPs and implement them in a +new, improved version of Distutils, called Distutils2 or Packaging. + + +.. _packaging-new-standard: + +The new standard: Packaging +--------------------------- + +The rules described in the first paragraph above apply to Packaging-based +projects too: a source distribution will have a name like +:file:`widget-0.9.7.zip`. One of the main differences with Distutils is that +distributions no longer have a :file:`setup.py` script; it used to cause a +number of issues. Now there is a unique script installed with Python itself:: + + pysetup install widget-0.9.7.zip + +Running this command is enough to build and install projects (Python modules or +packages, scripts or whole applications), without even having to unpack the +archive. It is also compatible with Distutils-based distributions. + +Unless you have to perform non-standard installations or customize the build +process, you can stop reading this manual ---the above command is everything you +need to get out of it. + +With :program:`pysetup`, you won't even have to manually download a distribution +before installing it; see :ref:`packaging-pysetup`. + + +.. _packaging-standard-install: + +Standard build and install +========================== + +As described in section :ref:`packaging-new-standard`, building and installing +a module distribution using Packaging usually comes down to one simple +command:: + + pysetup run install_dist + +How you actually run this command depends on the platform and the command line +interface it provides: + +* **Unix**: Use a shell prompt. +* **Windows**: Open a command prompt ("DOS console") or use :command:`Powershell`. +* **OS X**: Open a :command:`Terminal`. + + +.. _packaging-platform-variations: + +Platform variations +------------------- + +The setup command is meant to be run from the root directory of the source +distribution, i.e. the top-level subdirectory that the module source +distribution unpacks into. For example, if you've just downloaded a module +source distribution :file:`foo-1.0.tar.gz` onto a Unix system, the normal +steps to follow are these:: + + gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 + cd foo-1.0 + pysetup run install_dist + +On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the +archive file to :file:`C:\\Temp`, then it would unpack into +:file:`C:\\Temp\\foo-1.0`. To actually unpack the archive, you can use either +an archive manipulator with a graphical user interface (such as WinZip or 7-Zip) +or a command-line tool (such as :program:`unzip`, :program:`pkunzip` or, again, +:program:`7z`). Then, open a command prompt window ("DOS box" or +Powershell), and run:: + + cd c:\Temp\foo-1.0 + pysetup run install_dist + + +.. _packaging-splitting-up: + +Splitting the job up +-------------------- + +Running ``pysetup run install_dist`` builds and installs all modules in one go. If you +prefer to work incrementally ---especially useful if you want to customize the +build process, or if things are going wrong--- you can use the setup script to +do one thing at a time. This is a valuable tool when different users will perform +separately the build and install steps. For example, you might want to build a +module distribution and hand it off to a system administrator for installation +(or do it yourself, but with super-user or admin privileges). + +For example, to build everything in one step and then install everything +in a second step, you aptly invoke two distinct Packaging commands:: + + pysetup run build + pysetup run install_dist + +If you do this, you will notice that invoking the :command:`install_dist` command +first runs the :command:`build` command, which ---in this case--- quickly +notices it can spare itself the work, since everything in the :file:`build` +directory is up-to-date. + +You may often ignore this ability to divide the process in steps if all you do +is installing modules downloaded from the Internet, but it's very handy for +more advanced tasks. If you find yourself in the need for distributing your own +Python modules and extensions, though, you'll most likely run many individual +Packaging commands. + + +.. _packaging-how-build-works: + +How building works +------------------ + +As implied above, the :command:`build` command is responsible for collecting +and placing the files to be installed into a *build directory*. By default, +this is :file:`build`, under the distribution root. If you're excessively +concerned with speed, or want to keep the source tree pristine, you can specify +a different build directory with the :option:`--build-base` option. For example:: + + pysetup run build --build-base /tmp/pybuild/foo-1.0 + +(Or you could do this permanently with a directive in your system or personal +Packaging configuration file; see section :ref:`packaging-config-files`.) +In the usual case, however, all this is unnecessary. + +The build tree's default layout looks like so:: + + --- build/ --- lib/ + or + --- build/ --- lib./ + temp./ + +where ```` expands to a brief description of the current OS/hardware +platform and Python version. The first form, with just a :file:`lib` directory, +is used for pure module distributions (module distributions that +include only pure Python modules). If a module distribution contains any +extensions (modules written in C/C++), then the second form, with two ```` +directories, is used. In that case, the :file:`temp.{plat}` directory holds +temporary files generated during the compile/link process which are not intended +to be installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory +contains all Python modules (pure Python and extensions) to be installed. + +In the future, more directories will be added to handle Python scripts, +documentation, binary executables, and whatever else is required to install +Python modules and applications. + + +.. _packaging-how-install-works: + +How installation works +---------------------- + +After the :command:`build` command is run (whether explicitly or by the +:command:`install_dist` command on your behalf), the work of the :command:`install_dist` +command is relatively simple: all it has to do is copy the contents of +:file:`build/lib` (or :file:`build/lib.{plat}`) to the installation directory +of your choice. + +If you don't choose an installation directory ---i.e., if you just run +``pysetup run install_dist``\ --- then the :command:`install_dist` command +installs to the standard location for third-party Python modules. This location +varies by platform and depending on how you built/installed Python itself. On +Unix (and Mac OS X, which is also Unix-based), it also depends on whether the +module distribution being installed is pure Python or contains extensions +("non-pure"): + ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Platform | Standard installation location | Default value | Notes | ++=================+=====================================================+==================================================+=======+ +| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ +| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | ++-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ + +Notes: + +(1) + Most Linux distributions include Python as a standard part of the system, so + :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on + Linux. If you build Python yourself on Linux (or any Unix-like system), the + default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. + +(2) + The default installation directory on Windows was :file:`C:\\Program + Files\\Python` under Python 1.6a1, 1.5.2, and earlier. + +:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python +is installed to, and where it finds its libraries at run-time. They are always +the same under Windows, and very often the same under Unix and Mac OS X. You +can find out what your Python installation uses for :file:`{prefix}` and +:file:`{exec-prefix}` by running Python in interactive mode and typing a few +simple commands. + +.. TODO link to Doc/using instead of duplicating + +To start the interactive Python interpreter, you need to follow a slightly +different recipe for each platform. Under Unix, just type :command:`python` at +the shell prompt. Under Windows (assuming the Python executable is on your +:envvar:`PATH`, which is the usual case), you can choose :menuselection:`Start --> Run`, +type ``python`` and press ``enter``. Alternatively, you can simply execute +:command:`python` at a command prompt ("DOS console" or Powershell). + +Once the interpreter is started, you type Python code at the prompt. For +example, on my Linux system, I type the three Python statements shown below, +and get the output as shown, to find out my :file:`{prefix}` and :file:`{exec-prefix}`:: + + Python 3.3 (r32:88445, Apr 2 2011, 10:43:54) + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.prefix + '/usr' + >>> sys.exec_prefix + '/usr' + +If you don't want to install modules to the standard location, or if you don't +have permission to write there, then you need to read about alternate +installations in section :ref:`packaging-alt-install`. If you want to customize your +installation directories more heavily, see section :ref:`packaging-custom-install`. + + +.. _packaging-alt-install: + +Alternate installation +====================== + +Often, it is necessary or desirable to install modules to a location other than +the standard location for third-party Python modules. For example, on a Unix +system you might not have permission to write to the standard third-party module +directory. Or you might wish to try out a module before making it a standard +part of your local Python installation. This is especially true when upgrading +a distribution already present: you want to make sure your existing base of +scripts still works with the new version before actually upgrading. + +The Packaging :command:`install_dist` command is designed to make installing module +distributions to an alternate location simple and painless. The basic idea is +that you supply a base directory for the installation, and the +:command:`install_dist` command picks a set of directories (called an *installation +scheme*) under this base directory in which to install files. The details +differ across platforms, so read whichever of the following sections applies to +you. + + +.. _packaging-alt-install-prefix: + +Alternate installation: the home scheme +--------------------------------------- + +The idea behind the "home scheme" is that you build and maintain a personal +stash of Python modules. This scheme's name is derived from the concept of a +"home" directory on Unix, since it's not unusual for a Unix user to make their +home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. +In spite of its name's origin, this scheme can be used by anyone, regardless +of the operating system. + +Installing a new module distribution in this way is as simple as :: + + pysetup run install_dist --home + +where you can supply any directory you like for the :option:`--home` option. On +Unix, lazy typists can just type a tilde (``~``); the :command:`install_dist` command +will expand this to your home directory:: + + pysetup run install_dist --home ~ + +The :option:`--home` option defines the base directory for the installation. +Under it, files are installed to the following directories: + ++------------------------------+---------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+===========================+=============================+ +| pure module distribution | :file:`{home}/lib/python` | :option:`--install-purelib` | ++------------------------------+---------------------------+-----------------------------+ +| non-pure module distribution | :file:`{home}/lib/python` | :option:`--install-platlib` | ++------------------------------+---------------------------+-----------------------------+ +| scripts | :file:`{home}/bin` | :option:`--install-scripts` | ++------------------------------+---------------------------+-----------------------------+ +| data | :file:`{home}/share` | :option:`--install-data` | ++------------------------------+---------------------------+-----------------------------+ + + +.. _packaging-alt-install-home: + +Alternate installation: Unix (the prefix scheme) +------------------------------------------------ + +The "prefix scheme" is useful when you wish to use one Python installation to +run the build command, but install modules into the third-party module directory +of a different Python installation (or something that looks like a different +Python installation). If this sounds a trifle unusual, it is ---that's why the +"home scheme" comes first. However, there are at least two known cases where the +prefix scheme will be useful. + +First, consider that many Linux distributions put Python in :file:`/usr`, rather +than the more traditional :file:`/usr/local`. This is entirely appropriate, +since in those cases Python is part of "the system" rather than a local add-on. +However, if you are installing Python modules from source, you probably want +them to go in :file:`/usr/local/lib/python2.{X}` rather than +:file:`/usr/lib/python2.{X}`. This can be done with :: + + pysetup run install_dist --prefix /usr/local + +Another possibility is a network filesystem where the name used to write to a +remote directory is different from the name used to read it: for example, the +Python interpreter accessed as :file:`/usr/local/bin/python` might search for +modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to +be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could +be done with :: + + pysetup run install_dist --prefix=/mnt/@server/export + +In either case, the :option:`--prefix` option defines the installation base, and +the :option:`--exec-prefix` option defines the platform-specific installation +base, which is used for platform-specific files. (Currently, this just means +non-pure module distributions, but could be expanded to C libraries, binary +executables, etc.) If :option:`--exec-prefix` is not supplied, it defaults to +:option:`--prefix`. Files are installed as follows: + ++------------------------------+-----------------------------------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+=====================================================+=============================+ +| pure module distribution | :file:`{prefix}/lib/python{X.Y}/site-packages` | :option:`--install-purelib` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| non-pure module distribution | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :option:`--install-platlib` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| scripts | :file:`{prefix}/bin` | :option:`--install-scripts` | ++------------------------------+-----------------------------------------------------+-----------------------------+ +| data | :file:`{prefix}/share` | :option:`--install-data` | ++------------------------------+-----------------------------------------------------+-----------------------------+ + +There is no requirement that :option:`--prefix` or :option:`--exec-prefix` +actually point to an alternate Python installation; if the directories listed +above do not already exist, they are created at installation time. + +Incidentally, the real reason the prefix scheme is important is simply that a +standard Unix installation uses the prefix scheme, but with :option:`--prefix` +and :option:`--exec-prefix` supplied by Python itself as ``sys.prefix`` and +``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, +but every time you run ``pysetup run install_dist`` without any other +options, you're using it. + +Note that installing extensions to an alternate Python installation doesn't have +anything to do with how those extensions are built: in particular, extensions +will be compiled using the Python header files (:file:`Python.h` and friends) +installed with the Python interpreter used to run the build command. It is +therefore your responsibility to ensure compatibility between the interpreter +intended to run extensions installed in this way and the interpreter used to +build these same extensions. To avoid problems, it is best to make sure that +the two interpreters are the same version of Python (possibly different builds, +or possibly copies of the same build). (Of course, if your :option:`--prefix` +and :option:`--exec-prefix` don't even point to an alternate Python installation, +this is immaterial.) + + +.. _packaging-alt-install-windows: + +Alternate installation: Windows (the prefix scheme) +--------------------------------------------------- + +Windows has a different and vaguer notion of home directories than Unix, and +since its standard Python installation is simpler, the :option:`--prefix` option +has traditionally been used to install additional packages to arbitrary +locations. :: + + pysetup run install_dist --prefix "\Temp\Python" + +to install modules to the :file:`\\Temp\\Python` directory on the current drive. + +The installation base is defined by the :option:`--prefix` option; the +:option:`--exec-prefix` option is unsupported under Windows. Files are +installed as follows: + ++------------------------------+---------------------------+-----------------------------+ +| Type of file | Installation Directory | Override option | ++==============================+===========================+=============================+ +| pure module distribution | :file:`{prefix}` | :option:`--install-purelib` | ++------------------------------+---------------------------+-----------------------------+ +| non-pure module distribution | :file:`{prefix}` | :option:`--install-platlib` | ++------------------------------+---------------------------+-----------------------------+ +| scripts | :file:`{prefix}\\Scripts` | :option:`--install-scripts` | ++------------------------------+---------------------------+-----------------------------+ +| data | :file:`{prefix}\\Data` | :option:`--install-data` | ++------------------------------+---------------------------+-----------------------------+ + + +.. _packaging-custom-install: + +Custom installation +=================== + +Sometimes, the alternate installation schemes described in section +:ref:`packaging-alt-install` just don't do what you want. You might want to tweak +just one or two directories while keeping everything under the same base +directory, or you might want to completely redefine the installation scheme. +In either case, you're creating a *custom installation scheme*. + +You probably noticed the column of "override options" in the tables describing +the alternate installation schemes above. Those options are how you define a +custom installation scheme. These override options can be relative, absolute, +or explicitly defined in terms of one of the installation base directories. +(There are two installation base directories, and they are normally the same +---they only differ when you use the Unix "prefix scheme" and supply different +:option:`--prefix` and :option:`--exec-prefix` options.) + +For example, say you're installing a module distribution to your home directory +under Unix, but you want scripts to go in :file:`~/scripts` rather than +:file:`~/bin`. As you might expect, you can override this directory with the +:option:`--install-scripts` option and, in this case, it makes most sense to supply +a relative path, which will be interpreted relative to the installation base +directory (in our example, your home directory):: + + pysetup run install_dist --home ~ --install-scripts scripts + +Another Unix example: suppose your Python installation was built and installed +with a prefix of :file:`/usr/local/python`. Thus, in a standard installation, +scripts will wind up in :file:`/usr/local/python/bin`. If you want them in +:file:`/usr/local/bin` instead, you would supply this absolute directory for +the :option:`--install-scripts` option:: + + pysetup run install_dist --install-scripts /usr/local/bin + +This command performs an installation using the "prefix scheme", where the +prefix is whatever your Python interpreter was installed with ---in this case, +:file:`/usr/local/python`. + +If you maintain Python on Windows, you might want third-party modules to live in +a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` +itself. This is almost as easy as customizing the script installation directory +---you just have to remember that there are two types of modules to worry about, +pure modules and non-pure modules (i.e., modules from a non-pure distribution). +For example:: + + pysetup run install_dist --install-purelib Site --install-platlib Site + +.. XXX Nothing is installed right under prefix in windows, is it?? + +The specified installation directories are relative to :file:`{prefix}`. Of +course, you also have to ensure that these directories are in Python's module +search path, such as by putting a :file:`.pth` file in :file:`{prefix}`. See +section :ref:`packaging-search-path` to find out how to modify Python's search path. + +If you want to define an entire installation scheme, you just have to supply all +of the installation directory options. Using relative paths is recommended here. +For example, if you want to maintain all Python module-related files under +:file:`python` in your home directory, and you want a separate directory for +each platform that you use your home directory from, you might define the +following installation scheme:: + + pysetup run install_dist --home ~ \ + --install-purelib python/lib \ + --install-platlib python/'lib.$PLAT' \ + --install-scripts python/scripts \ + --install-data python/data + +or, equivalently, :: + + pysetup run install_dist --home ~/python \ + --install-purelib lib \ + --install-platlib 'lib.$PLAT' \ + --install-scripts scripts \ + --install-data data + +``$PLAT`` doesn't need to be defined as an environment variable ---it will also +be expanded by Packaging as it parses your command line options, just as it +does when parsing your configuration file(s). (More on that later.) + +Obviously, specifying the entire installation scheme every time you install a +new module distribution would be very tedious. To spare you all that work, you +can store it in a Packaging configuration file instead (see section +:ref:`packaging-config-files`), like so:: + + [install_dist] + install-base = $HOME + install-purelib = python/lib + install-platlib = python/lib.$PLAT + install-scripts = python/scripts + install-data = python/data + +or, equivalently, :: + + [install_dist] + install-base = $HOME/python + install-purelib = lib + install-platlib = lib.$PLAT + install-scripts = scripts + install-data = data + +Note that these two are *not* equivalent if you override their installation +base directory when running the setup script. For example, :: + + pysetup run install_dist --install-base /tmp + +would install pure modules to :file:`/tmp/python/lib` in the first case, and +to :file:`/tmp/lib` in the second case. (For the second case, you'd probably +want to supply an installation base of :file:`/tmp/python`.) + +You may have noticed the use of ``$HOME`` and ``$PLAT`` in the sample +configuration file. These are Packaging configuration variables, which +bear a strong resemblance to environment variables. In fact, you can use +environment variables in configuration files on platforms that have such a notion, but +Packaging additionally defines a few extra variables that may not be in your +environment, such as ``$PLAT``. Of course, on systems that don't have +environment variables, such as Mac OS 9, the configuration variables supplied by +the Packaging are the only ones you can use. See section :ref:`packaging-config-files` +for details. + +.. XXX which vars win out eventually in case of clash env or Packaging? + +.. XXX need some Windows examples---when would custom installation schemes be + needed on those platforms? + + +.. XXX Move this section to Doc/using + +.. _packaging-search-path: + +Modifying Python's search path +------------------------------ + +When the Python interpreter executes an :keyword:`import` statement, it searches +for both Python code and extension modules along a search path. A default value +for this path is configured into the Python binary when the interpreter is built. +You can obtain the search path by importing the :mod:`sys` module and printing +the value of ``sys.path``. :: + + $ python + Python 2.2 (#11, Oct 3 2002, 13:31:27) + [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import sys + >>> sys.path + ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', + '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', + '/usr/local/lib/python2.3/site-packages'] + >>> + +The null string in ``sys.path`` represents the current working directory. + +The expected convention for locally installed packages is to put them in the +:file:`{...}/site-packages/` directory, but you may want to choose a different +location for some reason. For example, if your site kept by convention all web +server-related software under :file:`/www`. Add-on Python modules might then +belong in :file:`/www/python`, and in order to import them, this directory would +have to be added to ``sys.path``. There are several ways to solve this problem. + +The most convenient way is to add a path configuration file to a directory +that's already on Python's path, usually to the :file:`.../site-packages/` +directory. Path configuration files have an extension of :file:`.pth`, and each +line must contain a single path that will be appended to ``sys.path``. (Because +the new paths are appended to ``sys.path``, modules in the added directories +will not override standard modules. This means you can't use this mechanism for +installing fixed versions of standard modules.) + +Paths can be absolute or relative, in which case they're relative to the +directory containing the :file:`.pth` file. See the documentation of +the :mod:`site` module for more information. + +A slightly less convenient way is to edit the :file:`site.py` file in Python's +standard library, and modify ``sys.path``. :file:`site.py` is automatically +imported when the Python interpreter is executed, unless the :option:`-S` switch +is supplied to suppress this behaviour. So you could simply edit +:file:`site.py` and add two lines to it:: + + import sys + sys.path.append('/www/python/') + +However, if you reinstall the same major version of Python (perhaps when +upgrading from 3.3 to 3.3.1, for example) :file:`site.py` will be overwritten by +the stock version. You'd have to remember that it was modified and save a copy +before doing the installation. + +Alternatively, there are two environment variables that can modify ``sys.path``. +:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python +installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, +the search path will be set to ``['', '/www/python/lib/pythonX.Y/', +'/www/python/lib/pythonX.Y/plat-linux2', ...]``. + +The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be +added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is +set to ``/www/python:/opt/py``, the search path will begin with +``['/www/python', '/opt/py']``. (Note that directories must exist in order to +be added to ``sys.path``; the :mod:`site` module removes non-existent paths.) + +Finally, ``sys.path`` is just a regular Python list, so any Python application +can modify it by adding or removing entries. + + +.. _packaging-config-files: + +Configuration files for Packaging +================================= + +As mentioned above, you can use configuration files to store personal or site +preferences for any option supported by any Packaging command. Depending on your +platform, you can use one of two or three possible configuration files. These +files will be read before parsing the command-line, so they take precedence over +default values. In turn, the command-line will override configuration files. +Lastly, if there are multiple configuration files, values from files read +earlier will be overridden by values from files read later. + +.. XXX "one of two or three possible..." seems wrong info. Below always 3 files + are indicated in the tables. + + +.. _packaging-config-filenames: + +Location and names of configuration files +----------------------------------------- + +The name and location of the configuration files vary slightly across +platforms. On Unix and Mac OS X, these are the three configuration files listed +in the order they are processed: + ++--------------+----------------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+==========================================================+=======+ +| system | :file:`{prefix}/lib/python{ver}/packaging/packaging.cfg` | \(1) | ++--------------+----------------------------------------------------------+-------+ +| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | ++--------------+----------------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+----------------------------------------------------------+-------+ + +Similarly, the configuration files on Windows ---also listed in the order they +are processed--- are these: + ++--------------+-------------------------------------------------+-------+ +| Type of file | Location and filename | Notes | ++==============+=================================================+=======+ +| system | :file:`{prefix}\\Lib\\packaging\\packaging.cfg` | \(4) | ++--------------+-------------------------------------------------+-------+ +| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | ++--------------+-------------------------------------------------+-------+ +| local | :file:`setup.cfg` | \(3) | ++--------------+-------------------------------------------------+-------+ + +On all platforms, the *personal* file can be temporarily disabled by +means of the `--no-user-cfg` option. + +Notes: + +(1) + Strictly speaking, the system-wide configuration file lives in the directory + where Packaging is installed. + +(2) + On Unix, if the :envvar:`HOME` environment variable is not defined, the + user's home directory will be determined with the :func:`getpwuid` function + from the standard :mod:`pwd` module. Packaging uses the + :func:`os.path.expanduser` function to do this. + +(3) + I.e., in the current directory (usually the location of the setup script). + +(4) + (See also note (1).) Python's default installation prefix is + :file:`C:\\Python`, so the system configuration file is normally + :file:`C:\\Python\\Lib\\packaging\\packaging.cfg`. + +(5) + On Windows, if the :envvar:`HOME` environment variable is not defined, + :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will + be tried. Packaging uses the :func:`os.path.expanduser` function to do this. + + +.. _packaging-config-syntax: + +Syntax of configuration files +----------------------------- + +All Packaging configuration files share the same syntax. Options defined in +them are grouped into sections, and each Packaging command gets its own section. +Additionally, there's a ``global`` section for options that affect every command. +Sections consist of one or more lines containing a single option specified as +``option = value``. + +For example, here's a complete configuration file that forces all commands to +run quietly by default:: + + [global] + verbose = 0 + +If this was the system configuration file, it would affect all processing +of any Python module distribution by any user on the current system. If it was +installed as your personal configuration file (on systems that support them), +it would affect only module distributions processed by you. Lastly, if it was +used as the :file:`setup.cfg` for a particular module distribution, it would +affect that distribution only. + +.. XXX "(on systems that support them)" seems wrong info + +If you wanted to, you could override the default "build base" directory and +make the :command:`build\*` commands always forcibly rebuild all files with +the following:: + + [build] + build-base = blib + force = 1 + +which corresponds to the command-line arguments:: + + pysetup run build --build-base blib --force + +except that including the :command:`build` command on the command-line means +that command will be run. Including a particular command in configuration files +has no such implication; it only means that if the command is run, the options +for it in the configuration file will apply. (This is also true if you run +other commands that derive values from it.) + +You can find out the complete list of options for any command using the +:option:`--help` option, e.g.:: + + pysetup run build --help + +and you can find out the complete list of global options by using +:option:`--help` without a command:: + + pysetup run --help + +See also the "Reference" section of the "Distributing Python Modules" manual. + +.. XXX no links to the relevant section exist. + + +.. _packaging-building-ext: + +Building extensions: tips and tricks +==================================== + +Whenever possible, Packaging tries to use the configuration information made +available by the Python interpreter used to run `pysetup`. +For example, the same compiler and linker flags used to compile Python will also +be used for compiling extensions. Usually this will work well, but in +complicated situations this might be inappropriate. This section discusses how +to override the usual Packaging behaviour. + + +.. _packaging-tweak-flags: + +Tweaking compiler/linker flags +------------------------------ + +Compiling a Python extension written in C or C++ will sometimes require +specifying custom flags for the compiler and linker in order to use a particular +library or produce a special kind of object code. This is especially true if the +extension hasn't been tested on your platform, or if you're trying to +cross-compile Python. + +.. TODO update to new setup.cfg + +In the most general case, the extension author might have foreseen that +compiling the extensions would be complicated, and provided a :file:`Setup` file +for you to edit. This will likely only be done if the module distribution +contains many separate extension modules, or if they often require elaborate +sets of compiler flags in order to work. + +A :file:`Setup` file, if present, is parsed in order to get a list of extensions +to build. Each line in a :file:`Setup` describes a single module. Lines have +the following structure:: + + module ... [sourcefile ...] [cpparg ...] [library ...] + + +Let's examine each of the fields in turn. + +* *module* is the name of the extension module to be built, and should be a + valid Python identifier. You can't just change this in order to rename a module + (edits to the source code would also be needed), so this should be left alone. + +* *sourcefile* is anything that's likely to be a source code file, at least + judging by the filename. Filenames ending in :file:`.c` are assumed to be + written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are + assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed + to be in Objective C. + +* *cpparg* is an argument for the C preprocessor, and is anything starting with + :option:`-I`, :option:`-D`, :option:`-U` or :option:`-C`. + +* *library* is anything ending in :file:`.a` or beginning with :option:`-l` or + :option:`-L`. + +If a particular platform requires a special library on your platform, you can +add it by editing the :file:`Setup` file and running ``pysetup run build``. +For example, if the module defined by the line :: + + foo foomodule.c + +must be linked with the math library :file:`libm.a` on your platform, simply add +:option:`-lm` to the line:: + + foo foomodule.c -lm + +Arbitrary switches intended for the compiler or the linker can be supplied with +the :option:`-Xcompiler` *arg* and :option:`-Xlinker` *arg* options:: + + foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm + +The next option after :option:`-Xcompiler` and :option:`-Xlinker` will be +appended to the proper command line, so in the above example the compiler will +be passed the :option:`-o32` option, and the linker will be passed +:option:`-shared`. If a compiler option requires an argument, you'll have to +supply multiple :option:`-Xcompiler` options; for example, to pass ``-x c++`` +the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. + +Compiler flags can also be supplied through setting the :envvar:`CFLAGS` +environment variable. If set, the contents of :envvar:`CFLAGS` will be added to +the compiler flags specified in the :file:`Setup` file. + + +.. _packaging-non-ms-compilers: + +Using non-Microsoft compilers on Windows +---------------------------------------- + +.. sectionauthor:: Rene Liebscher + + + +Borland/CodeGear C++ +^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the necessary steps to use Packaging with the Borland +C++ compiler version 5.5. First you have to know that Borland's object file +format (OMF) is different from the format used by the Python version you can +download from the Python or ActiveState Web site. (Python is built with +Microsoft Visual C++, which uses COFF as the object file format.) For this +reason, you have to convert Python's library :file:`python25.lib` into the +Borland format. You can do this as follows: + +.. Should we mention that users have to create cfg-files for the compiler? +.. see also http://community.borland.com/article/0,1410,21205,00.html + +:: + + coff2omf python25.lib python25_bcpp.lib + +The :file:`coff2omf` program comes with the Borland compiler. The file +:file:`python25.lib` is in the :file:`Libs` directory of your Python +installation. If your extension uses other libraries (zlib, ...) you have to +convert them too. + +The converted files have to reside in the same directories as the normal +libraries. + +How does Packaging manage to use these libraries with their changed names? If +the extension needs a library (eg. :file:`foo`) Packaging checks first if it +finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then +uses this library. In the case it doesn't find such a special library it uses +the default name (:file:`foo.lib`.) [#]_ + +To let Packaging compile your extension with Borland, C++ you now have to +type:: + + pysetup run build --compiler bcpp + +If you want to use the Borland C++ compiler as the default, you could specify +this in your personal or system-wide configuration file for Packaging (see +section :ref:`packaging-config-files`.) + + +.. seealso:: + + `C++Builder Compiler `_ + Information about the free C++ compiler from Borland, including links to the + download pages. + + `Creating Python Extensions Using Borland's Free Compiler `_ + Document describing how to use Borland's free command-line C++ compiler to build + Python. + + +GNU C / Cygwin / MinGW +^^^^^^^^^^^^^^^^^^^^^^ + +This section describes the necessary steps to use Packaging with the GNU C/C++ +compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter +that was built with Cygwin, everything should work without any of these +following steps. + +Not all extensions can be built with MinGW or Cygwin, but many can. Extensions +most likely to not work are those that use C++ or depend on Microsoft Visual C +extensions. + +To let Packaging compile your extension with Cygwin, you have to type:: + + pysetup run build --compiler=cygwin + +and for Cygwin in no-cygwin mode [#]_ or for MinGW, type:: + + pysetup run build --compiler=mingw32 + +If you want to use any of these options/compilers as default, you should +consider writing it in your personal or system-wide configuration file for +Packaging (see section :ref:`packaging-config-files`.) + +Older Versions of Python and MinGW +"""""""""""""""""""""""""""""""""" +The following instructions only apply if you're using a version of Python +inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with +:file:`binutils-2.13.90-20030111-1`). + +These compilers require some special libraries. This task is more complex than +for Borland's C++, because there is no program to convert the library. First +you have to create a list of symbols which the Python DLL exports. (You can find +a good program for this task at +http://www.emmestech.com/software/pexports-0.43/download_pexports.html). + +.. I don't understand what the next line means. --amk + (inclusive the references on data structures.) + +:: + + pexports python25.dll > python25.def + +The location of an installed :file:`python25.dll` will depend on the +installation options and the version and language of Windows. In a "just for +me" installation, it will appear in the root of the installation directory. In +a shared installation, it will be located in the system directory. + +Then you can create from these information an import library for gcc. :: + + /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a + +The resulting library has to be placed in the same directory as +:file:`python25.lib`. (Should be the :file:`libs` directory under your Python +installation directory.) + +If your extension uses other libraries (zlib,...) you might have to convert +them too. The converted files have to reside in the same directories as the +normal libraries do. + + +.. seealso:: + + `Building Python modules on MS Windows platform with MinGW `_ + Information about building the required libraries for the MinGW + environment. + + +.. rubric:: Footnotes + +.. [#] This also means you could replace all existing COFF-libraries with + OMF-libraries of the same name. + +.. [#] Check http://sources.redhat.com/cygwin/ and http://www.mingw.org/ for + more information. + +.. [#] Then you have no POSIX emulation available, but you also don't need + :file:`cygwin1.dll`. diff --git a/Doc/install/pysetup-config.rst b/Doc/install/pysetup-config.rst new file mode 100644 index 0000000..0ce9022 --- /dev/null +++ b/Doc/install/pysetup-config.rst @@ -0,0 +1,44 @@ +.. _packaging-pysetup-config: + +===================== +Pysetup Configuration +===================== + +Pysetup supports two configuration files: :file:`.pypirc` and :file:`packaging.cfg`. + +.. FIXME integrate with configfile instead of duplicating + +Configuring indexes +------------------- + +You can configure additional indexes in :file:`.pypirc` to be used for index-related +operations. By default, all configured index-servers and package-servers will be used +in an additive fashion. To limit operations to specific indexes, use the :option:`--index` +and :option:`--package-server options`:: + + $ pysetup install --index pypi --package-server django some.project + +Adding indexes to :file:`.pypirc`:: + + [packaging] + index-servers = + pypi + other + + package-servers = + django + + [pypi] + repository: + username: + password: + + [other] + repository: + username: + password: + + [django] + repository: + username: + password: diff --git a/Doc/install/pysetup-servers.rst b/Doc/install/pysetup-servers.rst new file mode 100644 index 0000000..ddaaa5b --- /dev/null +++ b/Doc/install/pysetup-servers.rst @@ -0,0 +1,61 @@ +.. _packaging-pysetup-servers: + +=============== +Package Servers +=============== + +Pysetup supports installing Python packages from *Package Servers* in addition +to PyPI indexes and mirrors. + +Package Servers are simple directory listings of Python distributions. Directories +can be served via HTTP or a local file system. This is useful when you want to +dump source distributions in a directory and not worry about the full index structure. + +Serving distributions from Apache +--------------------------------- +:: + + $ mkdir -p /var/www/html/python/distributions + $ cp *.tar.gz /var/www/html/python/distributions/ + + + ServerAdmin webmaster@domain.com + DocumentRoot "/var/www/html/python" + ServerName python.example.org + ErrorLog logs/python.example.org-error.log + CustomLog logs/python.example.org-access.log common + Options Indexes FollowSymLinks MultiViews + DirectoryIndex index.html index.htm + + + Options Indexes FollowSymLinks MultiViews + Order allow,deny + Allow from all + + + +Add the Apache based distribution server to :file:`.pypirc`:: + + [packaging] + package-servers = + apache + + [apache] + repository: http://python.example.org/distributions/ + + +Serving distributions from a file system +---------------------------------------- +:: + + $ mkdir -p /data/python/distributions + $ cp *.tar.gz /data/python/distributions/ + +Add the directory to :file:`.pypirc`:: + + [packaging] + package-servers = + local + + [local] + repository: file:///data/python/distributions/ diff --git a/Doc/install/pysetup.rst b/Doc/install/pysetup.rst new file mode 100644 index 0000000..b88c8e1 --- /dev/null +++ b/Doc/install/pysetup.rst @@ -0,0 +1,163 @@ +.. _packaging-pysetup: + +================ +Pysetup Tutorial +================ + +Getting started +--------------- + +Pysetup is a simple script that supports the following features: + +- install, remove, list, and verify Python packages; +- search for available packages on PyPI or any *Simple Index*; +- verify installed packages (md5sum, installed files, version). + + +Finding out what's installed +---------------------------- + +Pysetup makes it easy to find out what Python packages are installed:: + + $ pysetup search virtualenv + virtualenv 1.6 at /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info + + $ pysetup search --all + pyverify 0.8.1 at /opt/python3.3/lib/python3.3/site-packages/pyverify-0.8.1.dist-info + virtualenv 1.6 at /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info + wsgiref 0.1.2 at /opt/python3.3/lib/python3.3/wsgiref.egg-info + ... + + +Installing a distribution +------------------------- + +Pysetup can install a Python project from the following sources: + +- PyPI and Simple Indexes; +- source directories containing a valid :file:`setup.py` or :file:`setup.cfg`; +- distribution source archives (:file:`project-1.0.tar.gz`, :file:`project-1.0.zip`); +- HTTP (http://host/packages/project-1.0.tar.gz). + + +Installing from PyPI and Simple Indexes:: + + $ pysetup install project + $ pysetup install project==1.0 + +Installing from a distribution source archive:: + + $ pysetup install project-1.0.tar.gz + +Installing from a source directory containing a valid :file:`setup.py` or +:file:`setup.cfg`:: + + $ cd path/to/source/directory + $ pysetup install + + $ pysetup install path/to/source/directory + +Installing from HTTP:: + + $ pysetup install http://host/packages/project-1.0.tar.gz + + +Retrieving metadata +------------------- + +You can gather metadata from two sources, a project's source directory or an +installed distribution. The `pysetup metadata` command can retrieve one or +more metadata fields using the `-f` option and a metadata field as the +argument. :: + + $ pysetup metadata virtualenv -f version -f name + Version: + 1.6 + Name: + virtualenv + + $ pysetup metadata virtualenv --all + Metadata-Version: + 1.0 + Name: + virtualenv + Version: + 1.6 + Platform: + UNKNOWN + Summary: + Virtual Python Environment builder + ... + +.. seealso:: + + There are three metadata versions, 1.0, 1.1, and 1.2. The following PEPs + describe specifics of the field names, and their semantics and usage. 1.0 + :PEP:`241`, 1.1 :PEP:`314`, and 1.2 :PEP:`345` + + +Removing a distribution +----------------------- + +You can remove one or more installed distributions using the `pysetup remove` +command:: + + $ pysetup remove virtualenv + removing 'virtualenv': + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/dependency_links.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/entry_points.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/not-zip-safe + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/PKG-INFO + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/SOURCES.txt + /opt/python3.3/lib/python3.3/site-packages/virtualenv-1.6-py3.3.egg-info/top_level.txt + Proceed (y/n)? y + success: removed 6 files and 1 dirs + +The optional '-y' argument auto confirms, skipping the conformation prompt:: + + $ pysetup remove virtualenv -y + + +Getting help +------------ + +All pysetup actions take the `-h` and `--help` options which prints the commands +help string to stdout. :: + + $ pysetup remove -h + Usage: pysetup remove dist [-y] + or: pysetup remove --help + + Uninstall a Python package. + + positional arguments: + dist installed distribution name + + optional arguments: + -y auto confirm package removal + +Getting a list of all pysetup actions and global options:: + + $ pysetup --help + Usage: pysetup [options] action [action_options] + + Actions: + run: Run one or several commands + metadata: Display the metadata of a project + install: Install a project + remove: Remove a project + search: Search for a project + graph: Display a graph + create: Create a Project + + To get more help on an action, use: + + pysetup action --help + + Global options: + --verbose (-v) run verbosely (default) + --quiet (-q) run quietly (turns verbosity off) + --dry-run (-n) don't actually do anything + --help (-h) show detailed help message + --no-user-cfg ignore pydistutils.cfg in your home directory + --version Display the version diff --git a/Doc/library/depgraph-output.png b/Doc/library/depgraph-output.png new file mode 100644 index 0000000..960bb1b Binary files /dev/null and b/Doc/library/depgraph-output.png differ diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst index 238b79d..f1a0f6b 100644 --- a/Doc/library/distutils.rst +++ b/Doc/library/distutils.rst @@ -27,3 +27,8 @@ This package is discussed in two separate chapters: modules into an existing Python installation. You do not need to be a Python programmer to read this manual. + +.. toctree:: + :hidden: + + ../distutils/index diff --git a/Doc/library/packaging-misc.rst b/Doc/library/packaging-misc.rst new file mode 100644 index 0000000..5e56247 --- /dev/null +++ b/Doc/library/packaging-misc.rst @@ -0,0 +1,27 @@ +.. temporary file for modules that don't need a dedicated file yet + +:mod:`packaging.errors` --- Packaging exceptions +================================================ + +.. module:: packaging.errors + :synopsis: Packaging exceptions. + + +Provides exceptions used by the Packaging modules. Note that Packaging modules +may raise standard exceptions; in particular, SystemExit is usually raised for +errors that are obviously the end-user's fault (e.g. bad command-line arguments). + +This module is safe to use in ``from ... import *`` mode; it only exports +symbols whose names start with ``Packaging`` and end with ``Error``. + + +:mod:`packaging.manifest` --- The Manifest class +================================================ + +.. module:: packaging.manifest + :synopsis: The Manifest class, used for poking about the file system and + building lists of files. + + +This module provides the :class:`Manifest` class, used for poking about the +filesystem and building lists of files. diff --git a/Doc/library/packaging.command.rst b/Doc/library/packaging.command.rst new file mode 100644 index 0000000..98835c0 --- /dev/null +++ b/Doc/library/packaging.command.rst @@ -0,0 +1,111 @@ +:mod:`packaging.command` --- Standard Packaging commands +======================================================== + +.. module:: packaging.command + :synopsis: Standard packaging commands. + + +This subpackage contains one module for each standard Packaging command, such as +:command:`build` or :command:`upload`. Each command is implemented as a +separate module, with the command name as the name of the module and of the +class defined therein. + + + +:mod:`packaging.command.cmd` --- Abstract base class for Packaging commands +=========================================================================== + +.. module:: packaging.command.cmd + :synopsis: Abstract base class for commands. + + +This module supplies the abstract base class :class:`Command`. This class is +subclassed by the modules in the packaging.command subpackage. + + +.. class:: Command(dist) + + Abstract base class for defining command classes, the "worker bees" of the + Packaging. A useful analogy for command classes is to think of them as + subroutines with local variables called *options*. The options are declared + in :meth:`initialize_options` and defined (given their final values) in + :meth:`finalize_options`, both of which must be defined by every command + class. The distinction between the two is necessary because option values + might come from the outside world (command line, config file, ...), and any + options dependent on other options must be computed after these outside + influences have been processed --- hence :meth:`finalize_options`. The body + of the subroutine, where it does all its work based on the values of its + options, is the :meth:`run` method, which must also be implemented by every + command class. + + The class constructor takes a single argument *dist*, a + :class:`~packaging.dist.Distribution` instance. + + +Creating a new Packaging command +-------------------------------- + +This section outlines the steps to create a new Packaging command. + +.. XXX the following paragraph is focused on the stdlib; expand it to document + how to write and register a command in third-party projects + +A new command lives in a module in the :mod:`packaging.command` package. There +is a sample template in that directory called :file:`command_template`. Copy +this file to a new module with the same name as the new command you're +implementing. This module should implement a class with the same name as the +module (and the command). So, for instance, to create the command +``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy +:file:`command_template` to :file:`packaging/command/peel_banana.py`, then edit +it so that it's implementing the class :class:`peel_banana`, a subclass of +:class:`Command`. It must define the following methods: + +.. method:: Command.initialize_options() + + Set default values for all the options that this command supports. Note that + these defaults may be overridden by other commands, by the setup script, by + config files, or by the command line. Thus, this is not the place to code + dependencies between options; generally, :meth:`initialize_options` + implementations are just a bunch of ``self.foo = None`` assignments. + + +.. method:: Command.finalize_options() + + Set final values for all the options that this command supports. This is + always called as late as possible, i.e. after any option assignments from the + command line or from other commands have been done. Thus, this is the place + to to code option dependencies: if *foo* depends on *bar*, then it is safe to + set *foo* from *bar* as long as *foo* still has the same value it was + assigned in :meth:`initialize_options`. + + +.. method:: Command.run() + + A command's raison d'etre: carry out the action it exists to perform, + controlled by the options initialized in :meth:`initialize_options`, + customized by other commands, the setup script, the command line, and config + files, and finalized in :meth:`finalize_options`. All terminal output and + filesystem interaction should be done by :meth:`run`. + + +Command classes may define this attribute: + + +.. attribute:: Command.sub_commands + + *sub_commands* formalizes the notion of a "family" of commands, + e.g. ``install_dist`` as the parent with sub-commands ``install_lib``, + ``install_headers``, etc. The parent of a family of commands defines + *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name, + predicate)``, with *command_name* a string and *predicate* a function, a + string or ``None``. *predicate* is a method of the parent command that + determines whether the corresponding command is applicable in the current + situation. (E.g. ``install_headers`` is only applicable if we have any C + header files to install.) If *predicate* is ``None``, that command is always + applicable. + + *sub_commands* is usually defined at the *end* of a class, because + predicates can be methods of the class, so they must already have been + defined. The canonical example is the :command:`install_dist` command. + +.. XXX document how to add a custom command to another one's subcommands diff --git a/Doc/library/packaging.compiler.rst b/Doc/library/packaging.compiler.rst new file mode 100644 index 0000000..dac6263 --- /dev/null +++ b/Doc/library/packaging.compiler.rst @@ -0,0 +1,672 @@ +:mod:`packaging.compiler` --- Compiler classes +============================================== + +.. module:: packaging.compiler + :synopsis: Compiler classes to build C/C++ extensions or libraries. + + +This subpackage contains an abstract base class representing a compiler and +concrete implementations for common compilers. The compiler classes should not +be instantiated directly, but created using the :func:`new_compiler` factory +function. Compiler types provided by Packaging are listed in +:ref:`packaging-standard-compilers`. + + +Public functions +---------------- + +.. function:: new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0) + + Factory function to generate an instance of some + :class:`~.ccompiler.CCompiler` subclass for the requested platform or + compiler type. + + If no argument is given for *plat* and *compiler*, the default compiler type + for the platform (:attr:`os.name`) will be used: ``'unix'`` for Unix and + Mac OS X, ``'msvc'`` for Windows. + + If *plat* is given, it must be one of ``'posix'``, ``'darwin'`` or ``'nt'``. + An invalid value will not raise an exception but use the default compiler + type for the current platform. + + .. XXX errors should never pass silently; this behavior is particularly + harmful when a compiler type is given as first argument + + If *compiler* is given, *plat* will be ignored, allowing you to get for + example a ``'unix'`` compiler object under Windows or an ``'msvc'`` compiler + under Unix. However, not all compiler types can be instantiated on every + platform. + + +.. function:: customize_compiler(compiler) + + Do any platform-specific customization of a CCompiler instance. Mainly + needed on Unix to plug in the information that varies across Unices and is + stored in CPython's Makefile. + + +.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) + + Generate linker options for searching library directories and linking with + specific libraries. *libraries* and *library_dirs* are, respectively, lists + of library names (not filenames!) and search directories. Returns a list of + command-line options suitable for use with some compiler (depending on the + two format strings passed in). + + +.. function:: gen_preprocess_options(macros, include_dirs) + + Generate C preprocessor options (:option:`-D`, :option:`-U`, :option:`-I`) as + used by at least two types of compilers: the typical Unix compiler and Visual + C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)`` + means undefine (:option:`-U`) macro *name*, and ``(name, value)`` means + define (:option:`-D`) macro *name* to *value*. *include_dirs* is just a list + of directory names to be added to the header file search path (:option:`-I`). + Returns a list of command-line options suitable for either Unix compilers or + Visual C++. + + +.. function:: get_default_compiler(osname, platform) + + Determine the default compiler to use for the given platform. + + *osname* should be one of the standard Python OS names (i.e. the ones + returned by ``os.name``) and *platform* the common value returned by + ``sys.platform`` for the platform in question. + + The default values are ``os.name`` and ``sys.platform``. + + +.. function:: set_compiler(location) + + Add or change a compiler + + +.. function:: show_compilers() + + Print list of available compilers (used by the :option:`--help-compiler` + options to :command:`build`, :command:`build_ext`, :command:`build_clib`). + + +.. _packaging-standard-compilers: + +Standard compilers +------------------ + +Concrete subclasses of :class:`~.ccompiler.CCompiler` are provided in submodules +of the :mod:`packaging.compiler` package. You do not need to import them, using +:func:`new_compiler` is the public API to use. This table documents the +standard compilers; be aware that they can be replaced by other classes on your +platform. + +=============== ======================================================== ======= +name description notes +=============== ======================================================== ======= +``'unix'`` typical Unix-style command-line C compiler [#]_ +``'msvc'`` Microsoft compiler [#]_ +``'bcpp'`` Borland C++ compiler +``'cygwin'`` Cygwin compiler (Windows port of GCC) +``'mingw32'`` Mingw32 port of GCC (same as Cygwin in no-Cygwin mode) +=============== ======================================================== ======= + + +.. [#] The Unix compiler class assumes this behavior: + + * macros defined with :option:`-Dname[=value]` + + * macros undefined with :option:`-Uname` + + * include search directories specified with :option:`-Idir` + + * libraries specified with :option:`-llib` + + * library search directories specified with :option:`-Ldir` + + * compile handled by :program:`cc` (or similar) executable with + :option:`-c` option: compiles :file:`.c` to :file:`.o` + + * link static library handled by :program:`ar` command (possibly with + :program:`ranlib`) + + * link shared library handled by :program:`cc` :option:`-shared` + + +.. [#] On Windows, extension modules typically need to be compiled with the same + compiler that was used to compile CPython (for example Microsoft Visual + Studio .NET 2003 for CPython 2.4 and 2.5). The AMD64 and Itanium + binaries are created using the Platform SDK. + + Under the hood, there are actually two different subclasses of + :class:`~.ccompiler.CCompiler` defined: one is compatible with MSVC 2005 + and 2008, the other works with older versions. This should not be a + concern for regular use of the functions in this module. + + Packaging will normally choose the right compiler, linker etc. on its + own. To override this choice, the environment variables + *DISTUTILS_USE_SDK* and *MSSdk* must be both set. *MSSdk* indicates that + the current environment has been setup by the SDK's ``SetEnv.Cmd`` + script, or that the environment variables had been registered when the + SDK was installed; *DISTUTILS_USE_SDK* indicates that the user has made + an explicit choice to override the compiler selection done by Packaging. + + .. TODO document the envvars in Doc/using and the man page + + +:mod:`packaging.compiler.ccompiler` --- CCompiler base class +============================================================ + +.. module:: packaging.compiler.ccompiler + :synopsis: Abstract CCompiler class. + + +This module provides the abstract base class for the :class:`CCompiler` +classes. A :class:`CCompiler` instance can be used for all the compile and +link steps needed to build a single project. Methods are provided to set +options for the compiler --- macro definitions, include directories, link path, +libraries and the like. + +.. class:: CCompiler([verbose=0, dry_run=0, force=0]) + + The abstract base class :class:`CCompiler` defines the interface that must be + implemented by real compiler classes. The class also has some utility + methods used by several compiler classes. + + The basic idea behind a compiler abstraction class is that each instance can + be used for all the compile/link steps in building a single project. Thus, + attributes common to all of those compile and link steps --- include + directories, macros to define, libraries to link against, etc. --- are + attributes of the compiler instance. To allow for variability in how + individual files are treated, most of those attributes may be varied on a + per-compilation or per-link basis. + + The constructor for each subclass creates an instance of the Compiler object. + Flags are *verbose* (show verbose output), *dry_run* (don't actually execute + the steps) and *force* (rebuild everything, regardless of dependencies). All + of these flags default to ``0`` (off). Note that you probably don't want to + instantiate :class:`CCompiler` or one of its subclasses directly - use the + :func:`packaging.CCompiler.new_compiler` factory function instead. + + The following methods allow you to manually alter compiler options for the + instance of the Compiler class. + + + .. method:: CCompiler.add_include_dir(dir) + + Add *dir* to the list of directories that will be searched for header + files. The compiler is instructed to search directories in the order in + which they are supplied by successive calls to :meth:`add_include_dir`. + + + .. method:: CCompiler.set_include_dirs(dirs) + + Set the list of directories that will be searched to *dirs* (a list of + strings). Overrides any preceding calls to :meth:`add_include_dir`; + subsequent calls to :meth:`add_include_dir` add to the list passed to + :meth:`set_include_dirs`. This does not affect any list of standard + include directories that the compiler may search by default. + + + .. method:: CCompiler.add_library(libname) + + Add *libname* to the list of libraries that will be included in all links + driven by this compiler object. Note that *libname* should *not* be the + name of a file containing a library, but the name of the library itself: + the actual filename will be inferred by the linker, the compiler, or the + compiler class (depending on the platform). + + The linker will be instructed to link against libraries in the order they + were supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is + perfectly valid to duplicate library names; the linker will be instructed + to link against libraries as many times as they are mentioned. + + + .. method:: CCompiler.set_libraries(libnames) + + Set the list of libraries to be included in all links driven by this + compiler object to *libnames* (a list of strings). This does not affect + any standard system libraries that the linker may include by default. + + + .. method:: CCompiler.add_library_dir(dir) + + Add *dir* to the list of directories that will be searched for libraries + specified to :meth:`add_library` and :meth:`set_libraries`. The linker + will be instructed to search for libraries in the order they are supplied + to :meth:`add_library_dir` and/or :meth:`set_library_dirs`. + + + .. method:: CCompiler.set_library_dirs(dirs) + + Set the list of library search directories to *dirs* (a list of strings). + This does not affect any standard library search path that the linker may + search by default. + + + .. method:: CCompiler.add_runtime_library_dir(dir) + + Add *dir* to the list of directories that will be searched for shared + libraries at runtime. + + + .. method:: CCompiler.set_runtime_library_dirs(dirs) + + Set the list of directories to search for shared libraries at runtime to + *dirs* (a list of strings). This does not affect any standard search path + that the runtime linker may search by default. + + + .. method:: CCompiler.define_macro(name[, value=None]) + + Define a preprocessor macro for all compilations driven by this compiler + object. The optional parameter *value* should be a string; if it is not + supplied, then the macro will be defined without an explicit value and the + exact outcome depends on the compiler used (XXX true? does ANSI say + anything about this?) + + + .. method:: CCompiler.undefine_macro(name) + + Undefine a preprocessor macro for all compilations driven by this compiler + object. If the same macro is defined by :meth:`define_macro` and + undefined by :meth:`undefine_macro` the last call takes precedence + (including multiple redefinitions or undefinitions). If the macro is + redefined/undefined on a per-compilation basis (i.e. in the call to + :meth:`compile`), then that takes precedence. + + + .. method:: CCompiler.add_link_object(object) + + Add *object* to the list of object files (or analogues, such as explicitly + named library files or the output of "resource compilers") to be included + in every link driven by this compiler object. + + + .. method:: CCompiler.set_link_objects(objects) + + Set the list of object files (or analogues) to be included in every link + to *objects*. This does not affect any standard object files that the + linker may include by default (such as system libraries). + + The following methods implement methods for autodetection of compiler + options, providing some functionality similar to GNU :program:`autoconf`. + + + .. method:: CCompiler.detect_language(sources) + + Detect the language of a given file, or list of files. Uses the instance + attributes :attr:`language_map` (a dictionary), and :attr:`language_order` + (a list) to do the job. + + + .. method:: CCompiler.find_library_file(dirs, lib[, debug=0]) + + Search the specified list of directories for a static or shared library file + *lib* and return the full path to that file. If *debug* is true, look for a + debugging version (if that makes sense on the current platform). Return + ``None`` if *lib* wasn't found in any of the specified directories. + + + .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None]) + + Return a boolean indicating whether *funcname* is supported on the current + platform. The optional arguments can be used to augment the compilation + environment by providing additional include files and paths and libraries and + paths. + + + .. method:: CCompiler.library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + libraries. + + + .. method:: CCompiler.library_option(lib) + + Return the compiler option to add *dir* to the list of libraries linked into the + shared library or executable. + + + .. method:: CCompiler.runtime_library_dir_option(dir) + + Return the compiler option to add *dir* to the list of directories searched for + runtime libraries. + + + .. method:: CCompiler.set_executables(**args) + + Define the executables (and options for them) that will be run to perform the + various stages of compilation. The exact set of executables that may be + specified here depends on the compiler class (via the 'executables' class + attribute), but most will have: + + +--------------+------------------------------------------+ + | attribute | description | + +==============+==========================================+ + | *compiler* | the C/C++ compiler | + +--------------+------------------------------------------+ + | *linker_so* | linker used to create shared objects and | + | | libraries | + +--------------+------------------------------------------+ + | *linker_exe* | linker used to create binary executables | + +--------------+------------------------------------------+ + | *archiver* | static library creator | + +--------------+------------------------------------------+ + + On platforms with a command line (Unix, DOS/Windows), each of these is a string + that will be split into executable name and (optional) list of arguments. + (Splitting the string is done similarly to how Unix shells operate: words are + delimited by spaces, but quotes and backslashes can override this. See + :func:`packaging.util.split_quoted`.) + + The following methods invoke stages in the build process. + + + .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None]) + + Compile one or more source files. Generates object files (e.g. transforms a + :file:`.c` file to a :file:`.o` file.) + + *sources* must be a list of filenames, most likely C/C++ files, but in reality + anything that can be handled by a particular compiler and compiler class (e.g. + an ``'msvc'`` compiler` can handle resource files in *sources*). Return a list of + object filenames, one per source filename in *sources*. Depending on the + implementation, not all source files will necessarily be compiled, but all + corresponding object filenames will be returned. + + If *output_dir* is given, object files will be put under it, while retaining + their original path component. That is, :file:`foo/bar.c` normally compiles to + :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then + it would compile to :file:`build/foo/bar.o`. + + *macros*, if given, must be a list of macro definitions. A macro definition is + either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines + a macro; if the value is ``None``, the macro is defined without an explicit + value. The 1-tuple case undefines a macro. Later + definitions/redefinitions/undefinitions take precedence. + + *include_dirs*, if given, must be a list of strings, the directories to add to + the default include file search path for this compilation only. + + *debug* is a boolean; if true, the compiler will be instructed to output debug + symbols in (or alongside) the object file(s). + + *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms + that have the notion of a command line (e.g. Unix, DOS/Windows), they are most + likely lists of strings: extra command-line arguments to prepend/append to the + compiler command line. On other platforms, consult the implementation class + documentation. In any event, they are intended as an escape hatch for those + occasions when the abstract compiler framework doesn't cut the mustard. + + *depends*, if given, is a list of filenames that all targets depend on. If a + source file is older than any file in depends, then the source file will be + recompiled. This supports dependency tracking, but only at a coarse + granularity. + + Raises :exc:`CompileError` on failure. + + + .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None]) + + Link a bunch of stuff together to create a static library file. The "bunch of + stuff" consists of the list of object files supplied as *objects*, the extra + object files supplied to :meth:`add_link_object` and/or + :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or + :meth:`set_libraries`, and the libraries supplied as *libraries* (if any). + + *output_libname* should be a library name, not a filename; the filename will be + inferred from the library name. *output_dir* is the directory where the library + file will be put. XXX defaults to what? + + *debug* is a boolean; if true, debugging information will be included in the + library (note that on most platforms, it is the compile step where this matters: + the *debug* flag is included here just for consistency). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LibError` on failure. + + + .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a bunch of stuff together to create an executable or shared library file. + + The "bunch of stuff" consists of the list of object files supplied as *objects*. + *output_filename* should be a filename. If *output_dir* is supplied, + *output_filename* is relative to it (i.e. *output_filename* can provide + directory components if needed). + + *libraries* is a list of libraries to link against. These are library names, + not filenames, since they're translated into filenames in a platform-specific + way (e.g. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on + DOS/Windows). However, they can include a directory component, which means the + linker will look in that specific directory rather than searching all the normal + locations. + + *library_dirs*, if supplied, should be a list of directories to search for + libraries that were specified as bare library names (i.e. no directory + component). These are on top of the system default and those supplied to + :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs* + is a list of directories that will be embedded into the shared library and used + to search for other shared libraries that \*it\* depends on at run-time. (This + may only be relevant on Unix.) + + *export_symbols* is a list of symbols that the shared library will export. + (This appears to be relevant only on Windows.) + + *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the + slight distinction that it actually matters on most platforms (as opposed to + :meth:`create_static_lib`, which includes a *debug* flag mostly for form's + sake). + + *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of + course that they supply command-line arguments for the particular linker being + used). + + *target_lang* is the target language for which the given objects are being + compiled. This allows specific linkage time treatment of certain languages. + + Raises :exc:`LinkError` on failure. + + + .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None]) + + Link an executable. *output_progname* is the name of the file executable, while + *objects* are a list of object filenames to link in. Other arguments are as for + the :meth:`link` method. + + + .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared library. *output_libname* is the name of the output library, + while *objects* is a list of object filenames to link in. Other arguments are + as for the :meth:`link` method. + + + .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) + + Link a shared object. *output_filename* is the name of the shared object that + will be created, while *objects* is a list of object filenames to link in. + Other arguments are as for the :meth:`link` method. + + + .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None]) + + Preprocess a single C/C++ source file, named in *source*. Output will be written + to file named *output_file*, or *stdout* if *output_file* not supplied. + *macros* is a list of macro definitions as for :meth:`compile`, which will + augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`. + *include_dirs* is a list of directory names that will be added to the default + list, in the same way as :meth:`add_include_dir`. + + Raises :exc:`PreprocessError` on failure. + + The following utility methods are defined by the :class:`CCompiler` class, for + use by the various concrete subclasses. + + + .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir='']) + + Returns the filename of the executable for the given *basename*. Typically for + non-Windows platforms this is the same as the basename, while Windows will get + a :file:`.exe` added. + + + .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir='']) + + Returns the filename for the given library name on the current platform. On Unix + a library with *lib_type* of ``'static'`` will typically be of the form + :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form + :file:`liblibname.so`. + + + .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir='']) + + Returns the name of the object files for the given source files. + *source_filenames* should be a list of filenames. + + + .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir='']) + + Returns the name of a shared object file for the given file name *basename*. + + + .. method:: CCompiler.execute(func, args[, msg=None, level=1]) + + Invokes :func:`packaging.util.execute` This method invokes a Python function + *func* with the given arguments *args*, after logging and taking into account + the *dry_run* flag. XXX see also. + + + .. method:: CCompiler.spawn(cmd) + + Invokes :func:`packaging.util.spawn`. This invokes an external process to run + the given command. XXX see also. + + + .. method:: CCompiler.mkpath(name[, mode=511]) + + Invokes :func:`packaging.dir_util.mkpath`. This creates a directory and any + missing ancestor directories. XXX see also. + + + .. method:: CCompiler.move_file(src, dst) + + Invokes :meth:`packaging.file_util.move_file`. Renames *src* to *dst*. XXX see + also. + + +:mod:`packaging.compiler.extension` --- The Extension class +=========================================================== + +.. module:: packaging.compiler.extension + :synopsis: Class used to represent C/C++ extension modules. + + +This module provides the :class:`Extension` class, used to represent C/C++ +extension modules. + +.. class:: Extension + + The Extension class describes a single C or C++ extension module. It accepts + the following keyword arguments in its constructor + + +------------------------+--------------------------------+---------------------------+ + | argument name | value | type | + +========================+================================+===========================+ + | *name* | the full name of the | string | + | | extension, including any | | + | | packages --- i.e. *not* a | | + | | filename or pathname, but | | + | | Python dotted name | | + +------------------------+--------------------------------+---------------------------+ + | *sources* | list of source filenames, | string | + | | relative to the distribution | | + | | root (where the setup script | | + | | lives), in Unix form (slash- | | + | | separated) for portability. | | + | | Source files may be C, C++, | | + | | SWIG (.i), platform-specific | | + | | resource files, or whatever | | + | | else is recognized by the | | + | | :command:`build_ext` command | | + | | as source for a Python | | + | | extension. | | + +------------------------+--------------------------------+---------------------------+ + | *include_dirs* | list of directories to search | string | + | | for C/C++ header files (in | | + | | Unix form for portability) | | + +------------------------+--------------------------------+---------------------------+ + | *define_macros* | list of macros to define; each | (string, string) tuple or | + | | macro is defined using a | (name, ``None``) | + | | 2-tuple ``(name, value)``, | | + | | where *value* is | | + | | either the string to define it | | + | | to or ``None`` to define it | | + | | without a particular value | | + | | (equivalent of ``#define FOO`` | | + | | in source or :option:`-DFOO` | | + | | on Unix C compiler command | | + | | line) | | + +------------------------+--------------------------------+---------------------------+ + | *undef_macros* | list of macros to undefine | string | + | | explicitly | | + +------------------------+--------------------------------+---------------------------+ + | *library_dirs* | list of directories to search | string | + | | for C/C++ libraries at link | | + | | time | | + +------------------------+--------------------------------+---------------------------+ + | *libraries* | list of library names (not | string | + | | filenames or paths) to link | | + | | against | | + +------------------------+--------------------------------+---------------------------+ + | *runtime_library_dirs* | list of directories to search | string | + | | for C/C++ libraries at run | | + | | time (for shared extensions, | | + | | this is when the extension is | | + | | loaded) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_objects* | list of extra files to link | string | + | | with (e.g. object files not | | + | | implied by 'sources', static | | + | | library that must be | | + | | explicitly specified, binary | | + | | resource files, etc.) | | + +------------------------+--------------------------------+---------------------------+ + | *extra_compile_args* | any extra platform- and | string | + | | compiler-specific information | | + | | to use when compiling the | | + | | source files in 'sources'. For | | + | | platforms and compilers where | | + | | a command line makes sense, | | + | | this is typically a list of | | + | | command-line arguments, but | | + | | for other platforms it could | | + | | be anything. | | + +------------------------+--------------------------------+---------------------------+ + | *extra_link_args* | any extra platform- and | string | + | | compiler-specific information | | + | | to use when linking object | | + | | files together to create the | | + | | extension (or to create a new | | + | | static Python interpreter). | | + | | Similar interpretation as for | | + | | 'extra_compile_args'. | | + +------------------------+--------------------------------+---------------------------+ + | *export_symbols* | list of symbols to be exported | string | + | | from a shared extension. Not | | + | | used on all platforms, and not | | + | | generally necessary for Python | | + | | extensions, which typically | | + | | export exactly one symbol: | | + | | ``init`` + extension_name. | | + +------------------------+--------------------------------+---------------------------+ + | *depends* | list of files that the | string | + | | extension depends on | | + +------------------------+--------------------------------+---------------------------+ + | *language* | extension language (i.e. | string | + | | ``'c'``, ``'c++'``, | | + | | ``'objc'``). Will be detected | | + | | from the source extensions if | | + | | not provided. | | + +------------------------+--------------------------------+---------------------------+ diff --git a/Doc/library/packaging.database.rst b/Doc/library/packaging.database.rst new file mode 100644 index 0000000..2a8c39f --- /dev/null +++ b/Doc/library/packaging.database.rst @@ -0,0 +1,324 @@ +:mod:`packaging.database` --- Database of installed distributions +================================================================= + +.. module:: packaging.database + :synopsis: Functions to query and manipulate installed distributions. + + +This module provides an implementation of :PEP:`376`. It was originally +intended to land in :mod:`pkgutil`, but with the inclusion of Packaging in the +standard library, it was thought best to include it in a submodule of +:mod:`packaging`, leaving :mod:`pkgutil` to deal with imports. + +Installed Python distributions are represented by instances of +:class:`Distribution`, or :class:`EggInfoDistribution` for legacy egg formats. +Most functions also provide an extra argument ``use_egg_info`` to take legacy +distributions into account. + + +Classes representing installed distributions +-------------------------------------------- + +.. class:: Distribution(path) + + Class representing an installed distribution. It is different from + :class:`packaging.dist.Distribution` which holds the list of files, the + metadata and options during the run of a Packaging command. + + Instantiate with the *path* to a ``.dist-info`` directory. Instances can be + compared and sorted. Other available methods are: + + .. XXX describe how comparison works + + .. method:: get_distinfo_file(path, binary=False) + + Return a read-only file object for a file located at + :file:`{project-version}.dist-info/path}`. *path* should be a + ``'/'``-separated path relative to the ``.dist-info`` directory or an + absolute path; if it is an absolute path and doesn't start with the path + to the :file:`.dist-info` directory, a :class:`PackagingError` is raised. + + If *binary* is ``True``, the file is opened in binary mode. + + .. method:: get_resource_path(relative_path) + + .. TODO + + .. method:: list_distinfo_files(local=False) + + Return an iterator over all files located in the :file:`.dist-info` + directory. If *local* is ``True``, each returned path is transformed into + a local absolute path, otherwise the raw value found in the :file:`RECORD` + file is returned. + + .. method:: list_installed_files(local=False) + + Iterate over the files installed with the distribution and registered in + the :file:`RECORD` file and yield a tuple ``(path, md5, size)`` for each + line. If *local* is ``True``, the returned path is transformed into a + local absolute path, otherwise the raw value is returned. + + A local absolute path is an absolute path in which occurrences of ``'/'`` + have been replaced by :data:`os.sep`. + + .. method:: uses(path) + + Check whether *path* was installed by this distribution (i.e. if the path + is present in the :file:`RECORD` file). *path* can be a local absolute + path or a relative ``'/'``-separated path. Returns a boolean. + + Available attributes: + + .. attribute:: metadata + + Instance of :class:`packaging.metadata.Metadata` filled with the contents + of the :file:`{project-version}.dist-info/METADATA` file. + + .. attribute:: name + + Shortcut for ``metadata['Name']``. + + .. attribute:: version + + Shortcut for ``metadata['Version']``. + + .. attribute:: requested + + Boolean indicating whether this distribution was requested by the user of + automatically installed as a dependency. + + +.. class:: EggInfoDistribution(path) + + Class representing a legacy distribution. It is compatible with distutils' + and setuptools' :file:`.egg-info` and :file:`.egg` files and directories. + + .. FIXME should be named EggDistribution + + Instantiate with the *path* to an egg file or directory. Instances can be + compared and sorted. Other available methods are: + + .. method:: list_installed_files(local=False) + + .. method:: uses(path) + + Available attributes: + + .. attribute:: metadata + + Instance of :class:`packaging.metadata.Metadata` filled with the contents + of the :file:`{project-version}.egg-info/PKG-INFO` or + :file:`{project-version}.egg` file. + + .. attribute:: name + + Shortcut for ``metadata['Name']``. + + .. attribute:: version + + Shortcut for ``metadata['Version']``. + + +Functions to work with the database +----------------------------------- + +.. function:: get_distribution(name, use_egg_info=False, paths=None) + + Return an instance of :class:`Distribution` or :class:`EggInfoDistribution` + for the first installed distribution matching *name*. Egg distributions are + considered only if *use_egg_info* is true; if both a dist-info and an egg + file are found, the dist-info prevails. The directories to be searched are + given in *paths*, which defaults to :data:`sys.path`. Return ``None`` if no + matching distribution is found. + + .. FIXME param should be named use_egg + + +.. function:: get_distributions(use_egg_info=False, paths=None) + + Return an iterator of :class:`Distribution` instances for all installed + distributions found in *paths* (defaults to :data:`sys.path`). If + *use_egg_info* is true, also return instances of :class:`EggInfoDistribution` + for legacy distributions found. + + +.. function:: get_file_users(path) + + Return an iterator over all distributions using *path*, a local absolute path + or a relative ``'/'``-separated path. + + .. XXX does this work with prefixes or full file path only? + + +.. function:: obsoletes_distribution(name, version=None, use_egg_info=False) + + Return an iterator over all distributions that declare they obsolete *name*. + *version* is an optional argument to match only specific releases (see + :mod:`packaging.version`). If *use_egg_info* is true, legacy egg + distributions will be considered as well. + + +.. function:: provides_distribution(name, version=None, use_egg_info=False) + + Return an iterator over all distributions that declare they provide *name*. + *version* is an optional argument to match only specific releases (see + :mod:`packaging.version`). If *use_egg_info* is true, legacy egg + distributions will be considered as well. + + +Utility functions +----------------- + +.. function:: distinfo_dirname(name, version) + + Escape *name* and *version* into a filename-safe form and return the + directory name built from them, for example + :file:`{safename}-{safeversion}.dist-info.` In *name*, runs of + non-alphanumeric characters are replaced with one ``'_'``; in *version*, + spaces become dots, and runs of other non-alphanumeric characters (except + dots) a replaced by one ``'-'``. + + .. XXX wth spaces in version numbers? + +For performance purposes, the list of distributions is being internally +cached. Caching is enabled by default, but you can control it with these +functions: + +.. function:: clear_cache() + + Clear the cache. + +.. function:: disable_cache() + + Disable the cache, without clearing it. + +.. function:: enable_cache() + + Enable the internal cache, without clearing it. + + +Examples +-------- + +Print all information about a distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given a path to a ``.dist-info`` distribution, we shall print out all +information that can be obtained using functions provided in this module:: + + import sys + import packaging.database + + path = input() + # first create the Distribution instance + try: + dist = packaging.database.Distribution(path) + except IOError: + sys.exit('No such distribution') + + print('Information about %r' % dist.name) + print() + + print('Files') + print('=====') + for path, md5, size in dist.list_installed_files(): + print('* Path: %s' % path) + print(' Hash %s, Size: %s bytes' % (md5, size)) + print() + + print('Metadata') + print('========') + for key, value in dist.metadata.items(): + print('%20s: %s' % (key, value)) + print() + + print('Extra') + print('=====') + if dist.requested: + print('* It was installed by user request') + else: + print('* It was installed as a dependency') + +If we save the script above as ``print_info.py``, we can use it to extract +information from a :file:`.dist-info` directory. By typing in the console: + +.. code-block:: sh + + $ echo /tmp/choxie/choxie-2.0.0.9.dist-info | python3 print_info.py + +we get the following output: + +.. code-block:: none + + Information about 'choxie' + + Files + ===== + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/truffles.py + Hash 5e052db6a478d06bad9ae033e6bc08af, Size: 111 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/chocolate.py + Hash ac56bf496d8d1d26f866235b95f31030, Size: 214 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9/choxie/__init__.py + Hash 416aab08dfa846f473129e89a7625bbc, Size: 25 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/INSTALLER + Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/METADATA + Hash 696a209967fef3c8b8f5a7bb10386385, Size: 225 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/REQUESTED + Hash d41d8cd98f00b204e9800998ecf8427e, Size: 0 bytes + * Path: ../tmp/distutils2/tests/fake_dists/choxie-2.0.0.9.dist-info/RECORD + Hash None, Size: None bytes + + Metadata + ======== + Metadata-Version: 1.2 + Name: choxie + Version: 2.0.0.9 + Platform: [] + Supported-Platform: UNKNOWN + Summary: Chocolate with a kick! + Description: UNKNOWN + Keywords: [] + Home-page: UNKNOWN + Author: UNKNOWN + Author-email: UNKNOWN + Maintainer: UNKNOWN + Maintainer-email: UNKNOWN + License: UNKNOWN + Classifier: [] + Download-URL: UNKNOWN + Obsoletes-Dist: ['truffles (<=0.8,>=0.5)', 'truffles (<=0.9,>=0.6)'] + Project-URL: [] + Provides-Dist: ['truffles (1.0)'] + Requires-Dist: ['towel-stuff (0.1)'] + Requires-Python: UNKNOWN + Requires-External: [] + + Extra + ===== + * It was installed as a dependency + + +Find out obsoleted distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Now, we take tackle a different problem, we are interested in finding out +which distributions have been obsoleted. This can be easily done as follows:: + + import packaging.database + + # iterate over all distributions in the system + for dist in packaging.database.get_distributions(): + name, version = dist.name, dist.version + # find out which distributions obsolete this name/version combination + replacements = packaging.database.obsoletes_distribution(name, version) + if replacements: + print('%r %s is obsoleted by' % (name, version), + ', '.join(repr(r.name) for r in replacements)) + +This is how the output might look like: + +.. code-block:: none + + 'strawberry' 0.6 is obsoleted by 'choxie' + 'grammar' 1.0a4 is obsoleted by 'towel-stuff' diff --git a/Doc/library/packaging.depgraph.rst b/Doc/library/packaging.depgraph.rst new file mode 100644 index 0000000..c384788 --- /dev/null +++ b/Doc/library/packaging.depgraph.rst @@ -0,0 +1,199 @@ +:mod:`packaging.depgraph` --- Dependency graph builder +====================================================== + +.. module:: packaging.depgraph + :synopsis: Graph builder for dependencies between releases. + + +This module provides the means to analyse the dependencies between various +distributions and to create a graph representing these dependency relationships. +In this document, "distribution" refers to an instance of +:class:`packaging.database.Distribution` or +:class:`packaging.database.EggInfoDistribution`. + +.. XXX terminology problem with dist vs. release: dists are installed, but deps + use releases + +.. XXX explain how to use it with dists not installed: Distribution can only be + instantiated with a path, but this module is useful for remote dist too + +.. XXX functions should accept and return iterators, not lists + + +The :class:`DependencyGraph` class +---------------------------------- + +.. class:: DependencyGraph + + Represent a dependency graph between releases. The nodes are distribution + instances; the edge model dependencies. An edge from ``a`` to ``b`` means + that ``a`` depends on ``b``. + + .. method:: add_distribution(distribution) + + Add *distribution* to the graph. + + .. method:: add_edge(x, y, label=None) + + Add an edge from distribution *x* to distribution *y* with the given + *label* (string). + + .. method:: add_missing(distribution, requirement) + + Add a missing *requirement* (string) for the given *distribution*. + + .. method:: repr_node(dist, level=1) + + Print a subgraph starting from *dist*. *level* gives the depth of the + subgraph. + + Direct access to the graph nodes and edges is provided through these + attributes: + + .. attribute:: adjacency_list + + Dictionary mapping distributions to a list of ``(other, label)`` tuples + where ``other`` is a distribution and the edge is labeled with ``label`` + (i.e. the version specifier, if such was provided). + + .. attribute:: reverse_list + + Dictionary mapping distributions to a list of predecessors. This allows + efficient traversal. + + .. attribute:: missing + + Dictionary mapping distributions to a list of requirements that were not + provided by any distribution. + + +Auxiliary functions +------------------- + +.. function:: dependent_dists(dists, dist) + + Recursively generate a list of distributions from *dists* that are dependent + on *dist*. + + .. XXX what does member mean here: "dist is a member of *dists* for which we + are interested" + +.. function:: generate_graph(dists) + + Generate a :class:`DependencyGraph` from the given list of distributions. + + .. XXX make this alternate constructor a DepGraph classmethod or rename; + 'generate' can suggest it creates a file or an image, use 'make' + +.. function:: graph_to_dot(graph, f, skip_disconnected=True) + + Write a DOT output for the graph to the file-like object *f*. + + If *skip_disconnected* is true, all distributions that are not dependent on + any other distribution are skipped. + + .. XXX why is this not a DepGraph method? + + +Example Usage +------------- + +Depict all dependenciess in the system +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +First, we shall generate a graph of all the distributions on the system +and then create an image out of it using the tools provided by +`Graphviz `_:: + + from packaging.database import get_distributions + from packaging.depgraph import generate_graph + + dists = list(get_distributions()) + graph = generate_graph(dists) + +It would be interesting to print out the missing requirements. This can be done +as follows:: + + for dist, reqs in graph.missing.items(): + if reqs: + reqs = ' ,'.join(repr(req) for req in reqs) + print('Missing dependencies for %r: %s' % (dist.name, reqs)) + +Example output is: + +.. code-block:: none + + Missing dependencies for 'TurboCheetah': 'Cheetah' + Missing dependencies for 'TurboGears': 'ConfigObj', 'DecoratorTools', 'RuleDispatch' + Missing dependencies for 'jockey': 'PyKDE4.kdecore', 'PyKDE4.kdeui', 'PyQt4.QtCore', 'PyQt4.QtGui' + Missing dependencies for 'TurboKid': 'kid' + Missing dependencies for 'TurboJson: 'DecoratorTools', 'RuleDispatch' + +Now, we proceed with generating a graphical representation of the graph. First +we write it to a file, and then we generate a PNG image using the +:program:`dot` command-line tool:: + + from packaging.depgraph import graph_to_dot + with open('output.dot', 'w') as f: + # only show the interesting distributions, skipping the disconnected ones + graph_to_dot(graph, f, skip_disconnected=True) + +We can create the final picture using: + +.. code-block:: sh + + $ dot -Tpng output.dot > output.png + +An example result is: + +.. figure:: depgraph-output.png + :alt: Example PNG output from packaging.depgraph and dot + +If you want to include egg distributions as well, then the code requires only +one change, namely the line:: + + dists = list(packaging.database.get_distributions()) + +has to be replaced with:: + + dists = list(packaging.database.get_distributions(use_egg_info=True)) + +On many platforms, a richer graph is obtained because at the moment most +distributions are provided in the egg rather than the new standard +``.dist-info`` format. + +.. XXX missing image + + An example of a more involved graph for illustrative reasons can be seen + here: + + .. image:: depgraph_big.png + + +List all dependent distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We will list all distributions that are dependent on some given distibution. +This time, egg distributions will be considered as well:: + + import sys + from packaging.database import get_distribution, get_distributions + from packaging.depgraph import dependent_dists + + dists = list(get_distributions(use_egg_info=True)) + dist = get_distribution('bacon', use_egg_info=True) + if dist is None: + sys.exit('No such distribution in the system') + + deps = dependent_dists(dists, dist) + deps = ', '.join(repr(x.name) for x in deps) + print('Distributions depending on %r: %s' % (dist.name, deps)) + +And this is example output: + +.. with the dependency relationships as in the previous section + (depgraph_big) + +.. code-block:: none + + Distributions depending on 'bacon': 'towel-stuff', 'choxie', 'grammar' diff --git a/Doc/library/packaging.dist.rst b/Doc/library/packaging.dist.rst new file mode 100644 index 0000000..fb05b69 --- /dev/null +++ b/Doc/library/packaging.dist.rst @@ -0,0 +1,102 @@ +:mod:`packaging.dist` --- The Distribution class +================================================ + +.. module:: packaging.dist + :synopsis: Core Distribution class. + + +This module provides the :class:`Distribution` class, which represents the +module distribution being built/packaged/distributed/installed. + +.. class:: Distribution(arguments) + + A :class:`Distribution` describes how to build, package, distribute and + install a Python project. + + The arguments accepted by the constructor are laid out in the following + table. Some of them will end up in a metadata object, the rest will become + data attributes of the :class:`Distribution` instance. + + .. TODO improve constructor to take a Metadata object + named params? + (i.e. Distribution(metadata, cmdclass, py_modules, etc) + .. TODO also remove obsolete(?) script_name, etc. parameters? see what + py2exe and other tools need + + +--------------------+--------------------------------+-------------------------------------------------------------+ + | argument name | value | type | + +====================+================================+=============================================================+ + | *name* | The name of the project | string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *version* | The version number of the | See :mod:`packaging.version` | + | | release | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *summary* | A single line describing the | a string | + | | project | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *description* | Longer description of the | a string | + | | project | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author* | The name of the project author | a string | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *author_email* | The email address of the | a string | + | | project author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer* | The name of the current | a string | + | | maintainer, if different from | | + | | the author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *maintainer_email* | The email address of the | | + | | current maintainer, if | | + | | different from the author | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *home_page* | A URL for the proejct | a URL | + | | (homepage) | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *download_url* | A URL to download the project | a URL | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *packages* | A list of Python packages that | a list of strings | + | | packaging will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *py_modules* | A list of Python modules that | a list of strings | + | | packaging will manipulate | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *scripts* | A list of standalone scripts | a list of strings | + | | to be built and installed | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *ext_modules* | A list of Python extensions to | A list of instances of | + | | be built | :class:`packaging.compiler.extension.Extension` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *classifiers* | A list of categories for the | The list of available | + | | distribution | categorizations is at | + | | | http://pypi.python.org/pypi?:action=list_classifiers. | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *distclass* | the :class:`Distribution` | A subclass of | + | | class to use | :class:`packaging.dist.Distribution` | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_name* | The name of the setup.py | a string | + | | script - defaults to | | + | | ``sys.argv[0]`` | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *script_args* | Arguments to supply to the | a list of strings | + | | setup script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *options* | default options for the setup | a string | + | | script | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *license* | The license for the | a string; should be used when there is no suitable License | + | | distribution | classifier, or to specify a classifier | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *keywords* | Descriptive keywords | a list of strings; used by catalogs | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *platforms* | Platforms compatible with this | a list of strings; should be used when there is no | + | | distribution | suitable Platform classifier | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *cmdclass* | A mapping of command names to | a dictionary | + | | :class:`Command` subclasses | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *data_files* | A list of data files to | a list | + | | install | | + +--------------------+--------------------------------+-------------------------------------------------------------+ + | *package_dir* | A mapping of Python packages | a dictionary | + | | to directory names | | + +--------------------+--------------------------------+-------------------------------------------------------------+ diff --git a/Doc/library/packaging.fancy_getopt.rst b/Doc/library/packaging.fancy_getopt.rst new file mode 100644 index 0000000..2c69341 --- /dev/null +++ b/Doc/library/packaging.fancy_getopt.rst @@ -0,0 +1,75 @@ +:mod:`packaging.fancy_getopt` --- Wrapper around the getopt module +================================================================== + +.. module:: packaging.fancy_getopt + :synopsis: Additional getopt functionality. + + +.. warning:: + This module is deprecated and will be replaced with :mod:`optparse`. + +This module provides a wrapper around the standard :mod:`getopt` module that +provides the following additional features: + +* short and long options are tied together + +* options have help strings, so :func:`fancy_getopt` could potentially create a + complete usage summary + +* options set attributes of a passed-in object + +* boolean options can have "negative aliases" --- e.g. if :option:`--quiet` is + the "negative alias" of :option:`--verbose`, then :option:`--quiet` on the + command line sets *verbose* to false. + +.. function:: fancy_getopt(options, negative_opt, object, args) + + Wrapper function. *options* is a list of ``(long_option, short_option, + help_string)`` 3-tuples as described in the constructor for + :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names + to option names, both the key and value should be in the *options* list. + *object* is an object which will be used to store values (see the :meth:`getopt` + method of the :class:`FancyGetopt` class). *args* is the argument list. Will use + ``sys.argv[1:]`` if you pass ``None`` as *args*. + + +.. class:: FancyGetopt([option_table=None]) + + The option_table is a list of 3-tuples: ``(long_option, short_option, + help_string)`` + + If an option takes an argument, its *long_option* should have ``'='`` appended; + *short_option* should just be a single character, no ``':'`` in any case. + *short_option* should be ``None`` if a *long_option* doesn't have a + corresponding *short_option*. All option tuples must have long options. + +The :class:`FancyGetopt` class provides the following methods: + + +.. method:: FancyGetopt.getopt([args=None, object=None]) + + Parse command-line options in args. Store as attributes on *object*. + + If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is + ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores + option values there, and returns a tuple ``(args, object)``. If *object* is + supplied, it is modified in place and :func:`getopt` just returns *args*; in + both cases, the returned *args* is a modified copy of the passed-in *args* list, + which is left untouched. + + .. TODO and args returned are? + + +.. method:: FancyGetopt.get_option_order() + + Returns the list of ``(option, value)`` tuples processed by the previous run of + :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called + yet. + + +.. method:: FancyGetopt.generate_help([header=None]) + + Generate help text (a list of strings, one per suggested line of output) from + the option table for this :class:`FancyGetopt` object. + + If supplied, prints the supplied *header* at the top of the help. diff --git a/Doc/library/packaging.install.rst b/Doc/library/packaging.install.rst new file mode 100644 index 0000000..b619a98 --- /dev/null +++ b/Doc/library/packaging.install.rst @@ -0,0 +1,112 @@ +:mod:`packaging.install` --- Installation tools +=============================================== + +.. module:: packaging.install + :synopsis: Download and installation building blocks + + +Packaging provides a set of tools to deal with downloads and installation of +distributions. Their role is to download the distribution from indexes, resolve +the dependencies, and provide a safe way to install distributions. An operation +that fails will cleanly roll back, not leave half-installed distributions on the +system. Here's the basic process followed: + +#. Move all distributions that will be removed to a temporary location. + +#. Install all the distributions that will be installed in a temporary location. + +#. If the installation fails, move the saved distributions back to their + location and delete the installed distributions. + +#. Otherwise, move the installed distributions to the right location and delete + the temporary locations. + +This is a higher-level module built on :mod:`packaging.database` and +:mod:`packaging.pypi`. + + +Public functions +---------------- + +.. function:: get_infos(requirements, index=None, installed=None, \ + prefer_final=True) + + Return information about what's going to be installed and upgraded. + *requirements* is a string string containing the requirements for this + project, for example ``'FooBar 1.1'`` or ``'BarBaz (<1.2)'``. + + .. XXX are requirements comma-separated? + + If you want to use another index than the main PyPI, give its URI as *index* + argument. + + *installed* is a list of already installed distributions used to find + satisfied dependencies, obsoleted distributions and eventual conflicts. + + By default, alpha, beta and candidate versions are not picked up. Set + *prefer_final* to false to accept them too. + + The results are returned in a dictionary containing all the information + needed to perform installation of the requirements with the + :func:`install_from_infos` function: + + >>> get_install_info("FooBar (<=1.2)") + {'install': [], 'remove': [], 'conflict': []} + + .. TODO should return tuple or named tuple, not dict + .. TODO use "predicate" or "requirement" consistently in version and here + .. FIXME "info" cannot be plural in English, s/infos/info/ + + +.. function:: install(project) + + +.. function:: install_dists(dists, path, paths=None) + + Safely install all distributions provided in *dists* into *path*. *paths* is + a list of paths where already-installed distributions will be looked for to + find satisfied dependencies and conflicts (default: :data:`sys.path`). + Returns a list of installed dists. + + .. FIXME dists are instances of what? + + +.. function:: install_from_infos(install_path=None, install=[], remove=[], \ + conflicts=[], paths=None) + + Safely install and remove given distributions. This function is designed to + work with the return value of :func:`get_infos`: *install*, *remove* and + *conflicts* should be list of distributions returned by :func:`get_infos`. + If *install* is not empty, *install_path* must be given to specify the path + where the distributions should be installed. *paths* is a list of paths + where already-installed distributions will be looked for (default: + :data:`sys.path`). + + This function is a very basic installer; if *conflicts* is not empty, the + system will be in a conflicting state after the function completes. It is a + building block for more sophisticated installers with conflict resolution + systems. + + .. TODO document typical value for install_path + .. TODO document integration with default schemes, esp. user site-packages + + +.. function:: install_local_project(path) + + Install a distribution from a source directory, which must contain either a + Packaging-compliant :file:`setup.cfg` file or a legacy Distutils + :file:`setup.py` script (in which case Distutils will be used under the hood + to perform the installation). + + +.. function:: remove(project_name, paths=None, auto_confirm=True) + + Remove one distribution from the system. + + .. FIXME this is the only function using "project" instead of dist/release + +.. + Example usage + -------------- + + Get the scheme of what's gonna be installed if we install "foobar": diff --git a/Doc/library/packaging.metadata.rst b/Doc/library/packaging.metadata.rst new file mode 100644 index 0000000..332d69d --- /dev/null +++ b/Doc/library/packaging.metadata.rst @@ -0,0 +1,122 @@ +:mod:`packaging.metadata` --- Metadata handling +=============================================== + +.. module:: packaging.metadata + :synopsis: Class holding the metadata of a release. + + +.. TODO use sphinx-autogen to generate basic doc from the docstrings + +.. class:: Metadata + + This class can read and write metadata files complying with any of the + defined versions: 1.0 (:PEP:`241`), 1.1 (:PEP:`314`) and 1.2 (:PEP:`345`). It + implements methods to parse Metadata files and write them, and a mapping + interface to its contents. + + The :PEP:`345` implementation supports the micro-language for the environment + markers, and displays warnings when versions that are supposed to be + :PEP:`386`-compliant are violating the specification. + + +Reading metadata +---------------- + +The :class:`Metadata` class can be instantiated +with the path of the metadata file, and provides a dict-like interface to the +values:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata('PKG-INFO') + >>> metadata.keys()[:5] + ('Metadata-Version', 'Name', 'Version', 'Platform', 'Supported-Platform') + >>> metadata['Name'] + 'CLVault' + >>> metadata['Version'] + '0.5' + >>> metadata['Requires-Dist'] + ["pywin32; sys.platform == 'win32'", "Sphinx"] + + +The fields that support environment markers can be automatically ignored if +the object is instantiated using the ``platform_dependent`` option. +:class:`Metadata` will interpret in this case +the markers and will automatically remove the fields that are not compliant +with the running environment. Here's an example under Mac OS X. The win32 +dependency we saw earlier is ignored:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata('PKG-INFO', platform_dependent=True) + >>> metadata['Requires-Dist'] + ['Sphinx'] + + +If you want to provide your own execution context, let's say to test the +metadata under a particular environment that is not the current environment, +you can provide your own values in the ``execution_context`` option, which +is the dict that may contain one or more keys of the context the micro-language +expects. + +Here's an example, simulating a win32 environment:: + + >>> from packaging.metadata import Metadata + >>> context = {'sys.platform': 'win32'} + >>> metadata = Metadata('PKG-INFO', platform_dependent=True, + ... execution_context=context) + ... + >>> metadata['Requires-Dist'] = ["pywin32; sys.platform == 'win32'", + ... "Sphinx"] + ... + >>> metadata['Requires-Dist'] + ['pywin32', 'Sphinx'] + + +Writing metadata +---------------- + +Writing metadata can be done using the ``write`` method:: + + >>> metadata.write('/to/my/PKG-INFO') + +The class will pick the best version for the metadata, depending on the values +provided. If all the values provided exist in all versions, the class will +use :attr:`PKG_INFO_PREFERRED_VERSION`. It is set by default to 1.0, the most +widespread version. + + +Conflict checking and best version +---------------------------------- + +Some fields in :PEP:`345` have to comply with the version number specification +defined in :PEP:`386`. When they don't comply, a warning is emitted:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata() + >>> metadata['Requires-Dist'] = ['Funky (Groovie)'] + "Funky (Groovie)" is not a valid predicate + >>> metadata['Requires-Dist'] = ['Funky (1.2)'] + +See also :mod:`packaging.version`. + + +.. TODO talk about check() + + +:mod:`packaging.markers` --- Environment markers +================================================ + +.. module:: packaging.markers + :synopsis: Micro-language for environment markers + + +This is an implementation of environment markers `as defined in PEP 345 +`_. It is used +for some metadata fields. + +.. function:: interpret(marker, execution_context=None) + + Interpret a marker and return a boolean result depending on the environment. + Example: + + >>> interpret("python_version > '1.0'") + True diff --git a/Doc/library/packaging.pypi.dist.rst b/Doc/library/packaging.pypi.dist.rst new file mode 100644 index 0000000..aaaaab7 --- /dev/null +++ b/Doc/library/packaging.pypi.dist.rst @@ -0,0 +1,114 @@ +:mod:`packaging.pypi.dist` --- Classes representing query results +================================================================= + +.. module:: packaging.pypi.dist + :synopsis: Classes representing the results of queries to indexes. + + +Information coming from the indexes is held in instances of the classes defined +in this module. + +Keep in mind that each project (eg. FooBar) can have several releases +(eg. 1.1, 1.2, 1.3), and each of these releases can be provided in multiple +distributions (eg. a source distribution, a binary one, etc). + + +ReleaseInfo +----------- + +Each release has a project name, version, metadata, and related distributions. + +This information is stored in :class:`ReleaseInfo` +objects. + +.. class:: ReleaseInfo + + +DistInfo +--------- + +:class:`DistInfo` is a simple class that contains +information related to distributions; mainly the URLs where distributions +can be found. + +.. class:: DistInfo + + +ReleasesList +------------ + +The :mod:`~packaging.pypi.dist` module provides a class which works +with lists of :class:`ReleaseInfo` classes; +used to filter and order results. + +.. class:: ReleasesList + + +Example usage +------------- + +Build a list of releases and order them +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assuming we have a list of releases:: + + >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo + >>> fb10 = ReleaseInfo("FooBar", "1.0") + >>> fb11 = ReleaseInfo("FooBar", "1.1") + >>> fb11a = ReleaseInfo("FooBar", "1.1a1") + >>> ReleasesList("FooBar", [fb11, fb11a, fb10]) + >>> releases.sort_releases() + >>> releases.get_versions() + ['1.1', '1.1a1', '1.0'] + >>> releases.add_release("1.2a1") + >>> releases.get_versions() + ['1.1', '1.1a1', '1.0', '1.2a1'] + >>> releases.sort_releases() + ['1.2a1', '1.1', '1.1a1', '1.0'] + >>> releases.sort_releases(prefer_final=True) + >>> releases.get_versions() + ['1.1', '1.0', '1.2a1', '1.1a1'] + + +Add distribution related information to releases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It's easy to add distribution information to releases:: + + >>> from packaging.pypi.dist import ReleasesList, ReleaseInfo + >>> r = ReleaseInfo("FooBar", "1.0") + >>> r.add_distribution("sdist", url="http://example.org/foobar-1.0.tar.gz") + >>> r.dists + {'sdist': FooBar 1.0 sdist} + >>> r['sdist'].url + {'url': 'http://example.org/foobar-1.0.tar.gz', 'hashname': None, 'hashval': + None, 'is_external': True} + + +Getting attributes from the dist objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To abstract querying information returned from the indexes, attributes and +release information can be retrieved directly from dist objects. + +For instance, if you have a release instance that does not contain the metadata +attribute, it can be fetched by using the "fetch_metadata" method:: + + >>> r = Release("FooBar", "1.1") + >>> print r.metadata + None # metadata field is actually set to "None" + >>> r.fetch_metadata() + + +.. XXX add proper roles to these constructs + + +It's possible to retrieve a project's releases (`fetch_releases`), +metadata (`fetch_metadata`) and distributions (`fetch_distributions`) using +a similar work flow. + +.. XXX what is possible? + +Internally, this is possible because while retrieving information about +projects, releases or distributions, a reference to the client used is +stored which can be accessed using the objects `_index` attribute. diff --git a/Doc/library/packaging.pypi.rst b/Doc/library/packaging.pypi.rst new file mode 100644 index 0000000..93b61c9 --- /dev/null +++ b/Doc/library/packaging.pypi.rst @@ -0,0 +1,53 @@ +:mod:`packaging.pypi` --- Interface to projects indexes +======================================================= + +.. module:: packaging.pypi + :synopsis: Low-level and high-level APIs to query projects indexes. + + +Packaging queries PyPI to get information about projects or download them. The +low-level facilities used internally are also part of the public API designed to +be used by other tools. + +The :mod:`packaging.pypi` package provides those facilities, which can be +used to access information about Python projects registered at indexes, the +main one being PyPI, located ad http://pypi.python.org/. + +There is two ways to retrieve data from these indexes: a screen-scraping +interface called the "simple API", and XML-RPC. The first one uses HTML pages +located under http://pypi.python.org/simple/, the second one makes XML-RPC +requests to http://pypi.python.org/pypi/. All functions and classes also work +with other indexes such as mirrors, which typically implement only the simple +interface. + +Packaging provides a class that wraps both APIs to provide full query and +download functionality: :class:`packaging.pypi.client.ClientWrapper`. If you +want more control, you can use the underlying classes +:class:`packaging.pypi.simple.Crawler` and :class:`packaging.pypi.xmlrpc.Client` +to connect to one specific interface. + + +:mod:`packaging.pypi.client` --- High-level query API +===================================================== + +.. module:: packaging.pypi.client + :synopsis: Wrapper around :mod;`packaging.pypi.xmlrpc` and + :mod:`packaging.pypi.simple` to query indexes. + + +This module provides a high-level API to query indexes and search +for releases and distributions. The aim of this module is to choose the best +way to query the API automatically, either using XML-RPC or the simple index, +with a preference toward the latter. + +.. class:: ClientWrapper + + Instances of this class will use the simple interface or XML-RPC requests to + query indexes and return :class:`packaging.pypi.dist.ReleaseInfo` and + :class:`packaging.pypi.dist.ReleasesList` objects. + + .. method:: find_projects + + .. method:: get_release + + .. method:: get_releases diff --git a/Doc/library/packaging.pypi.simple.rst b/Doc/library/packaging.pypi.simple.rst new file mode 100644 index 0000000..ea5edca --- /dev/null +++ b/Doc/library/packaging.pypi.simple.rst @@ -0,0 +1,157 @@ +:mod:`packaging.pypi.simple` --- Crawler using the PyPI "simple" interface +========================================================================== + +.. module:: packaging.pypi.simple + :synopsis: Crawler using the screen-scraping "simple" interface to fetch info + and distributions. + + +`packaging.pypi.simple` can process Python Package Indexes and provides +useful information about distributions. It also can crawl local indexes, for +instance. + +You should use `packaging.pypi.simple` for: + + * Search distributions by name and versions. + * Process index external pages. + * Download distributions by name and versions. + +And should not be used for: + + * Things that will end up in too long index processing (like "finding all + distributions with a specific version, no matters the name") + + +API +--- + +.. class:: Crawler + + +Usage Exemples +--------------- + +To help you understand how using the `Crawler` class, here are some basic +usages. + +Request the simple index to get a specific distribution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Supposing you want to scan an index to get a list of distributions for +the "foobar" project. You can use the "get_releases" method for that. +The get_releases method will browse the project page, and return +:class:`ReleaseInfo` objects for each found link that rely on downloads. :: + + >>> from packaging.pypi.simple import Crawler + >>> crawler = Crawler() + >>> crawler.get_releases("FooBar") + [, ] + + +Note that you also can request the client about specific versions, using version +specifiers (described in `PEP 345 +`_):: + + >>> client.get_releases("FooBar < 1.2") + [, ] + + +`get_releases` returns a list of :class:`ReleaseInfo`, but you also can get the +best distribution that fullfil your requirements, using "get_release":: + + >>> client.get_release("FooBar < 1.2") + + + +Download distributions +^^^^^^^^^^^^^^^^^^^^^^ + +As it can get the urls of distributions provided by PyPI, the `Crawler` +client also can download the distributions and put it for you in a temporary +destination:: + + >>> client.download("foobar") + /tmp/temp_dir/foobar-1.2.tar.gz + + +You also can specify the directory you want to download to:: + + >>> client.download("foobar", "/path/to/my/dir") + /path/to/my/dir/foobar-1.2.tar.gz + + +While downloading, the md5 of the archive will be checked, if not matches, it +will try another time, then if fails again, raise `MD5HashDoesNotMatchError`. + +Internally, that's not the Crawler which download the distributions, but the +`DistributionInfo` class. Please refer to this documentation for more details. + + +Following PyPI external links +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default behavior for packaging is to *not* follow the links provided +by HTML pages in the "simple index", to find distributions related +downloads. + +It's possible to tell the PyPIClient to follow external links by setting the +`follow_externals` attribute, on instantiation or after:: + + >>> client = Crawler(follow_externals=True) + +or :: + + >>> client = Crawler() + >>> client.follow_externals = True + + +Working with external indexes, and mirrors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default `Crawler` behavior is to rely on the Python Package index stored +on PyPI (http://pypi.python.org/simple). + +As you can need to work with a local index, or private indexes, you can specify +it using the index_url parameter:: + + >>> client = Crawler(index_url="file://filesystem/path/") + +or :: + + >>> client = Crawler(index_url="http://some.specific.url/") + + +You also can specify mirrors to fallback on in case the first index_url you +provided doesnt respond, or not correctly. The default behavior for +`Crawler` is to use the list provided by Python.org DNS records, as +described in the :PEP:`381` about mirroring infrastructure. + +If you don't want to rely on these, you could specify the list of mirrors you +want to try by specifying the `mirrors` attribute. It's a simple iterable:: + + >>> mirrors = ["http://first.mirror","http://second.mirror"] + >>> client = Crawler(mirrors=mirrors) + + +Searching in the simple index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It's possible to search for projects with specific names in the package index. +Assuming you want to find all projects containing the "distutils" keyword:: + + >>> c.search_projects("distutils") + [, , , , , , ] + + +You can also search the projects starting with a specific text, or ending with +that text, using a wildcard:: + + >>> c.search_projects("distutils*") + [, , ] + + >>> c.search_projects("*distutils") + [, , , , ] diff --git a/Doc/library/packaging.pypi.xmlrpc.rst b/Doc/library/packaging.pypi.xmlrpc.rst new file mode 100644 index 0000000..0253d68 --- /dev/null +++ b/Doc/library/packaging.pypi.xmlrpc.rst @@ -0,0 +1,143 @@ +:mod:`packaging.pypi.xmlrpc` --- Crawler using the PyPI XML-RPC interface +========================================================================= + +.. module:: packaging.pypi.xmlrpc + :synopsis: Client using XML-RPC requests to fetch info and distributions. + + +Indexes can be queried using XML-RPC calls, and Packaging provides a simple +way to interface with XML-RPC. + +You should **use** XML-RPC when: + +* Searching the index for projects **on other fields than project + names**. For instance, you can search for projects based on the + author_email field. +* Searching all the versions that have existed for a project. +* you want to retrieve METADATAs information from releases or + distributions. + + +You should **avoid using** XML-RPC method calls when: + +* Retrieving the last version of a project +* Getting the projects with a specific name and version. +* The simple index can match your needs + + +When dealing with indexes, keep in mind that the index queries will always +return you :class:`packaging.pypi.dist.ReleaseInfo` and +:class:`packaging.pypi.dist.ReleasesList` objects. + +Some methods here share common APIs with the one you can find on +:class:`packaging.pypi.simple`, internally, :class:`packaging.pypi.client` +is inherited by :class:`Client` + + +API +--- + +.. class:: Client + + +Usage examples +-------------- + +Use case described here are use case that are not common to the other clients. +If you want to see all the methods, please refer to API or to usage examples +described in :class:`packaging.pypi.client.Client` + + +Finding releases +^^^^^^^^^^^^^^^^ + +It's a common use case to search for "things" within the index. We can +basically search for projects by their name, which is the most used way for +users (eg. "give me the last version of the FooBar project"). + +This can be accomplished using the following syntax:: + + >>> client = xmlrpc.Client() + >>> client.get_release("Foobar (<= 1.3)) + + >>> client.get_releases("FooBar (<= 1.3)") + [FooBar 1.1, FooBar 1.1.1, FooBar 1.2, FooBar 1.2.1] + + +And we also can find for specific fields:: + + >>> client.search_projects(field=value) + + +You could specify the operator to use, default is "or":: + + >>> client.search_projects(field=value, operator="and") + + +The specific fields you can search are: + +* name +* version +* author +* author_email +* maintainer +* maintainer_email +* home_page +* license +* summary +* description +* keywords +* platform +* download_url + + +Getting metadata information +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +XML-RPC is a prefered way to retrieve metadata information from indexes. +It's really simple to do so:: + + >>> client = xmlrpc.Client() + >>> client.get_metadata("FooBar", "1.1") + + + +Assuming we already have a :class:`packaging.pypi.ReleaseInfo` object defined, +it's possible to pass it to the xmlrpc client to retrieve and complete its +metadata:: + + >>> foobar11 = ReleaseInfo("FooBar", "1.1") + >>> client = xmlrpc.Client() + >>> returned_release = client.get_metadata(release=foobar11) + >>> returned_release + + + +Get all the releases of a project +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To retrieve all the releases for a project, you can build them using +`get_releases`:: + + >>> client = xmlrpc.Client() + >>> client.get_releases("FooBar") + [, , ] + + +Get information about distributions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Indexes have information about projects, releases **and** distributions. +If you're not familiar with those, please refer to the documentation of +:mod:`packaging.pypi.dist`. + +It's possible to retrieve information about distributions, e.g "what are the +existing distributions for this release ? How to retrieve them ?":: + + >>> client = xmlrpc.Client() + >>> release = client.get_distributions("FooBar", "1.1") + >>> release.dists + {'sdist': , 'bdist': } + +As you see, this does not return a list of distributions, but a release, +because a release can be used like a list of distributions. diff --git a/Doc/library/packaging.rst b/Doc/library/packaging.rst new file mode 100644 index 0000000..5bf2c06 --- /dev/null +++ b/Doc/library/packaging.rst @@ -0,0 +1,78 @@ +:mod:`packaging` --- Packaging support +====================================== + +.. module:: packaging + :synopsis: Packaging system and building blocks for other packaging systems. +.. sectionauthor:: Fred L. Drake, Jr. , distutils and packaging + contributors + + +The :mod:`packaging` package provides support for building, packaging, +distributing and installing additional projects into a Python installation. +Projects may include Python modules, extension modules, packages and scripts. +:mod:`packaging` also provides building blocks for other packaging systems +that are not tied to the command system. + +This manual is the reference documentation for those standalone building +blocks and for extending Packaging. If you're looking for the user-centric +guides to install a project or package your own code, head to `See also`__. + + +Building blocks +--------------- + +.. toctree:: + :maxdepth: 2 + :numbered: + + packaging-misc + packaging.version + packaging.metadata + packaging.database + packaging.depgraph + packaging.pypi + packaging.pypi.dist + packaging.pypi.simple + packaging.pypi.xmlrpc + packaging.install + + +The command machinery +--------------------- + +.. toctree:: + :maxdepth: 2 + :numbered: + + packaging.dist + packaging.command + packaging.compiler + packaging.fancy_getopt + + +Other utilities +---------------- + +.. toctree:: + :maxdepth: 2 + :numbered: + + packaging.util + packaging.tests.pypi_server + +.. XXX missing: compat config create (dir_util) run pypi.{base,mirrors} + + +.. __: + +.. seealso:: + + :ref:`packaging-index` + The manual for developers of Python projects who want to package and + distribute them. This describes how to use :mod:`packaging` to make + projects easily found and added to an existing Python installation. + + :ref:`packaging-install-index` + A user-centered manual which includes information on adding projects + into an existing Python installation. You do not need to be a Python + programmer to read this manual. diff --git a/Doc/library/packaging.tests.pypi_server.rst b/Doc/library/packaging.tests.pypi_server.rst new file mode 100644 index 0000000..f3b7720 --- /dev/null +++ b/Doc/library/packaging.tests.pypi_server.rst @@ -0,0 +1,105 @@ +:mod:`packaging.tests.pypi_server` --- PyPI mock server +======================================================= + +.. module:: packaging.tests.pypi_server + :synopsis: Mock server used to test PyPI-related modules and commands. + + +When you are testing code that works with Packaging, you might find these tools +useful. + + +The mock server +--------------- + +.. class:: PyPIServer + + PyPIServer is a class that implements an HTTP server running in a separate + thread. All it does is record the requests for further inspection. The recorded + data is available under ``requests`` attribute. The default + HTTP response can be overridden with the ``default_response_status``, + ``default_response_headers`` and ``default_response_data`` attributes. + + By default, when accessing the server with urls beginning with `/simple/`, + the server also record your requests, but will look for files under + the `/tests/pypiserver/simple/` path. + + You can tell the sever to serve static files for other paths. This could be + accomplished by using the `static_uri_paths` parameter, as below:: + + server = PyPIServer(static_uri_paths=["first_path", "second_path"]) + + + You need to create the content that will be served under the + `/tests/pypiserver/default` path. If you want to serve content from another + place, you also can specify another filesystem path (which needs to be under + `tests/pypiserver/`. This will replace the default behavior of the server, and + it will not serve content from the `default` dir :: + + server = PyPIServer(static_filesystem_paths=["path/to/your/dir"]) + + + If you just need to add some paths to the existing ones, you can do as shown, + keeping in mind that the server will always try to load paths in reverse order + (e.g here, try "another/super/path" then the default one) :: + + server = PyPIServer(test_static_path="another/super/path") + server = PyPIServer("another/super/path") + # or + server.static_filesystem_paths.append("another/super/path") + + + As a result of what, in your tests, while you need to use the PyPIServer, in + order to isolates the test cases, the best practice is to place the common files + in the `default` folder, and to create a directory for each specific test case:: + + server = PyPIServer(static_filesystem_paths = ["default", "test_pypi_server"], + static_uri_paths=["simple", "external"]) + + +Base class and decorator for tests +---------------------------------- + +.. class:: PyPIServerTestCase + + ``PyPIServerTestCase`` is a test case class with setUp and tearDown methods that + take care of a single PyPIServer instance attached as a ``pypi`` attribute on + the test class. Use it as one of the base classes in your test case:: + + + class UploadTestCase(PyPIServerTestCase): + + def test_something(self): + cmd = self.prepare_command() + cmd.ensure_finalized() + cmd.repository = self.pypi.full_address + cmd.run() + + environ, request_data = self.pypi.requests[-1] + self.assertEqual(request_data, EXPECTED_REQUEST_DATA) + + +.. decorator:: use_pypi_server + + You also can use a decorator for your tests, if you do not need the same server + instance along all you test case. So, you can specify, for each test method, + some initialisation parameters for the server. + + For this, you need to add a `server` parameter to your method, like this:: + + class SampleTestCase(TestCase): + + @use_pypi_server() + def test_something(self, server): + ... + + + The decorator will instantiate the server for you, and run and stop it just + before and after your method call. You also can pass the server initializer, + just like this:: + + class SampleTestCase(TestCase): + + @use_pypi_server("test_case_name") + def test_something(self, server): + ... diff --git a/Doc/library/packaging.util.rst b/Doc/library/packaging.util.rst new file mode 100644 index 0000000..b95d5b5 --- /dev/null +++ b/Doc/library/packaging.util.rst @@ -0,0 +1,186 @@ +:mod:`packaging.util` --- Miscellaneous utility functions +========================================================= + +.. module:: packaging.util + :synopsis: Miscellaneous utility functions. + + +This module contains various helpers for the other modules. + +.. XXX a number of functions are missing, but the module may be split first + (it's ginormous right now, some things could go to compat for example) + +.. function:: get_platform() + + Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; e.g. for IRIX the architecture isn't particularly + important (IRIX only runs on SGI hardware), but for Linux the kernel version + isn't particularly important. + + Examples of returned values: + + * ``linux-i586`` + * ``linux-alpha`` + * ``solaris-2.6-sun4u`` + * ``irix-5.3`` + * ``irix64-6.2`` + + For non-POSIX platforms, currently just returns ``sys.platform``. + + For Mac OS X systems the OS version reflects the minimal version on which + binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` + during the build of Python), not the OS version of the current system. + + For universal binary builds on Mac OS X the architecture value reflects + the univeral binary status instead of the architecture of the current + processor. For 32-bit universal binaries the architecture is ``fat``, + for 64-bit universal binaries the architecture is ``fat64``, and + for 4-way universal binaries the architecture is ``universal``. Starting + from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for + a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for + a univeral build with the i386 and x86_64 architectures + + Examples of returned values on Mac OS X: + + * ``macosx-10.3-ppc`` + + * ``macosx-10.3-fat`` + + * ``macosx-10.5-universal`` + + * ``macosx-10.6-intel`` + + .. XXX reinvention of platform module? + + +.. function:: convert_path(pathname) + + Return 'pathname' as a name that will work on the native filesystem, i.e. + split it on '/' and put it back together again using the current directory + separator. Needed because filenames in the setup script are always supplied + in Unix style, and have to be converted to the local convention before we + can actually use them in the filesystem. Raises :exc:`ValueError` on + non-Unix-ish systems if *pathname* either starts or ends with a slash. + + +.. function:: change_root(new_root, pathname) + + Return *pathname* with *new_root* prepended. If *pathname* is relative, this + is equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires + making *pathname* relative and then joining the two, which is tricky on + DOS/Windows. + + +.. function:: check_environ() + + Ensure that 'os.environ' has all the environment variables we guarantee that + users can use in config files, command-line options, etc. Currently this + includes: + + * :envvar:`HOME` - user's home directory (Unix only) + * :envvar:`PLAT` - description of the current platform, including hardware + and OS (see :func:`get_platform`) + + +.. function:: find_executable(executable, path=None) + + Search the path for a given executable name. + + +.. function:: subst_vars(s, local_vars) + + Perform shell/Perl-style variable substitution on *s*. Every occurrence of + ``$`` followed by a name is considered a variable, and variable is + substituted by the value found in the *local_vars* dictionary, or in + ``os.environ`` if it's not in *local_vars*. *os.environ* is first + checked/augmented to guarantee that it contains certain values: see + :func:`check_environ`. Raise :exc:`ValueError` for any variables not found + in either *local_vars* or ``os.environ``. + + Note that this is not a fully-fledged string interpolation function. A valid + ``$variable`` can consist only of upper and lower case letters, numbers and + an underscore. No { } or ( ) style quoting is available. + + +.. function:: split_quoted(s) + + Split a string up according to Unix shell-like rules for quotes and + backslashes. In short: words are delimited by spaces, as long as those spaces + are not escaped by a backslash, or inside a quoted string. Single and double + quotes are equivalent, and the quote characters can be backslash-escaped. + The backslash is stripped from any two-character escape sequence, leaving + only the escaped character. The quote characters are stripped from any + quoted string. Returns a list of words. + + .. TODO Should probably be moved into the standard library. + + +.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0]) + + Perform some action that affects the outside world (for instance, writing to + the filesystem). Such actions are special because they are disabled by the + *dry_run* flag. This method takes care of all that bureaucracy for you; + all you have to do is supply the function to call and an argument tuple for + it (to embody the "external action" being performed), and an optional message + to print. + + +.. function:: newer(source, target) + + Return true if *source* exists and is more recently modified than *target*, + or if *source* exists and *target* doesn't. Return false if both exist and + *target* is the same age or newer than *source*. Raise + :exc:`PackagingFileError` if *source* does not exist. + + +.. function:: strtobool(val) + + Convert a string representation of truth to true (1) or false (0). + + True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false + values are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises + :exc:`ValueError` if *val* is anything else. + +.. TODO Add :term: markup to bytecode when merging into the stdlib + +.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None]) + + Byte-compile a collection of Python source files to either :file:`.pyc` or + :file:`.pyo` files in the same directory. *py_files* is a list of files to + compile; any files that don't end in :file:`.py` are silently skipped. + *optimize* must be one of the following: + + * ``0`` - don't optimize (generate :file:`.pyc`) + * ``1`` - normal optimization (like ``python -O``) + * ``2`` - extra optimization (like ``python -OO``) + + If *force* is true, all files are recompiled regardless of timestamps. + + The source filename encoded in each bytecode file defaults to the filenames + listed in *py_files*; you can modify these with *prefix* and *basedir*. + *prefix* is a string that will be stripped off of each source filename, and + *base_dir* is a directory name that will be prepended (after *prefix* is + stripped). You can supply either or both (or neither) of *prefix* and + *base_dir*, as you wish. + + If *dry_run* is true, doesn't actually do anything that would affect the + filesystem. + + Byte-compilation is either done directly in this interpreter process with the + standard :mod:`py_compile` module, or indirectly by writing a temporary + script and executing it. Normally, you should let :func:`byte_compile` + figure out to use direct compilation or not (see the source for details). + The *direct* flag is used by the script generated in indirect mode; unless + you know what you're doing, leave it set to ``None``. + + +.. function:: rfc822_escape(header) + + Return a version of *header* escaped for inclusion in an :rfc:`822` header, by + ensuring there are 8 spaces space after each newline. Note that it does no + other modification of the string. + + .. TODO this _can_ be replaced diff --git a/Doc/library/packaging.version.rst b/Doc/library/packaging.version.rst new file mode 100644 index 0000000..f36cdab --- /dev/null +++ b/Doc/library/packaging.version.rst @@ -0,0 +1,104 @@ +:mod:`packaging.version` --- Version number classes +=================================================== + +.. module:: packaging.version + :synopsis: Classes that represent project version numbers. + + +This module contains classes and functions useful to deal with version numbers. +It's an implementation of version specifiers `as defined in PEP 345 +`_. + + +Version numbers +--------------- + +.. class:: NormalizedVersion(self, s, error_on_huge_major_num=True) + + A specific version of a distribution, as described in PEP 345. *s* is a + string object containing the version number (for example ``'1.2b1'``), + *error_on_huge_major_num* a boolean specifying whether to consider an + apparent use of a year or full date as the major version number an error. + + The rationale for the second argument is that there were projects using years + or full dates as version numbers, which could cause problems with some + packaging systems sorting. + + Instances of this class can be compared and sorted:: + + >>> NormalizedVersion('1.2b1') < NormalizedVersion('1.2') + True + + :class:`NormalizedVersion` is used internally by :class:`VersionPredicate` to + do its work. + + +.. class:: IrrationalVersionError + + Exception raised when an invalid string is given to + :class:`NormalizedVersion`. + + >>> NormalizedVersion("irrational_version_number") + ... + IrrationalVersionError: irrational_version_number + + +.. function:: suggest_normalized_version(s) + + Before standardization in PEP 386, various schemes were in use. Packaging + provides a function to try to convert any string to a valid, normalized + version:: + + >>> suggest_normalized_version('2.1-rc1') + 2.1c1 + + + If :func:`suggest_normalized_version` can't make sense of the given string, + it will return ``None``:: + + >>> print(suggest_normalized_version('not a version')) + None + + +Version predicates +------------------ + +.. class:: VersionPredicate(predicate) + + This class deals with the parsing of field values like + ``ProjectName (>=version)``. + + .. method:: match(version) + + Test if a version number matches the predicate: + + >>> version = VersionPredicate("ProjectName (<1.2, >1.0)") + >>> version.match("1.2.1") + False + >>> version.match("1.1.1") + True + + +Validation helpers +------------------ + +If you want to use :term:`LBYL`-style checks instead of instantiating the +classes and catching :class:`IrrationalVersionError` and :class:`ValueError`, +you can use these functions: + +.. function:: is_valid_version(predicate) + + Check whether the given string is a valid version number. Example of valid + strings: ``'1.2'``, ``'4.2.0.dev4'``, ``'2.5.4.post2'``. + + +.. function:: is_valid_versions(predicate) + + Check whether the given string is a valid value for specifying multiple + versions, such as in the Requires-Python field. Example: ``'2.7, >=3.2'``. + + +.. function:: is_valid_predicate(predicate) + + Check whether the given string is a valid version predicate. Examples: + ``'some.project == 4.5, <= 4.7'``, ``'speciallib (> 1.0, != 1.4.2, < 2.0)'``. diff --git a/Doc/library/python.rst b/Doc/library/python.rst index b67fbfc..07eadb4 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -25,4 +25,5 @@ overview: inspect.rst site.rst fpectl.rst + packaging.rst distutils.rst diff --git a/Doc/packaging/builtdist.rst b/Doc/packaging/builtdist.rst new file mode 100644 index 0000000..b52c208 --- /dev/null +++ b/Doc/packaging/builtdist.rst @@ -0,0 +1,307 @@ +.. _packaging-built-dist: + +**************************** +Creating Built Distributions +**************************** + +A "built distribution" is what you're probably used to thinking of either as a +"binary package" or an "installer" (depending on your background). It's not +necessarily binary, though, because it might contain only Python source code +and/or byte-code; and we don't call it a package, because that word is already +spoken for in Python. (And "installer" is a term specific to the world of +mainstream desktop systems.) + +A built distribution is how you make life as easy as possible for installers of +your module distribution: for users of RPM-based Linux systems, it's a binary +RPM; for Windows users, it's an executable installer; for Debian-based Linux +users, it's a Debian package; and so forth. Obviously, no one person will be +able to create built distributions for every platform under the sun, so the +Distutils are designed to enable module developers to concentrate on their +specialty---writing code and creating source distributions---while an +intermediary species called *packagers* springs up to turn source distributions +into built distributions for as many platforms as there are packagers. + +Of course, the module developer could be his own packager; or the packager could +be a volunteer "out there" somewhere who has access to a platform which the +original developer does not; or it could be software periodically grabbing new +source distributions and turning them into built distributions for as many +platforms as the software has access to. Regardless of who they are, a packager +uses the setup script and the :command:`bdist` command family to generate built +distributions. + +As a simple example, if I run the following command in the Distutils source +tree:: + + python setup.py bdist + +then the Distutils builds my module distribution (the Distutils itself in this +case), does a "fake" installation (also in the :file:`build` directory), and +creates the default type of built distribution for my platform. The default +format for built distributions is a "dumb" tar file on Unix, and a simple +executable installer on Windows. (That tar file is considered "dumb" because it +has to be unpacked in a specific location to work.) + +Thus, the above command on a Unix system creates +:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place +installs the Distutils just as though you had downloaded the source distribution +and run ``python setup.py install``. (The "right place" is either the root of +the filesystem or Python's :file:`{prefix}` directory, depending on the options +given to the :command:`bdist_dumb` command; the default is to make dumb +distributions relative to :file:`{prefix}`.) + +Obviously, for pure Python distributions, this isn't any simpler than just +running ``python setup.py install``\ ---but for non-pure distributions, which +include extensions that would need to be compiled, it can mean the difference +between someone being able to use your extensions or not. And creating "smart" +built distributions, such as an executable installer for +Windows, is far more convenient for users even if your distribution doesn't +include any extensions. + +The :command:`bdist` command has a :option:`--formats` option, similar to the +:command:`sdist` command, which you can use to select the types of built +distribution to generate: for example, :: + + python setup.py bdist --format=zip + +would, when run on a Unix system, create :file:`Distutils-1.0.{plat}.zip`\ +---again, this archive would be unpacked from the root directory to install the +Distutils. + +The available formats for built distributions are: + ++-------------+------------------------------+---------+ +| Format | Description | Notes | ++=============+==============================+=========+ +| ``gztar`` | gzipped tar file | (1),(3) | +| | (:file:`.tar.gz`) | | ++-------------+------------------------------+---------+ +| ``ztar`` | compressed tar file | \(3) | +| | (:file:`.tar.Z`) | | ++-------------+------------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | \(3) | ++-------------+------------------------------+---------+ +| ``zip`` | zip file (:file:`.zip`) | (2),(4) | ++-------------+------------------------------+---------+ +| ``wininst`` | self-extracting ZIP file for | \(4) | +| | Windows | | ++-------------+------------------------------+---------+ +| ``msi`` | Microsoft Installer. | | ++-------------+------------------------------+---------+ + + +Notes: + +(1) + default on Unix + +(2) + default on Windows + +(3) + requires external utilities: :program:`tar` and possibly one of :program:`gzip`, + :program:`bzip2`, or :program:`compress` + +(4) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +You don't have to use the :command:`bdist` command with the :option:`--formats` +option; you can also use the command that directly implements the format you're +interested in. Some of these :command:`bdist` "sub-commands" actually generate +several similar formats; for instance, the :command:`bdist_dumb` command +generates all the "dumb" archive formats (``tar``, ``ztar``, ``gztar``, and +``zip``). The :command:`bdist` sub-commands, and the formats generated by +each, are: + ++--------------------------+-----------------------+ +| Command | Formats | ++==========================+=======================+ +| :command:`bdist_dumb` | tar, ztar, gztar, zip | ++--------------------------+-----------------------+ +| :command:`bdist_wininst` | wininst | ++--------------------------+-----------------------+ +| :command:`bdist_msi` | msi | ++--------------------------+-----------------------+ + +The following sections give details on the individual :command:`bdist_\*` +commands. + + +.. _packaging-creating-dumb: + +Creating dumb built distributions +================================= + +.. XXX Need to document absolute vs. prefix-relative packages here, but first + I have to implement it! + + +.. _packaging-creating-wininst: + +Creating Windows Installers +=========================== + +Executable installers are the natural format for binary distributions on +Windows. They display a nice graphical user interface, display some information +about the module distribution to be installed taken from the metadata in the +setup script, let the user select a few options, and start or cancel the +installation. + +Since the metadata is taken from the setup script, creating Windows installers +is usually as easy as running:: + + python setup.py bdist_wininst + +or the :command:`bdist` command with the :option:`--formats` option:: + + python setup.py bdist --formats=wininst + +If you have a pure module distribution (only containing pure Python modules and +packages), the resulting installer will be version independent and have a name +like :file:`foo-1.0.win32.exe`. These installers can even be created on Unix +platforms or Mac OS X. + +If you have a non-pure distribution, the extensions can only be created on a +Windows platform, and will be Python version dependent. The installer filename +will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You +have to create a separate installer for every Python version you want to +support. + +.. TODO Add :term: markup to bytecode when merging into the stdlib + +The installer will try to compile pure modules into bytecode after installation +on the target system in normal and optimizing mode. If you don't want this to +happen for some reason, you can run the :command:`bdist_wininst` command with +the :option:`--no-target-compile` and/or the :option:`--no-target-optimize` +option. + +By default the installer will display the cool "Python Powered" logo when it is +run, but you can also supply your own 152x261 bitmap which must be a Windows +:file:`.bmp` file with the :option:`--bitmap` option. + +The installer will also display a large title on the desktop background window +when it is run, which is constructed from the name of your distribution and the +version number. This can be changed to another text by using the +:option:`--title` option. + +The installer file will be written to the "distribution directory" --- normally +:file:`dist/`, but customizable with the :option:`--dist-dir` option. + +.. _packaging-cross-compile-windows: + +Cross-compiling on Windows +========================== + +Starting with Python 2.6, packaging is capable of cross-compiling between +Windows platforms. In practice, this means that with the correct tools +installed, you can use a 32bit version of Windows to create 64bit extensions +and vice-versa. + +To build for an alternate platform, specify the :option:`--plat-name` option +to the build command. Valid values are currently 'win32', 'win-amd64' and +'win-ia64'. For example, on a 32bit version of Windows, you could execute:: + + python setup.py build --plat-name=win-amd64 + +to build a 64bit version of your extension. The Windows Installers also +support this option, so the command:: + + python setup.py build --plat-name=win-amd64 bdist_wininst + +would create a 64bit installation executable on your 32bit version of Windows. + +To cross-compile, you must download the Python source code and cross-compile +Python itself for the platform you are targetting - it is not possible from a +binary installtion of Python (as the .lib etc file for other platforms are +not included.) In practice, this means the user of a 32 bit operating +system will need to use Visual Studio 2008 to open the +:file:`PCBuild/PCbuild.sln` solution in the Python source tree and build the +"x64" configuration of the 'pythoncore' project before cross-compiling +extensions is possible. + +Note that by default, Visual Studio 2008 does not install 64bit compilers or +tools. You may need to reexecute the Visual Studio setup process and select +these tools (using Control Panel->[Add/Remove] Programs is a convenient way to +check or modify your existing install.) + +.. _packaging-postinstallation-script: + +The Postinstallation script +--------------------------- + +Starting with Python 2.3, a postinstallation script can be specified with the +:option:`--install-script` option. The basename of the script must be +specified, and the script filename must also be listed in the scripts argument +to the setup function. + +This script will be run at installation time on the target system after all the +files have been copied, with ``argv[1]`` set to :option:`-install`, and again at +uninstallation time before the files are removed with ``argv[1]`` set to +:option:`-remove`. + +The installation script runs embedded in the windows installer, every output +(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be +displayed in the GUI after the script has finished. + +Some functions especially useful in this context are available as additional +built-in functions in the installation script. + +.. currentmodule:: bdist_wininst-postinst-script + +.. function:: directory_created(path) + file_created(path) + + These functions should be called when a directory or file is created by the + postinstall script at installation time. It will register *path* with the + uninstaller, so that it will be removed when the distribution is uninstalled. + To be safe, directories are only removed if they are empty. + + +.. function:: get_special_folder_path(csidl_string) + + This function can be used to retrieve special folder locations on Windows like + the Start Menu or the Desktop. It returns the full path to the folder. + *csidl_string* must be one of the following strings:: + + "CSIDL_APPDATA" + + "CSIDL_COMMON_STARTMENU" + "CSIDL_STARTMENU" + + "CSIDL_COMMON_DESKTOPDIRECTORY" + "CSIDL_DESKTOPDIRECTORY" + + "CSIDL_COMMON_STARTUP" + "CSIDL_STARTUP" + + "CSIDL_COMMON_PROGRAMS" + "CSIDL_PROGRAMS" + + "CSIDL_FONTS" + + If the folder cannot be retrieved, :exc:`OSError` is raised. + + Which folders are available depends on the exact Windows version, and probably + also the configuration. For details refer to Microsoft's documentation of the + c:function:`SHGetSpecialFolderPath` function. + + +.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]]) + + This function creates a shortcut. *target* is the path to the program to be + started by the shortcut. *description* is the description of the shortcut. + *filename* is the title of the shortcut that the user will see. *arguments* + specifies the command-line arguments, if any. *workdir* is the working directory + for the program. *iconpath* is the file containing the icon for the shortcut, + and *iconindex* is the index of the icon in the file *iconpath*. Again, for + details consult the Microsoft documentation for the :class:`IShellLink` + interface. + + +Vista User Access Control (UAC) +=============================== + +Starting with Python 2.6, bdist_wininst supports a :option:`--user-access-control` +option. The default is 'none' (meaning no UAC handling is done), and other +valid values are 'auto' (meaning prompt for UAC elevation if Python was +installed for all users) and 'force' (meaning always prompt for elevation). diff --git a/Doc/packaging/commandhooks.rst b/Doc/packaging/commandhooks.rst new file mode 100644 index 0000000..8dc233b --- /dev/null +++ b/Doc/packaging/commandhooks.rst @@ -0,0 +1,31 @@ +============= +Command hooks +============= + +Packaging provides a way of extending its commands by the use of pre- and +post- command hooks. The hooks are simple Python functions (or any callable +objects) and are specified in the config file using their full qualified names. +The pre-hooks are run after the command is finalized (its options are +processed), but before it is run. The post-hooks are run after the command +itself. Both types of hooks receive an instance of the command object. + +Sample usage of hooks +===================== + +Firstly, you need to make sure your hook is present in the path. This is usually +done by dropping them to the same folder where `setup.py` file lives :: + + # file: myhooks.py + def my_install_hook(install_cmd): + print "Oh la la! Someone is installing my project!" + +Then, you need to point to it in your `setup.cfg` file, under the appropriate +command section :: + + [install_dist] + pre-hook.project = myhooks.my_install_hook + +The hooks defined in different config files (system-wide, user-wide and +package-wide) do not override each other as long as they are specified with +different aliases (additional names after the dot). The alias in the example +above is ``project``. diff --git a/Doc/packaging/commandref.rst b/Doc/packaging/commandref.rst new file mode 100644 index 0000000..0d7447d --- /dev/null +++ b/Doc/packaging/commandref.rst @@ -0,0 +1,349 @@ +.. _packaging-command-reference: + +***************** +Command Reference +***************** + +This reference briefly documents all standard Packaging commands and some of +their options. + +.. FIXME does not work: Use pysetup run --help-commands to list all + standard and extra commands availavble on your system, with their + description. Use pysetup run --help to get help about the options + of one command. + + +Preparing distributions +======================= + +:command:`check` +---------------- + +Perform some tests on the metadata of a distribution. + +For example, it verifies that all required metadata fields are provided in the +:file:`setup.cfg` file. + +.. TODO document reST checks + + +:command:`test` +--------------- + +Run a test suite. + +When doing test-driven development, or running automated builds that need +testing before they are installed for downloading or use, it's often useful to +be able to run a project's unit tests without actually installing the project +anywhere. The :command:`test` command runs project's unit tests without +actually installing it, by temporarily putting the project's source on +:data:`sys.path`, after first running :command:`build_ext -i` to ensure that any +C extensions are built. + +You can use this command in one of two ways: either by specifying a +unittest-compatible test suite for your project (or any callable that returns +it) or by passing a test runner function that will run your tests and display +results in the console. Both options take a Python dotted name in the form +``package.module.callable`` to specify the object to use. + +If none of these options are specified, Packaging will try to perform test +discovery using either unittest (for Python 3.2 and higher) or unittest2 (for +older versions, if installed). + +.. this is a pseudo-command name used to disambiguate the options in indexes and + links +.. program:: packaging test + +.. cmdoption:: --suite=NAME, -s NAME + + Specify the test suite (or module, class, or method) to be run. The default + for this option can be set by in the project's :file:`setup.cfg` file:: + + .. code-block:: cfg + + [test] + suite = mypackage.tests.get_all_tests + +.. cmdoption:: --runner=NAME, -r NAME + + Specify the test runner to be called. + + +:command:`config` +----------------- + +Perform distribution configuration. + + +The build step +============== + +This step is mainly useful to compile C/C++ libraries or extension modules. The +build commands can be run manually to check for syntax errors or packaging +issues (for example if the addition of a new source file was forgotten in the +:file:`setup.cfg` file), and is also run automatically by commands which need +it. Packaging checks the mtime of source and built files to avoid re-building +if it's not necessary. + + +:command:`build` +---------------- + +Build all files of a distribution, delegating to the other :command:`build_*` +commands to do the work. + + +:command:`build_clib` +--------------------- + +Build C libraries. + + +:command:`build_ext` +-------------------- + +Build C/C++ extension modules. + + +:command:`build_py` +------------------- + +Build the Python modules (just copy them to the build directory) and +byte-compile them to .pyc files. + + +:command:`build_scripts` +------------------------ +Build the scripts (just copy them to the build directory and adjust their +shebang if they're Python scripts). + + +:command:`clean` +---------------- + +Clean the build tree of the release. + +.. program:: packaging clean + +.. cmdoption:: --all, -a + + Remove build directories for modules, scripts, etc., not only temporary build + by-products. + + +Creating source and built distributions +======================================= + +:command:`sdist` +---------------- + +Build a source distribution for a release. + +It is recommended that you always build and upload a source distribution. Users +of OSes with easy access to compilers and users of advanced packaging tools will +prefer to compile from source rather than using pre-built distributions. For +Windows users, providing a binary installer is also recommended practice. + + +:command:`bdist` +---------------- + +Build a binary distribution for a release. + +This command will call other :command:`bdist_*` commands to create one or more +distributions depending on the options given. The default is to create a +.tar.gz archive on Unix and a zip archive on Windows or OS/2. + +.. program:: packaging bdist + +.. cmdoption:: --formats + + Binary formats to build (comma-separated list). + +.. cmdoption:: --show-formats + + Dump list of available formats. + + +:command:`bdist_dumb` +--------------------- + +Build a "dumb" installer, a simple archive of files that could be unpacked under +``$prefix`` or ``$exec_prefix``. + + +:command:`bdist_wininst` +------------------------ + +Build a Windows installer. + + +:command:`bdist_msi` +-------------------- + +Build a `Microsoft Installer`_ (.msi) file. + +.. _Microsoft Installer: http://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx + +In most cases, the :command:`bdist_msi` installer is a better choice than the +:command:`bdist_wininst` installer, because it provides better support for Win64 +platforms, allows administrators to perform non-interactive installations, and +allows installation through group policies. + + +Publishing distributions +======================== + +:command:`register` +------------------- + +This command registers the current release with the Python Package Index. This +is described in more detail in :PEP:`301`. + +.. TODO explain user and project registration with the web UI + + +:command:`upload` +----------------- + +Upload source and/or binary distributions to PyPI. + +The distributions have to be built on the same command line as the +:command:`upload` command; see :ref:`packaging-package-upload` for more info. + +.. program:: packaging upload + +.. cmdoption:: --sign, -s + + Sign each uploaded file using GPG (GNU Privacy Guard). The ``gpg`` program + must be available for execution on the system ``PATH``. + +.. cmdoption:: --identity=NAME, -i NAME + + Specify the identity or key name for GPG to use when signing. The value of + this option will be passed through the ``--local-user`` option of the + ``gpg`` program. + +.. cmdoption:: --show-response + + Display the full response text from server; this is useful for debugging + PyPI problems. + +.. cmdoption:: --repository=URL, -r URL + + The URL of the repository to upload to. Defaults to + http://pypi.python.org/pypi (i.e., the main PyPI installation). + +.. cmdoption:: --upload-docs + + Also run :command:`upload_docs`. Mainly useful as a default value in + :file:`setup.cfg` (on the command line, it's shorter to just type both + commands). + + +:command:`upload_docs` +---------------------- + +Upload HTML documentation to PyPI. + +PyPI now supports publishing project documentation at a URI of the form +``http://packages.python.org/``. :command:`upload_docs` will create +the necessary zip file out of a documentation directory and will post to the +repository. + +Note that to upload the documentation of a project, the corresponding version +must already be registered with PyPI, using the :command:`register` command --- +just like with :command:`upload`. + +Assuming there is an ``Example`` project with documentation in the subdirectory +:file:`docs`, for example:: + + Example/ + example.py + setup.cfg + docs/ + build/ + html/ + index.html + tips_tricks.html + conf.py + index.txt + tips_tricks.txt + +You can simply specify the directory with the HTML files in your +:file:`setup.cfg` file: + +.. code-block:: cfg + + [upload_docs] + upload-dir = docs/build/html + + +.. program:: packaging upload_docs + +.. cmdoption:: --upload-dir + + The directory to be uploaded to the repository. By default documentation + is searched for in ``docs`` (or ``doc``) directory in project root. + +.. cmdoption:: --show-response + + Display the full response text from server; this is useful for debugging + PyPI problems. + +.. cmdoption:: --repository=URL, -r URL + + The URL of the repository to upload to. Defaults to + http://pypi.python.org/pypi (i.e., the main PyPI installation). + + +The install step +================ + +These commands are used by end-users of a project using :program:`pysetup` or +another compatible installer. Each command will run the corresponding +:command:`build_*` command and then move the built files to their destination on +the target system. + + +:command:`install_dist` +----------------------- + +Install a distribution, delegating to the other :command:`install_*` commands to +do the work. + +.. program:: packaging install_dist + +.. cmdoption:: --user + + Install in user site-packages directory (see :PEP:`370`). + + +:command:`install_data` +----------------------- + +Install data files. + + +:command:`install_distinfo` +--------------------------- + +Install files recording details of the installation as specified in :PEP:`376`. + + +:command:`install_headers` +-------------------------- + +Install C/C++ header files. + + +:command:`install_lib` +---------------------- + +Install C library files. + + +:command:`install_scripts` +-------------------------- + +Install scripts. diff --git a/Doc/packaging/configfile.rst b/Doc/packaging/configfile.rst new file mode 100644 index 0000000..825b5cb --- /dev/null +++ b/Doc/packaging/configfile.rst @@ -0,0 +1,125 @@ +.. _packaging-setup-config: + +************************************ +Writing the Setup Configuration File +************************************ + +Often, it's not possible to write down everything needed to build a distribution +*a priori*: you may need to get some information from the user, or from the +user's system, in order to proceed. As long as that information is fairly +simple---a list of directories to search for C header files or libraries, for +example---then providing a configuration file, :file:`setup.cfg`, for users to +edit is a cheap and easy way to solicit it. Configuration files also let you +provide default values for any command option, which the installer can then +override either on the command line or by editing the config file. + +The setup configuration file is a useful middle-ground between the setup script +---which, ideally, would be opaque to installers [#]_---and the command line to +the setup script, which is outside of your control and entirely up to the +installer. In fact, :file:`setup.cfg` (and any other Distutils configuration +files present on the target system) are processed after the contents of the +setup script, but before the command line. This has several useful +consequences: + +.. If you have more advanced needs, such as determining which extensions to + build based on what capabilities are present on the target system, then you + need the Distutils auto-configuration facility. This started to appear in + Distutils 0.9 but, as of this writing, isn't mature or stable enough yet + for real-world use. + +* installers can override some of what you put in :file:`setup.py` by editing + :file:`setup.cfg` + +* you can provide non-standard defaults for options that are not easily set in + :file:`setup.py` + +* installers can override anything in :file:`setup.cfg` using the command-line + options to :file:`setup.py` + +The basic syntax of the configuration file is simple:: + + [command] + option = value + ... + +where *command* is one of the Distutils commands (e.g. :command:`build_py`, +:command:`install_dist`), and *option* is one of the options that command supports. +Any number of options can be supplied for each command, and any number of +command sections can be included in the file. Blank lines are ignored, as are +comments, which run from a ``'#'`` character until the end of the line. Long +option values can be split across multiple lines simply by indenting the +continuation lines. + +You can find out the list of options supported by a particular command with the +universal :option:`--help` option, e.g. :: + + > python setup.py --help build_ext + [...] + Options for 'build_ext' command: + --build-lib (-b) directory for compiled extension modules + --build-temp (-t) directory for temporary files (build by-products) + --inplace (-i) ignore build-lib and put compiled extensions into the + source directory alongside your pure Python modules + --include-dirs (-I) list of directories to search for header files + --define (-D) C preprocessor macros to define + --undef (-U) C preprocessor macros to undefine + --swig-opts list of SWIG command-line options + [...] + +.. XXX do we want to support ``setup.py --help metadata``? + +Note that an option spelled :option:`--foo-bar` on the command line is spelled +:option:`foo_bar` in configuration files. + +For example, say you want your extensions to be built "in-place"---that is, you +have an extension :mod:`pkg.ext`, and you want the compiled extension file +(:file:`ext.so` on Unix, say) to be put in the same source directory as your +pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the +:option:`--inplace` option on the command line to ensure this:: + + python setup.py build_ext --inplace + +But this requires that you always specify the :command:`build_ext` command +explicitly, and remember to provide :option:`--inplace`. An easier way is to +"set and forget" this option, by encoding it in :file:`setup.cfg`, the +configuration file for this distribution:: + + [build_ext] + inplace = 1 + +This will affect all builds of this module distribution, whether or not you +explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in +your source distribution, it will also affect end-user builds---which is +probably a bad idea for this option, since always building extensions in-place +would break installation of the module distribution. In certain peculiar cases, +though, modules are built right in their installation directory, so this is +conceivably a useful ability. (Distributing extensions that expect to be built +in their installation directory is almost always a bad idea, though.) + +Another example: certain commands take options that vary from project to +project but not depending on the installation system, for example, +:command:`test` needs to know where your test suite is located and what test +runner to use; likewise, :command:`upload_docs` can find HTML documentation in +a :file:`doc` or :file:`docs` directory, but needs an option to find files in +:file:`docs/build/html`. Instead of having to type out these options each +time you want to run the command, you can put them in the project's +:file:`setup.cfg`:: + + [test] + suite = packaging.tests + + [upload_docs] + upload-dir = docs/build/html + + +.. seealso:: + + :ref:`packaging-config-syntax` in "Installing Python Projects" + More information on the configuration files is available in the manual for + system administrators. + + +.. rubric:: Footnotes + +.. [#] This ideal probably won't be achieved until auto-configuration is fully + supported by the Distutils. diff --git a/Doc/packaging/examples.rst b/Doc/packaging/examples.rst new file mode 100644 index 0000000..594ade0 --- /dev/null +++ b/Doc/packaging/examples.rst @@ -0,0 +1,334 @@ +.. _packaging-examples: + +******** +Examples +******** + +This chapter provides a number of basic examples to help get started with +Packaging. + + +.. _packaging-pure-mod: + +Pure Python distribution (by module) +==================================== + +If you're just distributing a couple of modules, especially if they don't live +in a particular package, you can specify them individually using the +:option:`py_modules` option in the setup script. + +In the simplest case, you'll have two files to worry about: a setup script and +the single module you're distributing, :file:`foo.py` in this example:: + + / + setup.py + foo.py + +(In all diagrams in this section, ** will refer to the distribution root +directory.) A minimal setup script to describe this situation would be:: + + from packaging.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo']) + +Note that the name of the distribution is specified independently with the +:option:`name` option, and there's no rule that says it has to be the same as +the name of the sole module in the distribution (although that's probably a good +convention to follow). However, the distribution name is used to generate +filenames, so you should stick to letters, digits, underscores, and hyphens. + +Since :option:`py_modules` is a list, you can of course specify multiple +modules, e.g. if you're distributing modules :mod:`foo` and :mod:`bar`, your +setup might look like this:: + + / + setup.py + foo.py + bar.py + +and the setup script might be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + py_modules=['foo', 'bar']) + +You can put module source files into another directory, but if you have enough +modules to do that, it's probably easier to specify modules by package rather +than listing them individually. + + +.. _packaging-pure-pkg: + +Pure Python distribution (by package) +===================================== + +If you have more than a couple of modules to distribute, especially if they are +in multiple packages, it's probably easier to specify whole packages rather than +individual modules. This works even if your modules are not in a package; you +can just tell the Distutils to process modules from the root package, and that +works the same as any other package (except that you don't have to have an +:file:`__init__.py` file). + +The setup script from the last example could also be written as :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['']) + +(The empty string stands for the root package.) + +If those two files are moved into a subdirectory, but remain in the root +package, e.g.:: + + / + setup.py + src/ + foo.py + bar.py + +then you would still specify the root package, but you have to tell the +Distutils where source files in the root package live:: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'': 'src'}, + packages=['']) + +More typically, though, you will want to distribute multiple modules in the same +package (or in sub-packages). For example, if the :mod:`foo` and :mod:`bar` +modules belong in package :mod:`foobar`, one way to lay out your source tree is + +:: + + / + setup.py + foobar/ + __init__.py + foo.py + bar.py + +This is in fact the default layout expected by the Distutils, and the one that +requires the least work to describe in your setup script:: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar']) + +If you want to put modules in directories not named for their package, then you +need to use the :option:`package_dir` option again. For example, if the +:file:`src` directory holds modules in the :mod:`foobar` package:: + + / + setup.py + src/ + __init__.py + foo.py + bar.py + +an appropriate setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': 'src'}, + packages=['foobar']) + +Or, you might put modules from your main package right in the distribution +root:: + + / + setup.py + __init__.py + foo.py + bar.py + +in which case your setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + package_dir={'foobar': ''}, + packages=['foobar']) + +(The empty string also stands for the current directory.) + +If you have sub-packages, they must be explicitly listed in :option:`packages`, +but any entries in :option:`package_dir` automatically extend to sub-packages. +(In other words, the Distutils does *not* scan your source tree, trying to +figure out which directories correspond to Python packages by looking for +:file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: + + / + setup.py + foobar/ + __init__.py + foo.py + bar.py + subfoo/ + __init__.py + blah.py + +then the corresponding setup script would be :: + + from packaging.core import setup + setup(name='foobar', + version='1.0', + packages=['foobar', 'foobar.subfoo']) + +(Again, the empty string in :option:`package_dir` stands for the current +directory.) + + +.. _packaging-single-ext: + +Single extension module +======================= + +Extension modules are specified using the :option:`ext_modules` option. +:option:`package_dir` has no effect on where extension source files are found; +it only affects the source for pure Python modules. The simplest case, a +single extension module in a single C source file, is:: + + / + setup.py + foo.c + +If the :mod:`foo` extension belongs in the root package, the setup script for +this could be :: + + from packaging.core import setup, Extension + setup(name='foobar', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])]) + +If the extension actually belongs in a package, say :mod:`foopkg`, then + +With exactly the same source tree layout, this extension can be put in the +:mod:`foopkg` package simply by changing the name of the extension:: + + from packaging.core import setup, Extension + setup(name='foobar', + version='1.0', + packages=['foopkg'], + ext_modules=[Extension('foopkg.foo', ['foo.c'])]) + + +Checking metadata +================= + +The ``check`` command allows you to verify if your project's metadata +meets the minimum requirements to build a distribution. + +To run it, just call it using your :file:`setup.py` script. If something is +missing, ``check`` will display a warning. + +Let's take an example with a simple script:: + + from packaging.core import setup + + setup(name='foobar') + +.. TODO configure logging StreamHandler to match this output + +Running the ``check`` command will display some warnings:: + + $ python setup.py check + running check + warning: check: missing required metadata: version, home_page + warning: check: missing metadata: either (author and author_email) or + (maintainer and maintainer_email) must be supplied + + +If you use the reStructuredText syntax in the ``long_description`` field and +`Docutils `_ is installed you can check if +the syntax is fine with the ``check`` command, using the ``restructuredtext`` +option. + +For example, if the :file:`setup.py` script is changed like this:: + + from packaging.core import setup + + desc = """\ + Welcome to foobar! + =============== + + This is the description of the ``foobar`` project. + """ + + setup(name='foobar', + version='1.0', + author=u'Tarek Ziadé', + author_email='tarek@ziade.org', + summary='Foobar utilities' + description=desc, + home_page='http://example.com') + +Where the long description is broken, ``check`` will be able to detect it +by using the :mod:`docutils` parser:: + + $ python setup.py check --restructuredtext + running check + warning: check: Title underline too short. (line 2) + warning: check: Could not finish the parsing. + + +.. _packaging-reading-metadata: + +Reading the metadata +==================== + +The :func:`packaging.core.setup` function provides a command-line interface +that allows you to query the metadata fields of a project through the +:file:`setup.py` script of a given project:: + + $ python setup.py --name + foobar + +This call reads the ``name`` metadata by running the +:func:`packaging.core.setup` function. When a source or binary +distribution is created with Distutils, the metadata fields are written +in a static file called :file:`PKG-INFO`. When a Distutils-based project is +installed in Python, the :file:`PKG-INFO` file is copied alongside the modules +and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, +where ``NAME`` is the name of the project, ``VERSION`` its version as defined +in the Metadata, and ``pyX.X`` the major and minor version of Python like +``2.7`` or ``3.2``. + +You can read back this static file, by using the +:class:`packaging.dist.Metadata` class and its +:func:`read_pkg_file` method:: + + >>> from packaging.metadata import Metadata + >>> metadata = Metadata() + >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) + >>> metadata.name + 'distribute' + >>> metadata.version + '0.6.8' + >>> metadata.description + 'Easily download, build, install, upgrade, and uninstall Python packages' + +Notice that the class can also be instantiated with a metadata file path to +loads its values:: + + >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' + >>> Metadata(pkg_info_path).name + 'distribute' + + +.. XXX These comments have been here for at least ten years. Write the + sections or delete the comments (we can maybe ask Greg Ward about + the planned contents). (Unindent to make them section titles) + + .. multiple-ext:: + + Multiple extension modules + ========================== + + Putting it all together + ======================= diff --git a/Doc/packaging/extending.rst b/Doc/packaging/extending.rst new file mode 100644 index 0000000..f2d3863 --- /dev/null +++ b/Doc/packaging/extending.rst @@ -0,0 +1,95 @@ +.. _extending-packaging: + +******************* +Extending Distutils +******************* + +Distutils can be extended in various ways. Most extensions take the form of new +commands or replacements for existing commands. New commands may be written to +support new types of platform-specific packaging, for example, while +replacements for existing commands may be made to modify details of how the +command operates on a package. + +Most extensions of the packaging are made within :file:`setup.py` scripts that +want to modify existing commands; many simply add a few file extensions that +should be copied into packages in addition to :file:`.py` files as a +convenience. + +Most packaging command implementations are subclasses of the +:class:`packaging.cmd.Command` class. New commands may directly inherit from +:class:`Command`, while replacements often derive from :class:`Command` +indirectly, directly subclassing the command they are replacing. Commands are +required to derive from :class:`Command`. + +.. .. _extend-existing: + Extending existing commands + =========================== + + +.. .. _new-commands: + Writing new commands + ==================== + + +Integrating new commands +======================== + +There are different ways to integrate new command implementations into +packaging. The most difficult is to lobby for the inclusion of the new features +in packaging itself, and wait for (and require) a version of Python that +provides that support. This is really hard for many reasons. + +The most common, and possibly the most reasonable for most needs, is to include +the new implementations with your :file:`setup.py` script, and cause the +:func:`packaging.core.setup` function use them:: + + from packaging.core import setup + from packaging.command.build_py import build_py as _build_py + + class build_py(_build_py): + """Specialized Python source builder.""" + + # implement whatever needs to be different... + + setup(..., cmdclass={'build_py': build_py}) + +This approach is most valuable if the new implementations must be used to use a +particular package, as everyone interested in the package will need to have the +new command implementation. + +Beginning with Python 2.4, a third option is available, intended to allow new +commands to be added which can support existing :file:`setup.py` scripts without +requiring modifications to the Python installation. This is expected to allow +third-party extensions to provide support for additional packaging systems, but +the commands can be used for anything packaging commands can be used for. A new +configuration option, :option:`command_packages` (command-line option +:option:`--command-packages`), can be used to specify additional packages to be +searched for modules implementing commands. Like all packaging options, this +can be specified on the command line or in a configuration file. This option +can only be set in the ``[global]`` section of a configuration file, or before +any commands on the command line. If set in a configuration file, it can be +overridden from the command line; setting it to an empty string on the command +line causes the default to be used. This should never be set in a configuration +file provided with a package. + +This new option can be used to add any number of packages to the list of +packages searched for command implementations; multiple package names should be +separated by commas. When not specified, the search is only performed in the +:mod:`packaging.command` package. When :file:`setup.py` is run with the option +:option:`--command-packages` :option:`distcmds,buildcmds`, however, the packages +:mod:`packaging.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched +in that order. New commands are expected to be implemented in modules of the +same name as the command by classes sharing the same name. Given the example +command-line option above, the command :command:`bdist_openpkg` could be +implemented by the class :class:`distcmds.bdist_openpkg.bdist_openpkg` or +:class:`buildcmds.bdist_openpkg.bdist_openpkg`. + + +Adding new distribution types +============================= + +Commands that create distributions (files in the :file:`dist/` directory) need +to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that +:command:`upload` can upload it to PyPI. The *filename* in the pair contains no +path information, only the name of the file itself. In dry-run mode, pairs +should still be added to represent what would have been created. diff --git a/Doc/packaging/index.rst b/Doc/packaging/index.rst new file mode 100644 index 0000000..1b597d7 --- /dev/null +++ b/Doc/packaging/index.rst @@ -0,0 +1,45 @@ +.. _packaging-index: + +############################## + Distributing Python Projects +############################## + +:Authors: Greg Ward, Anthony Baxter and Packaging contributors +:Email: distutils-sig@python.org +:Release: |version| +:Date: |today| + +This document describes Packaging for Python authors, describing how to use the +module to make Python applications, packages or modules easily available to a +wider audience with very little overhead for build/release/install mechanics. + +.. toctree:: + :maxdepth: 2 + :numbered: + + tutorial + setupcfg + introduction + setupscript + configfile + sourcedist + builtdist + packageindex + uploading + examples + extending + commandhooks + commandref + + +.. seealso:: + + :ref:`packaging-install-index` + A user-centered manual which includes information on adding projects + into an existing Python installation. You do not need to be a Python + programmer to read this manual. + + :mod:`packaging` + A library reference for developers of packaging tools wanting to use + standalone building blocks like :mod:`~packaging.version` or + :mod:`~packaging.metadata`, or extend Packaging itself. diff --git a/Doc/packaging/introduction.rst b/Doc/packaging/introduction.rst new file mode 100644 index 0000000..db0ffbb --- /dev/null +++ b/Doc/packaging/introduction.rst @@ -0,0 +1,193 @@ +.. _packaging-intro: + +***************************** +An Introduction to Packaging +***************************** + +This document covers using Packaging to distribute your Python modules, +concentrating on the role of developer/distributor. If you're looking for +information on installing Python modules you should refer to the +:ref:`packaging-install-index` chapter. + +Throughout this documentation, the terms "Distutils", "the Distutils" and +"Packaging" will be used interchangeably. + +.. _packaging-concepts: + +Concepts & Terminology +====================== + +Using Distutils is quite simple both for module developers and for +users/administrators installing third-party modules. As a developer, your +responsibilities (apart from writing solid, well-documented and well-tested +code, of course!) are: + +* writing a setup script (:file:`setup.py` by convention) + +* (optional) writing a setup configuration file + +* creating a source distribution + +* (optional) creating one or more "built" (binary) distributions of your + project + +All of these tasks are covered in this document. + +Not all module developers have access to multiple platforms, so one cannot +expect them to create buildt distributions for every platform. To remedy +this, it is hoped that intermediaries called *packagers* will arise to address +this need. Packagers take source distributions released by module developers, +build them on one or more platforms and release the resulting built +distributions. Thus, users on a greater range of platforms will be able to +install the most popular Python modules in the most natural way for their +platform without having to run a setup script or compile a single line of code. + + +.. _packaging-simple-example: + +A Simple Example +================ + +A setup script is usually quite simple, although since it's written in Python +there are no arbitrary limits to what you can do with it, though you should be +careful about putting expensive operations in your setup script. +Unlike, say, Autoconf-style configure scripts the setup script may be run +multiple times in the course of building and installing a module +distribution. + +If all you want to do is distribute a module called :mod:`foo`, contained in a +file :file:`foo.py`, then your setup script can be as simple as:: + + from packaging.core import setup + setup(name='foo', + version='1.0', + py_modules=['foo']) + +Some observations: + +* most information that you supply to the Distutils is supplied as keyword + arguments to the :func:`setup` function + +* those keyword arguments fall into two categories: package metadata (name, + version number, etc.) and information about what's in the package (a list + of pure Python modules in this case) + +* modules are specified by module name, not filename (the same will hold true + for packages and extensions) + +* it's recommended that you supply a little more metadata than we have in the + example. In particular your name, email address and a URL for the + project if appropriate (see section :ref:`packaging-setup-script` for an example) + +To create a source distribution for this module you would create a setup +script, :file:`setup.py`, containing the above code and run:: + + python setup.py sdist + +which will create an archive file (e.g., tarball on Unix, ZIP file on Windows) +containing your setup script :file:`setup.py`, and your module :file:`foo.py`. +The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and +will unpack into a directory :file:`foo-1.0`. + +If an end-user wishes to install your :mod:`foo` module all he has to do is +download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and from the +:file:`foo-1.0` directory run :: + + python setup.py install + +which will copy :file:`foo.py` to the appropriate directory for +third-party modules in their Python installation. + +This simple example demonstrates some fundamental concepts of Distutils. +First, both developers and installers have the same basic user interface, i.e. +the setup script. The difference is which Distutils *commands* they use: the +:command:`sdist` command is almost exclusively for module developers, while +:command:`install` is more often used by installers (although some developers +will want to install their own code occasionally). + +If you want to make things really easy for your users, you can create more +than one built distributions for them. For instance, if you are running on a +Windows machine and want to make things easy for other Windows users, you can +create an executable installer (the most appropriate type of built distribution +for this platform) with the :command:`bdist_wininst` command. For example:: + + python setup.py bdist_wininst + +will create an executable installer, :file:`foo-1.0.win32.exe`, in the current +directory. You can find out what distribution formats are available at any time +by running :: + + python setup.py bdist --help-formats + + +.. _packaging-python-terms: + +General Python terminology +========================== + +If you're reading this document, you probably have a good idea of what Python +modules, extensions and so forth are. Nevertheless, just to be sure that +everyone is on the same page, here's a quick overview of Python terms: + +module + The basic unit of code reusability in Python: a block of code imported by + some other code. Three types of modules are important to us here: pure + Python modules, extension modules and packages. + +pure Python module + A module written in Python and contained in a single :file:`.py` file (and + possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes + referred to as a "pure module." + +extension module + A module written in the low-level language of the Python implementation: C/C++ + for Python, Java for Jython. Typically contained in a single dynamically + loaded pre-compiled file, e.g. a shared object (:file:`.so`) file for Python + extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python + extensions on Windows, or a Java class file for Jython extensions. Note that + currently Distutils only handles C/C++ extensions for Python. + +package + A module that contains other modules, typically contained in a directory of + the filesystem and distinguished from other directories by the presence of a + file :file:`__init__.py`. + +root package + The root of the hierarchy of packages. (This isn't really a package, + since it doesn't have an :file:`__init__.py` file. But... we have to + call it something, right?) The vast majority of the standard library is + in the root package, as are many small standalone third-party modules that + don't belong to a larger module collection. Unlike regular packages, + modules in the root package can be found in many directories: in fact, + every directory listed in ``sys.path`` contributes modules to the root + package. + + +.. _packaging-term: + +Distutils-specific terminology +============================== + +The following terms apply more specifically to the domain of distributing Python +modules using Distutils: + +module distribution + A collection of Python modules distributed together as a single downloadable + resource and meant to be installed all as one. Examples of some well-known + module distributions are NumPy, SciPy, PIL (the Python Imaging + Library) or mxBase. (Module distributions would be called a *package*, + except that term is already taken in the Python context: a single module + distribution may contain zero, one, or many Python packages.) + +pure module distribution + A module distribution that contains only pure Python modules and packages. + Sometimes referred to as a "pure distribution." + +non-pure module distribution + A module distribution that contains at least one extension module. Sometimes + referred to as a "non-pure distribution." + +distribution root + The top-level directory of your source tree (or source distribution). The + directory where :file:`setup.py` exists. Generally :file:`setup.py` will + be run from this directory. diff --git a/Doc/packaging/packageindex.rst b/Doc/packaging/packageindex.rst new file mode 100644 index 0000000..cd1d598 --- /dev/null +++ b/Doc/packaging/packageindex.rst @@ -0,0 +1,104 @@ +.. _packaging-package-index: + +********************************** +Registering with the Package Index +********************************** + +The Python Package Index (PyPI) holds metadata describing distributions +packaged with packaging. The packaging command :command:`register` is used to +submit your distribution's metadata to the index. It is invoked as follows:: + + python setup.py register + +Distutils will respond with the following prompt:: + + running register + We need to know who you are, so please choose either: + 1. use your existing login, + 2. register as a new user, + 3. have the server generate a new password for you (and email it to you), or + 4. quit + Your selection [default 1]: + +Note: if your username and password are saved locally, you will not see this +menu. + +If you have not registered with PyPI, then you will need to do so now. You +should choose option 2, and enter your details as required. Soon after +submitting your details, you will receive an email which will be used to confirm +your registration. + +Once you are registered, you may choose option 1 from the menu. You will be +prompted for your PyPI username and password, and :command:`register` will then +submit your metadata to the index. + +You may submit any number of versions of your distribution to the index. If you +alter the metadata for a particular version, you may submit it again and the +index will be updated. + +PyPI holds a record for each (name, version) combination submitted. The first +user to submit information for a given name is designated the Owner of that +name. They may submit changes through the :command:`register` command or through +the web interface. They may also designate other users as Owners or Maintainers. +Maintainers may edit the package information, but not designate other Owners or +Maintainers. + +By default PyPI will list all versions of a given package. To hide certain +versions, the Hidden property should be set to yes. This must be edited through +the web interface. + + +.. _packaging-pypirc: + +The .pypirc file +================ + +The format of the :file:`.pypirc` file is as follows:: + + [packaging] + index-servers = + pypi + + [pypi] + repository: + username: + password: + +The *packaging* section defines a *index-servers* variable that lists the +name of all sections describing a repository. + +Each section describing a repository defines three variables: + +- *repository*, that defines the url of the PyPI server. Defaults to + ``http://www.python.org/pypi``. +- *username*, which is the registered username on the PyPI server. +- *password*, that will be used to authenticate. If omitted the user + will be prompt to type it when needed. + +If you want to define another server a new section can be created and +listed in the *index-servers* variable:: + + [packaging] + index-servers = + pypi + other + + [pypi] + repository: + username: + password: + + [other] + repository: http://example.com/pypi + username: + password: + +:command:`register` can then be called with the -r option to point the +repository to work with:: + + python setup.py register -r http://example.com/pypi + +For convenience, the name of the section that describes the repository +may also be used:: + + python setup.py register -r other diff --git a/Doc/packaging/setupcfg.rst b/Doc/packaging/setupcfg.rst new file mode 100644 index 0000000..be6c8c9 --- /dev/null +++ b/Doc/packaging/setupcfg.rst @@ -0,0 +1,648 @@ +.. highlightlang:: cfg + +******************************************* +Specification of the :file:`setup.cfg` file +******************************************* + +.. :version: 1.0 + +This document describes the :file:`setup.cfg`, an ini-style configuration file +(compatible with :class:`configparser.RawConfigParser`) configuration file used +by Packaging to replace the :file:`setup.py` file. + +Each section contains a description of its options. + +- Options that are marked *multi* can have multiple values, one value per + line. +- Options that are marked *optional* can be omitted. +- Options that are marked *environ* can use environment markers, as described + in :PEP:`345`. + + +The sections are: + +global + Global options not related to one command. + +metadata + Name, version and other information defined by :PEP:`345`. + +files + Modules, scripts, data, documentation and other files to include in the + distribution. + +command sections + Options given for specific commands, identical to those that can be given + on the command line. + + +Global options +============== + +Contains global options for Packaging. This section is shared with Distutils. + + +commands + Defined Packaging command. A command is defined by its fully + qualified name. *optional*, *multi* + + Examples:: + + [global] + commands = + package.setup.CustomSdistCommand + package.setup.BdistDeb + +compilers + Defined Packaging compiler. A compiler is defined by its fully + qualified name. *optional*, *multi* + + Example:: + + [global] + compilers = + hotcompiler.SmartCCompiler + +setup_hook + defines a callable that will be called right after the + :file:`setup.cfg` file is read. The callable receives the configuration + in form of a mapping and can make some changes to it. *optional* + + Example:: + + [global] + setup_hook = package.setup.customize_dist + + +Metadata +======== + +The metadata section contains the metadata for the project as described in +:PEP:`345`. Field names are case-insensitive. + +Fields: + +name + Name of the project. + +version + Version of the project. Must comply with :PEP:`386`. + +platform + Platform specification describing an operating system + supported by the distribution which is not listed in the "Operating System" + Trove classifiers (:PEP:`301`). *optional*, *multi* + +supported-platform + Binary distributions containing a PKG-INFO file will + use the Supported-Platform field in their metadata to specify the OS and + CPU for which the binary distribution was compiled. The semantics of + the Supported-Platform field are free form. *optional*, *multi* + +summary + A one-line summary of what the distribution does. + (Used to be called *description* in Distutils1.) + +description + A longer description. (Used to be called *long_description* + in Distutils1.) A file can be provided in the *description-file* field. + *optional* + +description-file + path to a text file that will be used for the + **description** field. *optional* + +keywords + A list of additional keywords to be used to assist searching + for the distribution in a larger catalog. Comma or space-separated. + *optional* + +home-page + The URL for the distribution's home page. + +download-url + The URL from which this version of the distribution + can be downloaded. *optional* + +author + Author's name. *optional* + +author-email + Author's e-mail. *optional* + +maintainer + Maintainer's name. *optional* + +maintainer-email + Maintainer's e-mail. *optional* + +license + A text indicating the term of uses, when a trove classifier does + not match. *optional*. + +classifiers + Classification for the distribution, as described in PEP 301. + *optional*, *multi*, *environ* + +requires-dist + name of another packaging project required as a dependency. + The format is *name (version)* where version is an optional + version declaration, as described in PEP 345. *optional*, *multi*, *environ* + +provides-dist + name of another packaging project contained within this + distribution. Same format than *requires-dist*. *optional*, *multi*, + *environ* + +obsoletes-dist + name of another packaging project this version obsoletes. + Same format than *requires-dist*. *optional*, *multi*, *environ* + +requires-python + Specifies the Python version the distribution requires. + The value is a version number, as described in PEP 345. + *optional*, *multi*, *environ* + +requires-externals + a dependency in the system. This field is free-form, + and just a hint for downstream maintainers. *optional*, *multi*, + *environ* + +project-url + A label, followed by a browsable URL for the project. + "label, url". The label is limited to 32 signs. *optional*, *multi* + + +Example:: + + [metadata] + name = pypi2rpm + version = 0.1 + author = Tarek Ziadé + author-email = tarek@ziade.org + summary = Script that transforms an sdist archive into a RPM package + description-file = README + home-page = http://bitbucket.org/tarek/pypi2rpm/wiki/Home + project-url: + Repository, http://bitbucket.org/tarek/pypi2rpm/ + RSS feed, https://bitbucket.org/tarek/pypi2rpm/rss + classifier = + Development Status :: 3 - Alpha + License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1) + +You should not give any explicit value for metadata-version: it will be guessed +from the fields present in the file. + + +Files +===== + +This section describes the files included in the project. + +packages_root + the root directory containing all packages and modules + (default: current directory). *optional* + +packages + a list of packages the project includes *optional*, *multi* + +modules + a list of packages the project includes *optional*, *multi* + +scripts + a list of scripts the project includes *optional*, *multi* + +extra_files + a list of patterns to include extra files *optional*, + *multi* + +Example:: + + [files] + packages_root = src + packages = + pypi2rpm + pypi2rpm.command + + scripts = + pypi2rpm/pypi2rpm.py + + extra_files = + setup.py + README + + +.. Note:: + The :file:`setup.cfg` configuration file is included by default. Contrary to + Distutils, :file:`README` (or :file:`README.txt`) and :file:`setup.py` are + not included by default. + + +Resources +--------- + +This section describes the files used by the project which must not be installed +in the same place that python modules or libraries, they are called +**resources**. They are for example documentation files, script files, +databases, etc... + +For declaring resources, you must use this notation:: + + source = destination + +Data-files are declared in the **resources** field in the **file** section, for +example:: + + [files] + resources = + source1 = destination1 + source2 = destination2 + +The **source** part of the declaration are relative paths of resources files +(using unix path separator **/**). For example, if you've this source tree:: + + foo/ + doc/ + doc.man + scripts/ + foo.sh + +Your setup.cfg will look like:: + + [files] + resources = + doc/doc.man = destination_doc + scripts/foo.sh = destination_scripts + +The final paths where files will be placed are composed by : **source** + +**destination**. In the previous example, **doc/doc.man** will be placed in +**destination_doc/doc/doc.man** and **scripts/foo.sh** will be placed in +**destination_scripts/scripts/foo.sh**. (If you want more control on the final +path, take a look at base_prefix_). + +The **destination** part of resources declaration are paths with categories. +Indeed, it's generally a bad idea to give absolute path as it will be cross +incompatible. So, you must use resources categories in your **destination** +declaration. Categories will be replaced by their real path at the installation +time. Using categories is all benefit, your declaration will be simpler, cross +platform and it will allow packager to place resources files where they want +without breaking your code. + +Categories can be specified by using this syntax:: + + {category} + +Default categories are: + +* config +* appdata +* appdata.arch +* appdata.persistent +* appdata.disposable +* help +* icon +* scripts +* doc +* info +* man + +A special category also exists **{distribution.name}** that will be replaced by +the name of the distribution, but as most of the defaults categories use them, +so it's not necessary to add **{distribution.name}** into your destination. + +If you use categories in your declarations, and you are encouraged to do, final +path will be:: + + source + destination_expanded + +.. _example_final_path: + +For example, if you have this setup.cfg:: + + [metadata] + name = foo + + [files] + resources = + doc/doc.man = {doc} + +And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final +path will be:: + + {datadir}/doc/foo/doc/doc.man + +Where {datafir} category will be platform-dependent. + + +More control on source part +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Glob syntax +""""""""""" + +When you declare source file, you can use a glob-like syntax to match multiples file, for example:: + + scripts/* = {script} + +Will match all the files in the scripts directory and placed them in the script category. + +Glob tokens are: + + * ``*``: match all files. + * ``?``: match any character. + * ``**``: match any level of tree recursion (even 0). + * ``{}``: will match any part separated by comma (example: ``{sh,bat}``). + +.. TODO Add examples + +Order of declaration +"""""""""""""""""""" + +The order of declaration is important if one file match multiple rules. The last +rules matched by file is used, this is useful if you have this source tree:: + + foo/ + doc/ + index.rst + setup.rst + documentation.txt + doc.tex + README + +And you want all the files in the doc directory to be placed in {doc} category, +but README must be placed in {help} category, instead of listing all the files +one by one, you can declare them in this way:: + + [files] + resources = + doc/* = {doc} + doc/README = {help} + +Exclude +""""""" + +You can exclude some files of resources declaration by giving no destination, it +can be useful if you have a non-resources file in the same directory of +resources files:: + + foo/ + doc/ + RELEASES + doc.tex + documentation.txt + docu.rst + +Your **files** section will be:: + + [files] + resources = + doc/* = {doc} + doc/RELEASES = + +More control on destination part +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. _base_prefix: + +Defining a base prefix +"""""""""""""""""""""" + +When you define your resources, you can have more control of how the final path +is compute. + +By default, the final path is:: + + destination + source + +This can generate long paths, for example (example_final_path_):: + + {datadir}/doc/foo/doc/doc.man + +When you declare your source, you can use whitespace to split the source in +**prefix** **suffix**. So, for example, if you have this source:: + + docs/ doc.man + +The **prefix** is "docs/" and the **suffix** is "doc.html". + +.. note:: + + Separator can be placed after a path separator or replace it. So these two + sources are equivalent:: + + docs/ doc.man + docs doc.man + +.. note:: + + Glob syntax is working the same way with standard source and splitted source. + So these rules:: + + docs/* + docs/ * + docs * + + Will match all the files in the docs directory. + +When you use splitted source, the final path is compute in this way:: + + destination + prefix + +So for example, if you have this setup.cfg:: + + [metadata] + name = foo + + [files] + resources = + doc/ doc.man = {doc} + +And if **{doc}** is replaced by **{datadir}/doc/{distribution.name}**, final +path will be:: + + {datadir}/doc/foo/doc.man + + +Overwriting paths for categories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This part is intended for system administrators or downstream OS packagers. + +The real paths of categories are registered in the *sysconfig.cfg* file +installed in your python installation. This file uses an ini format too. +The content of the file is organized into several sections: + +* globals: Standard categories's paths. +* posix_prefix: Standard paths for categories and installation paths for posix + system. +* other ones XXX + +Standard categories paths are platform independent, they generally refers to +other categories, which are platform dependent. :mod:`sysconfig` will choose +these category from sections matching os.name. For example:: + + doc = {datadir}/doc/{distribution.name} + +It refers to datadir category, which can be different between platforms. In +posix system, it may be:: + + datadir = /usr/share + +So the final path will be:: + + doc = /usr/share/doc/{distribution.name} + +The platform-dependent categories are: + +* confdir +* datadir +* libdir +* base + + +Defining extra categories +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. TODO + + +Examples +^^^^^^^^ + +These examples are incremental but work unitarily. + +Resources in root dir +""""""""""""""""""""" + +Source tree:: + + babar-1.0/ + README + babar.sh + launch.sh + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + *.sh = {scripts} + +So babar.sh and launch.sh will be placed in {scripts} directory. + +Now let's move all the scripts into a scripts directory. + +Resources in sub-directory +"""""""""""""""""""""""""" + +Source tree:: + + babar-1.1/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.sh = {scripts} + +It's important to use the separator after scripts/ to install all the shell +scripts into {scripts} instead of {scripts}/scripts. + +Now let's add some docs. + +Resources in multiple sub-directories +""""""""""""""""""""""""""""""""""""" + +Source tree:: + + babar-1.2/ + README + scripts/ + babar.sh + launch.sh + LAUNCH + docs/ + api + man + babar.py + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.sh = {scripts} + doc/ * = {doc} + doc/ man = {man} + +You want to place all the file in the docs script into {doc} category, instead +of man, which must be placed into {man} category, we will use the order of +declaration of globs to choose the destination, the last glob that match the +file is used. + +Now let's add some scripts for windows users. + +Complete example +"""""""""""""""" + +Source tree:: + + babar-1.3/ + README + doc/ + api + man + scripts/ + babar.sh + launch.sh + babar.bat + launch.bat + LAUNCH + +:file:`setup.cfg`:: + + [files] + resources = + README = {doc} + scripts/ LAUNCH = {doc} + scripts/ *.{sh,bat} = {scripts} + doc/ * = {doc} + doc/ man = {man} + +We use brace expansion syntax to place all the shell and batch scripts into +{scripts} category. + + +Command sections +================ + +To pass options to commands without having to type them on the command line +for each invocation, you can write them in the :file:`setup.cfg` file, in a +section named after the command. Example:: + + [sdist] + # special function to add custom files + manifest-builders = package.setup.list_extra_files + + [build] + use-2to3 = True + + [build_ext] + inplace = on + + [check] + strict = on + all = on + +Option values given in the configuration file can be overriden on the command +line. See :ref:`packaging-setup-config` for more information. diff --git a/Doc/packaging/setupscript.rst b/Doc/packaging/setupscript.rst new file mode 100644 index 0000000..3a1a98b --- /dev/null +++ b/Doc/packaging/setupscript.rst @@ -0,0 +1,689 @@ +.. _packaging-setup-script: + +************************ +Writing the Setup Script +************************ + +The setup script is the center of all activity in building, distributing, and +installing modules using Distutils. The main purpose of the setup script is +to describe your module distribution to Distutils, so that the various +commands that operate on your modules do the right thing. As we saw in section +:ref:`packaging-simple-example`, the setup script consists mainly of a +call to :func:`setup` where the most information is supplied as +keyword arguments to :func:`setup`. + +Here's a slightly more involved example, which we'll follow for the next couple +of sections: a setup script that could be used for Packaging itself:: + + #!/usr/bin/env python + + from packaging.core import setup, find_packages + + setup(name='Packaging', + version='1.0', + summary='Python Distribution Utilities', + keywords=['packaging', 'packaging'], + author=u'Tarek Ziadé', + author_email='tarek@ziade.org', + home_page='http://bitbucket.org/tarek/packaging/wiki/Home', + license='PSF', + packages=find_packages()) + + +There are only two differences between this and the trivial one-file +distribution presented in section :ref:`packaging-simple-example`: more +metadata and the specification of pure Python modules by package rather than +by module. This is important since Ristutils consist of a couple of dozen +modules split into (so far) two packages; an explicit list of every module +would be tedious to generate and difficult to maintain. For more information +on the additional metadata, see section :ref:`packaging-metadata`. + +Note that any pathnames (files or directories) supplied in the setup script +should be written using the Unix convention, i.e. slash-separated. The +Distutils will take care of converting this platform-neutral representation into +whatever is appropriate on your current platform before actually using the +pathname. This makes your setup script portable across operating systems, which +of course is one of the major goals of the Distutils. In this spirit, all +pathnames in this document are slash-separated. + +This, of course, only applies to pathnames given to Distutils functions. If +you, for example, use standard Python functions such as :func:`glob.glob` or +:func:`os.listdir` to specify files, you should be careful to write portable +code instead of hardcoding path separators:: + + glob.glob(os.path.join('mydir', 'subdir', '*.html')) + os.listdir(os.path.join('mydir', 'subdir')) + + +.. _packaging-listing-packages: + +Listing whole packages +====================== + +The :option:`packages` option tells the Distutils to process (build, distribute, +install, etc.) all pure Python modules found in each package mentioned in the +:option:`packages` list. In order to do this, of course, there has to be a +correspondence between package names and directories in the filesystem. The +default correspondence is the most obvious one, i.e. package :mod:`packaging` is +found in the directory :file:`packaging` relative to the distribution root. +Thus, when you say ``packages = ['foo']`` in your setup script, you are +promising that the Distutils will find a file :file:`foo/__init__.py` (which +might be spelled differently on your system, but you get the idea) relative to +the directory where your setup script lives. If you break this promise, the +Distutils will issue a warning but still process the broken package anyways. + +If you use a different convention to lay out your source directory, that's no +problem: you just have to supply the :option:`package_dir` option to tell the +Distutils about your convention. For example, say you keep all Python source +under :file:`lib`, so that modules in the "root package" (i.e., not in any +package at all) are in :file:`lib`, modules in the :mod:`foo` package are in +:file:`lib/foo`, and so forth. Then you would put :: + + package_dir = {'': 'lib'} + +in your setup script. The keys to this dictionary are package names, and an +empty package name stands for the root package. The values are directory names +relative to your distribution root. In this case, when you say ``packages = +['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists. + +Another possible convention is to put the :mod:`foo` package right in +:file:`lib`, the :mod:`foo.bar` package in :file:`lib/bar`, etc. This would be +written in the setup script as :: + + package_dir = {'foo': 'lib'} + +A ``package: dir`` entry in the :option:`package_dir` dictionary implicitly +applies to all packages below *package*, so the :mod:`foo.bar` case is +automatically handled here. In this example, having ``packages = ['foo', +'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and +:file:`lib/bar/__init__.py`. (Keep in mind that although :option:`package_dir` +applies recursively, you must explicitly list all packages in +:option:`packages`: the Distutils will *not* recursively scan your source tree +looking for any directory with an :file:`__init__.py` file.) + + +.. _packaging-listing-modules: + +Listing individual modules +========================== + +For a small module distribution, you might prefer to list all modules rather +than listing packages---especially the case of a single module that goes in the +"root package" (i.e., no package at all). This simplest case was shown in +section :ref:`packaging-simple-example`; here is a slightly more involved +example:: + + py_modules = ['mod1', 'pkg.mod2'] + +This describes two modules, one of them in the "root" package, the other in the +:mod:`pkg` package. Again, the default package/directory layout implies that +these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and +that :file:`pkg/__init__.py` exists as well. And again, you can override the +package/directory correspondence using the :option:`package_dir` option. + + +.. _packaging-describing-extensions: + +Describing extension modules +============================ + +Just as writing Python extension modules is a bit more complicated than writing +pure Python modules, describing them to the Distutils is a bit more complicated. +Unlike pure modules, it's not enough just to list modules or packages and expect +the Distutils to go out and find the right files; you have to specify the +extension name, source file(s), and any compile/link requirements (include +directories, libraries to link with, etc.). + +.. XXX read over this section + +All of this is done through another keyword argument to :func:`setup`, the +:option:`ext_modules` option. :option:`ext_modules` is just a list of +:class:`Extension` instances, each of which describes a single extension module. +Suppose your distribution includes a single extension, called :mod:`foo` and +implemented by :file:`foo.c`. If no additional instructions to the +compiler/linker are needed, describing this extension is quite simple:: + + Extension('foo', ['foo.c']) + +The :class:`Extension` class can be imported from :mod:`packaging.core` along +with :func:`setup`. Thus, the setup script for a module distribution that +contains only this one extension and nothing else might be:: + + from packaging.core import setup, Extension + setup(name='foo', + version='1.0', + ext_modules=[Extension('foo', ['foo.c'])]) + +The :class:`Extension` class (actually, the underlying extension-building +machinery implemented by the :command:`build_ext` command) supports a great deal +of flexibility in describing Python extensions, which is explained in the +following sections. + + +Extension names and packages +---------------------------- + +The first argument to the :class:`Extension` constructor is always the name of +the extension, including any package names. For example, :: + + Extension('foo', ['src/foo1.c', 'src/foo2.c']) + +describes an extension that lives in the root package, while :: + + Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c']) + +describes the same extension in the :mod:`pkg` package. The source files and +resulting object code are identical in both cases; the only difference is where +in the filesystem (and therefore where in Python's namespace hierarchy) the +resulting extension lives. + +If you have a number of extensions all in the same package (or all under the +same base package), use the :option:`ext_package` keyword argument to +:func:`setup`. For example, :: + + setup(..., + ext_package='pkg', + ext_modules=[Extension('foo', ['foo.c']), + Extension('subpkg.bar', ['bar.c'])]) + +will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to +:mod:`pkg.subpkg.bar`. + + +Extension source files +---------------------- + +The second argument to the :class:`Extension` constructor is a list of source +files. Since the Distutils currently only support C, C++, and Objective-C +extensions, these are normally C/C++/Objective-C source files. (Be sure to use +appropriate extensions to distinguish C++\ source files: :file:`.cc` and +:file:`.cpp` seem to be recognized by both Unix and Windows compilers.) + +However, you can also include SWIG interface (:file:`.i`) files in the list; the +:command:`build_ext` command knows how to deal with SWIG extensions: it will run +SWIG on the interface file and compile the resulting C/C++ file into your +extension. + +.. XXX SWIG support is rough around the edges and largely untested! + +This warning notwithstanding, options to SWIG can be currently passed like +this:: + + setup(..., + ext_modules=[Extension('_foo', ['foo.i'], + swig_opts=['-modern', '-I../include'])], + py_modules=['foo']) + +Or on the command line like this:: + + > python setup.py build_ext --swig-opts="-modern -I../include" + +On some platforms, you can include non-source files that are processed by the +compiler and included in your extension. Currently, this just means Windows +message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for +Visual C++. These will be compiled to binary resource (:file:`.res`) files and +linked into the executable. + + +Preprocessor options +-------------------- + +Three optional arguments to :class:`Extension` will help if you need to specify +include directories to search or preprocessor macros to define/undefine: +``include_dirs``, ``define_macros``, and ``undef_macros``. + +For example, if your extension requires header files in the :file:`include` +directory under your distribution root, use the ``include_dirs`` option:: + + Extension('foo', ['foo.c'], include_dirs=['include']) + +You can specify absolute directories there; if you know that your extension will +only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get +away with :: + + Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11']) + +You should avoid this sort of non-portable usage if you plan to distribute your +code: it's probably better to write C code like :: + + #include + +If you need to include header files from some other Python extension, you can +take advantage of the fact that header files are installed in a consistent way +by the Distutils :command:`install_header` command. For example, the Numerical +Python header files are installed (on a standard Unix installation) to +:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ +according to your platform and Python installation.) Since the Python include +directory---\ :file:`/usr/local/include/python1.5` in this case---is always +included in the search path when building Python extensions, the best approach +is to write C code like :: + + #include + +.. TODO check if it's d2.sysconfig or the new sysconfig module now + +If you must put the :file:`Numerical` include directory right into your header +search path, though, you can find that directory using the Distutils +:mod:`packaging.sysconfig` module:: + + from packaging.sysconfig import get_python_inc + incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical') + setup(..., + Extension(..., include_dirs=[incdir])) + +Even though this is quite portable---it will work on any Python installation, +regardless of platform---it's probably easier to just write your C code in the +sensible way. + +You can define and undefine preprocessor macros with the ``define_macros`` and +``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)`` +tuples, where ``name`` is the name of the macro to define (a string) and +``value`` is its value: either a string or ``None``. (Defining a macro ``FOO`` +to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with +most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is +just a list of macros to undefine. + +For example:: + + Extension(..., + define_macros=[('NDEBUG', '1'), + ('HAVE_STRFTIME', None)], + undef_macros=['HAVE_FOO', 'HAVE_BAR']) + +is the equivalent of having this at the top of every C source file:: + + #define NDEBUG 1 + #define HAVE_STRFTIME + #undef HAVE_FOO + #undef HAVE_BAR + + +Library options +--------------- + +You can also specify the libraries to link against when building your extension, +and the directories to search for those libraries. The ``libraries`` option is +a list of libraries to link against, ``library_dirs`` is a list of directories +to search for libraries at link-time, and ``runtime_library_dirs`` is a list of +directories to search for shared (dynamically loaded) libraries at run-time. + +For example, if you need to link against libraries known to be in the standard +library search path on target systems :: + + Extension(..., + libraries=['gdbm', 'readline']) + +If you need to link with libraries in a non-standard location, you'll have to +include the location in ``library_dirs``:: + + Extension(..., + library_dirs=['/usr/X11R6/lib'], + libraries=['X11', 'Xt']) + +(Again, this sort of non-portable construct should be avoided if you intend to +distribute your code.) + +.. XXX Should mention clib libraries here or somewhere else! + + +Other options +------------- + +There are still some other options which can be used to handle special cases. + +The :option:`optional` option is a boolean; if it is true, +a build failure in the extension will not abort the build process, but +instead simply not install the failing extension. + +The :option:`extra_objects` option is a list of object files to be passed to the +linker. These files must not have extensions, as the default extension for the +compiler is used. + +:option:`extra_compile_args` and :option:`extra_link_args` can be used to +specify additional command-line options for the respective compiler and linker +command lines. + +:option:`export_symbols` is only useful on Windows. It can contain a list of +symbols (functions or variables) to be exported. This option is not needed when +building compiled extensions: Distutils will automatically add ``initmodule`` +to the list of exported symbols. + +The :option:`depends` option is a list of files that the extension depends on +(for example header files). The build command will call the compiler on the +sources to rebuild extension if any on this files has been modified since the +previous build. + +Relationships between Distributions and Packages +================================================ + +.. FIXME rewrite to update to PEP 345 (but without dist/release confusion) + +A distribution may relate to packages in three specific ways: + +#. It can require packages or modules. + +#. It can provide packages or modules. + +#. It can obsolete packages or modules. + +These relationships can be specified using keyword arguments to the +:func:`packaging.core.setup` function. + +Dependencies on other Python modules and packages can be specified by supplying +the *requires* keyword argument to :func:`setup`. The value must be a list of +strings. Each string specifies a package that is required, and optionally what +versions are sufficient. + +To specify that any version of a module or package is required, the string +should consist entirely of the module or package name. Examples include +``'mymodule'`` and ``'xml.parsers.expat'``. + +If specific versions are required, a sequence of qualifiers can be supplied in +parentheses. Each qualifier may consist of a comparison operator and a version +number. The accepted comparison operators are:: + + < > == + <= >= != + +These can be combined by using multiple qualifiers separated by commas (and +optional whitespace). In this case, all of the qualifiers must be matched; a +logical AND is used to combine the evaluations. + +Let's look at a bunch of examples: + ++-------------------------+----------------------------------------------+ +| Requires Expression | Explanation | ++=========================+==============================================+ +| ``==1.0`` | Only version ``1.0`` is compatible | ++-------------------------+----------------------------------------------+ +| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` | +| | is compatible, except ``1.5.1`` | ++-------------------------+----------------------------------------------+ + +Now that we can specify dependencies, we also need to be able to specify what we +provide that other distributions can require. This is done using the *provides* +keyword argument to :func:`setup`. The value for this keyword is a list of +strings, each of which names a Python module or package, and optionally +identifies the version. If the version is not specified, it is assumed to match +that of the distribution. + +Some examples: + ++---------------------+----------------------------------------------+ +| Provides Expression | Explanation | ++=====================+==============================================+ +| ``mypkg`` | Provide ``mypkg``, using the distribution | +| | version | ++---------------------+----------------------------------------------+ +| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of | +| | the distribution version | ++---------------------+----------------------------------------------+ + +A package can declare that it obsoletes other packages using the *obsoletes* +keyword argument. The value for this is similar to that of the *requires* +keyword: a list of strings giving module or package specifiers. Each specifier +consists of a module or package name optionally followed by one or more version +qualifiers. Version qualifiers are given in parentheses after the module or +package name. + +The versions identified by the qualifiers are those that are obsoleted by the +distribution being described. If no qualifiers are given, all versions of the +named module or package are understood to be obsoleted. + +.. _packaging-installing-scripts: + +Installing Scripts +================== + +So far we have been dealing with pure and non-pure Python modules, which are +usually not run by themselves but imported by scripts. + +Scripts are files containing Python source code, intended to be started from the +command line. Scripts don't require Distutils to do anything very complicated. +The only clever feature is that if the first line of the script starts with +``#!`` and contains the word "python", the Distutils will adjust the first line +to refer to the current interpreter location. By default, it is replaced with +the current interpreter location. The :option:`--executable` (or :option:`-e`) +option will allow the interpreter path to be explicitly overridden. + +The :option:`scripts` option simply is a list of files to be handled in this +way. From the PyXML setup script:: + + setup(..., + scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val']) + +All the scripts will also be added to the ``MANIFEST`` file if no template is +provided. See :ref:`packaging-manifest`. + +.. _packaging-installing-package-data: + +Installing Package Data +======================= + +Often, additional files need to be installed into a package. These files are +often data that's closely related to the package's implementation, or text files +containing documentation that might be of interest to programmers using the +package. These files are called :dfn:`package data`. + +Package data can be added to packages using the ``package_data`` keyword +argument to the :func:`setup` function. The value must be a mapping from +package name to a list of relative path names that should be copied into the +package. The paths are interpreted as relative to the directory containing the +package (information from the ``package_dir`` mapping is used if appropriate); +that is, the files are expected to be part of the package in the source +directories. They may contain glob patterns as well. + +The path names may contain directory portions; any necessary directories will be +created in the installation. + +For example, if a package should contain a subdirectory with several data files, +the files can be arranged like this in the source tree:: + + setup.py + src/ + mypkg/ + __init__.py + module.py + data/ + tables.dat + spoons.dat + forks.dat + +The corresponding call to :func:`setup` might be:: + + setup(..., + packages=['mypkg'], + package_dir={'mypkg': 'src/mypkg'}, + package_data={'mypkg': ['data/*.dat']}) + + +All the files that match ``package_data`` will be added to the ``MANIFEST`` +file if no template is provided. See :ref:`packaging-manifest`. + + +.. _packaging-additional-files: + +Installing Additional Files +=========================== + +The :option:`data_files` option can be used to specify additional files needed +by the module distribution: configuration files, message catalogs, data files, +anything which doesn't fit in the previous categories. + +:option:`data_files` specifies a sequence of (*directory*, *files*) pairs in the +following way:: + + setup(..., + data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), + ('config', ['cfg/data.cfg']), + ('/etc/init.d', ['init-script'])]) + +Note that you can specify the directory names where the data files will be +installed, but you cannot rename the data files themselves. + +Each (*directory*, *files*) pair in the sequence specifies the installation +directory and the files to install there. If *directory* is a relative path, it +is interpreted relative to the installation prefix (Python's ``sys.prefix`` for +pure-Python packages, ``sys.exec_prefix`` for packages that contain extension +modules). Each file name in *files* is interpreted relative to the +:file:`setup.py` script at the top of the package source distribution. No +directory information from *files* is used to determine the final location of +the installed file; only the name of the file is used. + +You can specify the :option:`data_files` options as a simple sequence of files +without specifying a target directory, but this is not recommended, and the +:command:`install_dist` command will print a warning in this case. To install data +files directly in the target directory, an empty string should be given as the +directory. + +All the files that match ``data_files`` will be added to the ``MANIFEST`` file +if no template is provided. See :ref:`packaging-manifest`. + + + +.. _packaging-metadata: + +Metadata reference +================== + +The setup script may include additional metadata beyond the name and version. +This table describes required and additional information: + +.. TODO synchronize with setupcfg; link to it (but don't remove it, it's a + useful summary) + ++----------------------+---------------------------+-----------------+--------+ +| Meta-Data | Description | Value | Notes | ++======================+===========================+=================+========+ +| ``name`` | name of the project | short string | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``version`` | version of this release | short string | (1)(2) | ++----------------------+---------------------------+-----------------+--------+ +| ``author`` | project author's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``author_email`` | email address of the | email address | \(3) | +| | project author | | | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer`` | project maintainer's name | short string | \(3) | ++----------------------+---------------------------+-----------------+--------+ +| ``maintainer_email`` | email address of the | email address | \(3) | +| | project maintainer | | | ++----------------------+---------------------------+-----------------+--------+ +| ``home_page`` | home page for the project | URL | \(1) | ++----------------------+---------------------------+-----------------+--------+ +| ``summary`` | short description of the | short string | | +| | project | | | ++----------------------+---------------------------+-----------------+--------+ +| ``description`` | longer description of the | long string | \(5) | +| | project | | | ++----------------------+---------------------------+-----------------+--------+ +| ``download_url`` | location where the | URL | | +| | project may be downloaded | | | ++----------------------+---------------------------+-----------------+--------+ +| ``classifiers`` | a list of classifiers | list of strings | \(4) | ++----------------------+---------------------------+-----------------+--------+ +| ``platforms`` | a list of platforms | list of strings | | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the release | short string | \(6) | ++----------------------+---------------------------+-----------------+--------+ + +Notes: + +(1) + These fields are required. + +(2) + It is recommended that versions take the form *major.minor[.patch[.sub]]*. + +(3) + Either the author or the maintainer must be identified. + +(4) + The list of classifiers is available from the `PyPI website + `_. See also :mod:`packaging.create`. + +(5) + The ``description`` field is used by PyPI when you are registering a + release, to build its PyPI page. + +(6) + The ``license`` field is a text indicating the license covering the + distribution where the license is not a selection from the "License" Trove + classifiers. See the ``Classifier`` field. Notice that + there's a ``licence`` distribution option which is deprecated but still + acts as an alias for ``license``. + +'short string' + A single line of text, not more than 200 characters. + +'long string' + Multiple lines of plain text in reStructuredText format (see + http://docutils.sf.net/). + +'list of strings' + See below. + +In Python 2.x, "string value" means a unicode object. If a byte string (str or +bytes) is given, it has to be valid ASCII. + +.. TODO move this section to the version document, keep a summary, add a link + +Encoding the version information is an art in itself. Python projects generally +adhere to the version format *major.minor[.patch][sub]*. The major number is 0 +for initial, experimental releases of software. It is incremented for releases +that represent major milestones in a project. The minor number is incremented +when important new features are added to the project. The patch number +increments when bug-fix releases are made. Additional trailing version +information is sometimes used to indicate sub-releases. These are +"a1,a2,...,aN" (for alpha releases, where functionality and API may change), +"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN" +(for final pre-release release testing). Some examples: + +0.1.0 + the first, experimental release of a project + +1.0.1a2 + the second alpha release of the first patch version of 1.0 + +:option:`classifiers` are specified in a Python list:: + + setup(..., + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Environment :: Web Environment', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Python Software Foundation License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Topic :: Communications :: Email', + 'Topic :: Office/Business', + 'Topic :: Software Development :: Bug Tracking', + ]) + + +Debugging the setup script +========================== + +Sometimes things go wrong, and the setup script doesn't do what the developer +wants. + +Distutils catches any exceptions when running the setup script, and print a +simple error message before the script is terminated. The motivation for this +behaviour is to not confuse administrators who don't know much about Python and +are trying to install a project. If they get a big long traceback from deep +inside the guts of Distutils, they may think the project or the Python +installation is broken because they don't read all the way down to the bottom +and see that it's a permission problem. + +.. FIXME DISTUTILS_DEBUG is dead, document logging/warnings here + +On the other hand, this doesn't help the developer to find the cause of the +failure. For this purpose, the DISTUTILS_DEBUG environment variable can be set +to anything except an empty string, and Packaging will now print detailed +information about what it is doing, and prints the full traceback in case an +exception occurs. diff --git a/Doc/packaging/sourcedist.rst b/Doc/packaging/sourcedist.rst new file mode 100644 index 0000000..0cd4df3 --- /dev/null +++ b/Doc/packaging/sourcedist.rst @@ -0,0 +1,273 @@ +.. _packaging-source-dist: + +****************************** +Creating a Source Distribution +****************************** + +As shown in section :ref:`packaging-simple-example`, you use the :command:`sdist` command +to create a source distribution. In the simplest case, :: + + python setup.py sdist + +(assuming you haven't specified any :command:`sdist` options in the setup script +or config file), :command:`sdist` creates the archive of the default format for +the current platform. The default format is a gzip'ed tar file +(:file:`.tar.gz`) on Unix, and ZIP file on Windows. + +You can specify as many formats as you like using the :option:`--formats` +option, for example:: + + python setup.py sdist --formats=gztar,zip + +to create a gzipped tarball and a zip file. The available formats are: + ++-----------+-------------------------+---------+ +| Format | Description | Notes | ++===========+=========================+=========+ +| ``zip`` | zip file (:file:`.zip`) | (1),(3) | ++-----------+-------------------------+---------+ +| ``gztar`` | gzip'ed tar file | \(2) | +| | (:file:`.tar.gz`) | | ++-----------+-------------------------+---------+ +| ``bztar`` | bzip2'ed tar file | | +| | (:file:`.tar.bz2`) | | ++-----------+-------------------------+---------+ +| ``ztar`` | compressed tar file | \(4) | +| | (:file:`.tar.Z`) | | ++-----------+-------------------------+---------+ +| ``tar`` | tar file (:file:`.tar`) | | ++-----------+-------------------------+---------+ + +Notes: + +(1) + default on Windows + +(2) + default on Unix + +(3) + requires either external :program:`zip` utility or :mod:`zipfile` module (part + of the standard Python library since Python 1.6) + +(4) + requires the :program:`compress` program. Notice that this format is now + pending for deprecation and will be removed in the future versions of Python. + +When using any ``tar`` format (``gztar``, ``bztar``, ``ztar`` or +``tar``) under Unix, you can specify the ``owner`` and ``group`` names +that will be set for each member of the archive. + +For example, if you want all files of the archive to be owned by root:: + + python setup.py sdist --owner=root --group=root + + +.. _packaging-manifest: + +Specifying the files to distribute +================================== + +If you don't supply an explicit list of files (or instructions on how to +generate one), the :command:`sdist` command puts a minimal default set into the +source distribution: + +* all Python source files implied by the :option:`py_modules` and + :option:`packages` options + +* all C source files mentioned in the :option:`ext_modules` or + :option:`libraries` options + +* scripts identified by the :option:`scripts` option + See :ref:`packaging-installing-scripts`. + +* anything that looks like a test script: :file:`test/test\*.py` (currently, the + Packaging don't do anything with test scripts except include them in source + distributions, but in the future there will be a standard for testing Python + module distributions) + +* the configuration file :file:`setup.cfg` + +* all files that matches the ``package_data`` metadata. + See :ref:`packaging-installing-package-data`. + +* all files that matches the ``data_files`` metadata. + See :ref:`packaging-additional-files`. + +Contrary to Distutils, :file:`README` (or :file:`README.txt`) and +:file:`setup.py` are not included by default. + +Sometimes this is enough, but usually you will want to specify additional files +to distribute. The typical way to do this is to write a *manifest template*, +called :file:`MANIFEST.in` by default. The manifest template is just a list of +instructions for how to generate your manifest file, :file:`MANIFEST`, which is +the exact list of files to include in your source distribution. The +:command:`sdist` command processes this template and generates a manifest based +on its instructions and what it finds in the filesystem. + +If you prefer to roll your own manifest file, the format is simple: one filename +per line, regular files (or symlinks to them) only. If you do supply your own +:file:`MANIFEST`, you must specify everything: the default set of files +described above does not apply in this case. + +:file:`MANIFEST` files start with a comment indicating they are generated. +Files without this comment are not overwritten or removed. + +See :ref:`packaging-manifest-template` section for a syntax reference. + + +.. _packaging-manifest-options: + +Manifest-related options +======================== + +The normal course of operations for the :command:`sdist` command is as follows: + +* if the manifest file, :file:`MANIFEST` doesn't exist, read :file:`MANIFEST.in` + and create the manifest + +* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest + with just the default file set + +* if either :file:`MANIFEST.in` or the setup script (:file:`setup.py`) are more + recent than :file:`MANIFEST`, recreate :file:`MANIFEST` by reading + :file:`MANIFEST.in` + +* use the list of files now in :file:`MANIFEST` (either just generated or read + in) to create the source distribution archive(s) + +There are a couple of options that modify this behaviour. First, use the +:option:`--no-defaults` and :option:`--no-prune` to disable the standard +"include" and "exclude" sets. + +Second, you might just want to (re)generate the manifest, but not create a +source distribution:: + + python setup.py sdist --manifest-only + +:option:`-o` is a shortcut for :option:`--manifest-only`. + + +.. _packaging-manifest-template: + +The MANIFEST.in template +======================== + +A :file:`MANIFEST.in` file can be added in a project to define the list of +files to include in the distribution built by the :command:`sdist` command. + +When :command:`sdist` is run, it will look for the :file:`MANIFEST.in` file +and interpret it to generate the :file:`MANIFEST` file that contains the +list of files that will be included in the package. + +This mechanism can be used when the default list of files is not enough. +(See :ref:`packaging-manifest`). + +Principle +--------- + +The manifest template has one command per line, where each command specifies a +set of files to include or exclude from the source distribution. For an +example, let's look at the Packaging' own manifest template:: + + include *.txt + recursive-include examples *.txt *.py + prune examples/sample?/build + +The meanings should be fairly clear: include all files in the distribution root +matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory +matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching +:file:`examples/sample?/build`. All of this is done *after* the standard +include set, so you can exclude files from the standard set with explicit +instructions in the manifest template. (Or, you can use the +:option:`--no-defaults` option to disable the standard set entirely.) + +The order of commands in the manifest template matters: initially, we have the +list of default files as described above, and each command in the template adds +to or removes from that list of files. Once we have fully processed the +manifest template, we remove files that should not be included in the source +distribution: + +* all files in the Packaging "build" tree (default :file:`build/`) + +* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`, + :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs` + +Now we have our complete list of files, which is written to the manifest for +future reference, and then used to build the source distribution archive(s). + +You can disable the default set of included files with the +:option:`--no-defaults` option, and you can disable the standard exclude set +with :option:`--no-prune`. + +Following the Packaging' own manifest template, let's trace how the +:command:`sdist` command builds the list of files to include in the Packaging +source distribution: + +#. include all Python source files in the :file:`packaging` and + :file:`packaging/command` subdirectories (because packages corresponding to + those two directories were mentioned in the :option:`packages` option in the + setup script---see section :ref:`packaging-setup-script`) + +#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard + files) + +#. include :file:`test/test\*.py` (standard files) + +#. include :file:`\*.txt` in the distribution root (this will find + :file:`README.txt` a second time, but such redundancies are weeded out later) + +#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree + under :file:`examples`, + +#. exclude all files in the sub-trees starting at directories matching + :file:`examples/sample?/build`\ ---this may exclude files included by the + previous two steps, so it's important that the ``prune`` command in the manifest + template comes after the ``recursive-include`` command + +#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`, + :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs` + directories + +Just like in the setup script, file and directory names in the manifest template +should always be slash-separated; the Packaging will take care of converting +them to the standard representation on your platform. That way, the manifest +template is portable across operating systems. + +Commands +-------- + +The manifest template commands are: + ++-------------------------------------------+-----------------------------------------------+ +| Command | Description | ++===========================================+===============================================+ +| :command:`include pat1 pat2 ...` | include all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed | +| | patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of | +| ...` | the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree | +| | matching --- & any of the listed patterns | ++-------------------------------------------+-----------------------------------------------+ +| :command:`prune dir` | exclude all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ +| :command:`graft dir` | include all files under *dir* | ++-------------------------------------------+-----------------------------------------------+ + +The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of +regular filename characters, ``?`` matches any single regular filename +character, and ``[range]`` matches any of the characters in *range* (e.g., +``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename +character" is platform-specific: on Unix it is anything except slash; on Windows +anything except backslash or colon. diff --git a/Doc/packaging/tutorial.rst b/Doc/packaging/tutorial.rst new file mode 100644 index 0000000..04f41e5 --- /dev/null +++ b/Doc/packaging/tutorial.rst @@ -0,0 +1,112 @@ +================== +Packaging tutorial +================== + +Welcome to the Packaging tutorial! We will learn how to use Packaging +to package your project. + +.. TODO merge with introduction.rst + + +Getting started +--------------- + +Packaging works with the *setup.cfg* file. It contains all the metadata for +your project, as defined in PEP 345, but also declare what your project +contains. + +Let's say you have a project called *CLVault* containing one package called +*clvault*, and a few scripts inside. You can use the *pysetup* script to create +a *setup.cfg* file for the project. The script will ask you a few questions:: + + $ mkdir CLVault + $ cd CLVault + $ pysetup create + Project name [CLVault]: + Current version number: 0.1 + Package description: + >Command-line utility to store and retrieve passwords + Author name: Tarek Ziade + Author e-mail address: tarek@ziade.org + Project Home Page: http://bitbucket.org/tarek/clvault + Do you want to add a package ? (y/n): y + Package name: clvault + Do you want to add a package ? (y/n): n + Do you want to set Trove classifiers? (y/n): y + Please select the project status: + + 1 - Planning + 2 - Pre-Alpha + 3 - Alpha + 4 - Beta + 5 - Production/Stable + 6 - Mature + 7 - Inactive + + Status: 3 + What license do you use: GPL + Matching licenses: + + 1) License :: OSI Approved :: GNU General Public License (GPL) + 2) License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) + + Type the number of the license you wish to use or ? to try again:: 1 + Do you want to set other trove identifiers (y/n) [n]: n + Wrote "setup.cfg". + + +A setup.cfg file is created, containing the metadata of your project and the +list of the packages it contains:: + + $ cat setup.cfg + [metadata] + name = CLVault + version = 0.1 + author = Tarek Ziade + author_email = tarek@ziade.org + description = Command-line utility to store and retrieve passwords + home_page = http://bitbucket.org/tarek/clvault + + classifier = Development Status :: 3 - Alpha + License :: OSI Approved :: GNU General Public License (GPL) + + [files] + packages = clvault + + +Our project will depend on the *keyring* project. Let's add it in the +[metadata] section:: + + [metadata] + ... + requires_dist = + keyring + + +Running commands +---------------- + +You can run useful commands on your project once the setup.cfg file is ready: + +- sdist: creates a source distribution +- register: register your project to PyPI +- upload: upload the distribution to PyPI +- install_dist: install it + +All commands are run using the run script:: + + $ pysetup run install_dist + $ pysetup run sdist + $ pysetup run upload + +If you want to push a source distribution of your project to PyPI, do:: + + $ pysetup run sdist register upload + + +Installing the project +---------------------- + +The project can be installed by manually running the packaging install command:: + + $ pysetup run install_dist diff --git a/Doc/packaging/uploading.rst b/Doc/packaging/uploading.rst new file mode 100644 index 0000000..297518b --- /dev/null +++ b/Doc/packaging/uploading.rst @@ -0,0 +1,80 @@ +.. _packaging-package-upload: + +*************************************** +Uploading Packages to the Package Index +*************************************** + +The Python Package Index (PyPI) not only stores the package info, but also the +package data if the author of the package wishes to. The packaging command +:command:`upload` pushes the distribution files to PyPI. + +The command is invoked immediately after building one or more distribution +files. For example, the command :: + + python setup.py sdist bdist_wininst upload + +will cause the source distribution and the Windows installer to be uploaded to +PyPI. Note that these will be uploaded even if they are built using an earlier +invocation of :file:`setup.py`, but that only distributions named on the command +line for the invocation including the :command:`upload` command are uploaded. + +The :command:`upload` command uses the username, password, and repository URL +from the :file:`$HOME/.pypirc` file (see section :ref:`packaging-pypirc` for more on this +file). If a :command:`register` command was previously called in the same +command, and if the password was entered in the prompt, :command:`upload` will +reuse the entered password. This is useful if you do not want to store a clear +text password in the :file:`$HOME/.pypirc` file. + +You can specify another PyPI server with the :option:`--repository=*url*` +option:: + + python setup.py sdist bdist_wininst upload -r http://example.com/pypi + +See section :ref:`packaging-pypirc` for more on defining several servers. + +You can use the :option:`--sign` option to tell :command:`upload` to sign each +uploaded file using GPG (GNU Privacy Guard). The :program:`gpg` program must +be available for execution on the system :envvar:`PATH`. You can also specify +which key to use for signing using the :option:`--identity=*name*` option. + +Other :command:`upload` options include :option:`--repository=` or +:option:`--repository=
` where *url* is the url of the server and +*section* the name of the section in :file:`$HOME/.pypirc`, and +:option:`--show-response` (which displays the full response text from the PyPI +server for help in debugging upload problems). + +PyPI package display +==================== + +The ``description`` field plays a special role at PyPI. It is used by +the server to display a home page for the registered package. + +If you use the `reStructuredText `_ +syntax for this field, PyPI will parse it and display an HTML output for +the package home page. + +The ``description`` field can be filled from a text file located in the +project:: + + from packaging.core import setup + + fp = open('README.txt') + try: + description = fp.read() + finally: + fp.close() + + setup(name='Packaging', + description=description) + +In that case, :file:`README.txt` is a regular reStructuredText text file located +in the root of the package besides :file:`setup.py`. + +To prevent registering broken reStructuredText content, you can use the +:program:`rst2html` program that is provided by the :mod:`docutils` package +and check the ``description`` from the command line:: + + $ python setup.py --description | rst2html.py > output.html + +:mod:`docutils` will display a warning if there's something wrong with your +syntax. diff --git a/Doc/tools/sphinxext/indexcontent.html b/Doc/tools/sphinxext/indexcontent.html index d5e17cd..10f9d26 100644 --- a/Doc/tools/sphinxext/indexcontent.html +++ b/Doc/tools/sphinxext/indexcontent.html @@ -20,9 +20,9 @@ tutorial for C/C++ programmers

- - -- cgit v0.12 From 5da37be7f25f07a48229dd1d409447c3ffbf3952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Wed, 1 Jun 2011 20:44:40 +0200 Subject: Kill trailing whitespace --- Doc/install/pysetup-config.rst | 4 ++-- Doc/install/pysetup-servers.rst | 2 +- Doc/packaging/introduction.rst | 52 ++++++++++++++++++++--------------------- Doc/packaging/setupscript.rst | 2 +- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Doc/install/pysetup-config.rst b/Doc/install/pysetup-config.rst index 0ce9022..a473bfe 100644 --- a/Doc/install/pysetup-config.rst +++ b/Doc/install/pysetup-config.rst @@ -11,8 +11,8 @@ Pysetup supports two configuration files: :file:`.pypirc` and :file:`packaging.c Configuring indexes ------------------- -You can configure additional indexes in :file:`.pypirc` to be used for index-related -operations. By default, all configured index-servers and package-servers will be used +You can configure additional indexes in :file:`.pypirc` to be used for index-related +operations. By default, all configured index-servers and package-servers will be used in an additive fashion. To limit operations to specific indexes, use the :option:`--index` and :option:`--package-server options`:: diff --git a/Doc/install/pysetup-servers.rst b/Doc/install/pysetup-servers.rst index ddaaa5b..c6106de 100644 --- a/Doc/install/pysetup-servers.rst +++ b/Doc/install/pysetup-servers.rst @@ -8,7 +8,7 @@ Pysetup supports installing Python packages from *Package Servers* in addition to PyPI indexes and mirrors. Package Servers are simple directory listings of Python distributions. Directories -can be served via HTTP or a local file system. This is useful when you want to +can be served via HTTP or a local file system. This is useful when you want to dump source distributions in a directory and not worry about the full index structure. Serving distributions from Apache diff --git a/Doc/packaging/introduction.rst b/Doc/packaging/introduction.rst index db0ffbb..a757ffc 100644 --- a/Doc/packaging/introduction.rst +++ b/Doc/packaging/introduction.rst @@ -33,13 +33,13 @@ code, of course!) are: All of these tasks are covered in this document. -Not all module developers have access to multiple platforms, so one cannot +Not all module developers have access to multiple platforms, so one cannot expect them to create buildt distributions for every platform. To remedy this, it is hoped that intermediaries called *packagers* will arise to address this need. Packagers take source distributions released by module developers, -build them on one or more platforms and release the resulting built -distributions. Thus, users on a greater range of platforms will be able to -install the most popular Python modules in the most natural way for their +build them on one or more platforms and release the resulting built +distributions. Thus, users on a greater range of platforms will be able to +install the most popular Python modules in the most natural way for their platform without having to run a setup script or compile a single line of code. @@ -69,14 +69,14 @@ Some observations: arguments to the :func:`setup` function * those keyword arguments fall into two categories: package metadata (name, - version number, etc.) and information about what's in the package (a list + version number, etc.) and information about what's in the package (a list of pure Python modules in this case) * modules are specified by module name, not filename (the same will hold true for packages and extensions) -* it's recommended that you supply a little more metadata than we have in the - example. In particular your name, email address and a URL for the +* it's recommended that you supply a little more metadata than we have in the + example. In particular your name, email address and a URL for the project if appropriate (see section :ref:`packaging-setup-script` for an example) To create a source distribution for this module you would create a setup @@ -102,10 +102,10 @@ This simple example demonstrates some fundamental concepts of Distutils. First, both developers and installers have the same basic user interface, i.e. the setup script. The difference is which Distutils *commands* they use: the :command:`sdist` command is almost exclusively for module developers, while -:command:`install` is more often used by installers (although some developers +:command:`install` is more often used by installers (although some developers will want to install their own code occasionally). -If you want to make things really easy for your users, you can create more +If you want to make things really easy for your users, you can create more than one built distributions for them. For instance, if you are running on a Windows machine and want to make things easy for other Windows users, you can create an executable installer (the most appropriate type of built distribution @@ -125,18 +125,18 @@ by running :: General Python terminology ========================== -If you're reading this document, you probably have a good idea of what Python -modules, extensions and so forth are. Nevertheless, just to be sure that +If you're reading this document, you probably have a good idea of what Python +modules, extensions and so forth are. Nevertheless, just to be sure that everyone is on the same page, here's a quick overview of Python terms: module - The basic unit of code reusability in Python: a block of code imported by - some other code. Three types of modules are important to us here: pure + The basic unit of code reusability in Python: a block of code imported by + some other code. Three types of modules are important to us here: pure Python modules, extension modules and packages. pure Python module A module written in Python and contained in a single :file:`.py` file (and - possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes + possibly associated :file:`.pyc` and/or :file:`.pyo` files). Sometimes referred to as a "pure module." extension module @@ -148,18 +148,18 @@ extension module currently Distutils only handles C/C++ extensions for Python. package - A module that contains other modules, typically contained in a directory of - the filesystem and distinguished from other directories by the presence of a + A module that contains other modules, typically contained in a directory of + the filesystem and distinguished from other directories by the presence of a file :file:`__init__.py`. root package - The root of the hierarchy of packages. (This isn't really a package, - since it doesn't have an :file:`__init__.py` file. But... we have to - call it something, right?) The vast majority of the standard library is - in the root package, as are many small standalone third-party modules that - don't belong to a larger module collection. Unlike regular packages, - modules in the root package can be found in many directories: in fact, - every directory listed in ``sys.path`` contributes modules to the root + The root of the hierarchy of packages. (This isn't really a package, + since it doesn't have an :file:`__init__.py` file. But... we have to + call it something, right?) The vast majority of the standard library is + in the root package, as are many small standalone third-party modules that + don't belong to a larger module collection. Unlike regular packages, + modules in the root package can be found in many directories: in fact, + every directory listed in ``sys.path`` contributes modules to the root package. @@ -175,8 +175,8 @@ module distribution A collection of Python modules distributed together as a single downloadable resource and meant to be installed all as one. Examples of some well-known module distributions are NumPy, SciPy, PIL (the Python Imaging - Library) or mxBase. (Module distributions would be called a *package*, - except that term is already taken in the Python context: a single module + Library) or mxBase. (Module distributions would be called a *package*, + except that term is already taken in the Python context: a single module distribution may contain zero, one, or many Python packages.) pure module distribution @@ -189,5 +189,5 @@ non-pure module distribution distribution root The top-level directory of your source tree (or source distribution). The - directory where :file:`setup.py` exists. Generally :file:`setup.py` will + directory where :file:`setup.py` exists. Generally :file:`setup.py` will be run from this directory. diff --git a/Doc/packaging/setupscript.rst b/Doc/packaging/setupscript.rst index 3a1a98b..5f302a8 100644 --- a/Doc/packaging/setupscript.rst +++ b/Doc/packaging/setupscript.rst @@ -9,7 +9,7 @@ installing modules using Distutils. The main purpose of the setup script is to describe your module distribution to Distutils, so that the various commands that operate on your modules do the right thing. As we saw in section :ref:`packaging-simple-example`, the setup script consists mainly of a -call to :func:`setup` where the most information is supplied as +call to :func:`setup` where the most information is supplied as keyword arguments to :func:`setup`. Here's a slightly more involved example, which we'll follow for the next couple -- cgit v0.12 From 9028928156b187eb0d684233960c568b46f11088 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 1 Jun 2011 16:17:23 -0700 Subject: Forward port doc updates for builtin functions. --- Doc/library/functions.rst | 54 ++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 134c19c..1c1c167 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -290,19 +290,18 @@ are always available. They are listed here in alphabetical order. The resulting list is sorted alphabetically. For example: >>> import struct - >>> dir() # doctest: +SKIP + >>> dir() # show the names in the module namespace ['__builtins__', '__doc__', '__name__', 'struct'] - >>> dir(struct) # doctest: +NORMALIZE_WHITESPACE + >>> dir(struct) # show the names in the struct module ['Struct', '__builtins__', '__doc__', '__file__', '__name__', '__package__', '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', 'unpack', 'unpack_from'] - >>> class Foo: - ... def __dir__(self): - ... return ["kan", "ga", "roo"] - ... - >>> f = Foo() - >>> dir(f) - ['ga', 'kan', 'roo'] + >>> class Shape(object): + def __dir__(self): + return ['area', 'perimeter', 'location'] + >>> s = Shape() + >>> dir(s) + ['area', 'perimeter', 'location'] .. note:: @@ -333,15 +332,21 @@ are always available. They are listed here in alphabetical order. :meth:`__next__` method of the iterator returned by :func:`enumerate` returns a tuple containing a count (from *start* which defaults to 0) and the corresponding value obtained from iterating over *iterable*. - :func:`enumerate` is useful for obtaining an indexed series: ``(0, seq[0])``, - ``(1, seq[1])``, ``(2, seq[2])``, .... For example: - >>> for i, season in enumerate(['Spring', 'Summer', 'Fall', 'Winter']): - ... print(i, season) - 0 Spring - 1 Summer - 2 Fall - 3 Winter + >>> for i, season in enumerate('Spring Summer Fall Winter'.split(), start=1): + print(i, season) + 1 Spring + 2 Summer + 3 Fall + 4 Winter + + Equivalent to:: + + def enumerate(sequence, start=0): + n = start + for elem in sequence: + yield n, elem + n += 1 .. function:: eval(expression, globals=None, locals=None) @@ -652,10 +657,10 @@ are always available. They are listed here in alphabetical order. One useful application of the second form of :func:`iter` is to read lines of a file until a certain line is reached. The following example reads a file - until ``"STOP"`` is reached: :: + until the :meth:`readline` method returns an empty string:: - with open("mydata.txt") as fp: - for line in iter(fp.readline, "STOP"): + with open('mydata.txt') as fp: + for line in iter(fp.readline, ''): process_line(line) @@ -1169,8 +1174,9 @@ are always available. They are listed here in alphabetical order. It can be called either on the class (such as ``C.f()``) or on an instance (such as ``C().f()``). The instance is ignored except for its class. - Static methods in Python are similar to those found in Java or C++. For a more - advanced concept, see :func:`classmethod` in this section. + Static methods in Python are similar to those found in Java or C++. Also see + :func:`classmethod` for a variant that is useful for creating alternate class + constructors. For more information on static methods, consult the documentation on the standard type hierarchy in :ref:`types`. @@ -1270,6 +1276,10 @@ are always available. They are listed here in alphabetical order. references. The zero argument form automatically searches the stack frame for the class (``__class__``) and the first argument. + For practical suggestions on how to design cooperative classes using + :func:`super`, see `guide to using super() + `_. + .. function:: tuple([iterable]) -- cgit v0.12 From ab244398ba03f260b8daeefe7bf93b2067a77a8e Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Thu, 2 Jun 2011 11:28:07 +0200 Subject: this documentation is now handled by all the new packaging contributors --- Doc/packaging/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/packaging/index.rst b/Doc/packaging/index.rst index 1b597d7..0e80818 100644 --- a/Doc/packaging/index.rst +++ b/Doc/packaging/index.rst @@ -4,7 +4,7 @@ Distributing Python Projects ############################## -:Authors: Greg Ward, Anthony Baxter and Packaging contributors +:Authors: The Fellowship of The Packaging :Email: distutils-sig@python.org :Release: |version| :Date: |today| -- cgit v0.12 From 721ccd0ce128234a3a4fb703f056154c4b9a9e36 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Thu, 2 Jun 2011 12:00:44 +0200 Subject: added the generate-setup action --- Lib/packaging/run.py | 18 +++++++++++++++++- Lib/packaging/util.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py index 2d22bfd..de9dd13 100644 --- a/Lib/packaging/run.py +++ b/Lib/packaging/run.py @@ -9,7 +9,7 @@ from copy import copy from packaging import logger from packaging.dist import Distribution -from packaging.util import _is_archive_file +from packaging.util import _is_archive_file, generate_setup_py from packaging.command import get_command_class, STANDARD_COMMANDS from packaging.install import install, install_local_project, remove from packaging.database import get_distribution, get_distributions @@ -38,6 +38,14 @@ Usage: pysetup create Create a new Python package. """ +generate_usage = """\ +Usage: pysetup generate-setup + or: pysetup generate-setup --help + +Generates a setup.py script for backward-compatibility purposes. +""" + + graph_usage = """\ Usage: pysetup graph dist or: pysetup graph --help @@ -204,6 +212,13 @@ def _create(distpatcher, args, **kw): return main() +@action_help(generate_usage) +def _generate(distpatcher, args, **kw): + generate_setup_py() + print('The setup.py was generated') + + + @action_help(graph_usage) def _graph(dispatcher, args, **kw): name = args[1] @@ -381,6 +396,7 @@ actions = [ ('list', 'Search for local projects', _list), ('graph', 'Display a graph', _graph), ('create', 'Create a Project', _create), + ('generate-setup', 'Generates a backward-comptatible setup.py', _generate) ] diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index e839320..4e5bd2c 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -1087,7 +1087,7 @@ def generate_setup_py(): Raises a PackagingFileError when a setup.py already exists. """ if os.path.exists("setup.py"): - raise PackagingFileError("a setup.py file alreadyexists") + raise PackagingFileError("a setup.py file already exists") with open("setup.py", "w", encoding='utf-8') as fp: fp.write(_SETUP_TMPL % {'func': getsource(cfg_to_args)}) -- cgit v0.12 From 78af7d8392b5671f6877b259df34c36b884dc103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Thu, 2 Jun 2011 14:53:59 +0200 Subject: Make packaging.tests.support.LoggingCatcher.get_logs flush the log handler. This removes the need to call flush manually in each test, except when testing code that creates warning without checking them. --- Lib/packaging/tests/support.py | 20 +++++++++++++------- Lib/packaging/tests/test_command_check.py | 11 ++++++----- Lib/packaging/tests/test_manifest.py | 3 --- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py index 66b5583..ae65a1d 100644 --- a/Lib/packaging/tests/support.py +++ b/Lib/packaging/tests/support.py @@ -90,17 +90,23 @@ class LoggingCatcher: def get_logs(self, *levels): """Return all log messages with level in *levels*. - Without explicit levels given, returns all messages. - *levels* defaults to all levels. For log calls with arguments (i.e. - logger.info('bla bla %s', arg)), the messages - Returns a list. + Without explicit levels given, returns all messages. *levels* defaults + to all levels. For log calls with arguments (i.e. + logger.info('bla bla %r', arg)), the messages will be formatted before + being returned (e.g. "bla bla 'thing'"). + + Returns a list. Automatically flushes the loghandler after being + called. Example: self.get_logs(logging.WARN, logging.DEBUG). """ if not levels: - return [log.getMessage() for log in self.loghandler.buffer] - return [log.getMessage() for log in self.loghandler.buffer - if log.levelno in levels] + messages = [log.getMessage() for log in self.loghandler.buffer] + else: + messages = [log.getMessage() for log in self.loghandler.buffer + if log.levelno in levels] + self.loghandler.flush() + return messages class TempdirManager: diff --git a/Lib/packaging/tests/test_command_check.py b/Lib/packaging/tests/test_command_check.py index 8b32673..0bdd616 100644 --- a/Lib/packaging/tests/test_command_check.py +++ b/Lib/packaging/tests/test_command_check.py @@ -36,7 +36,6 @@ class CheckTestCase(support.LoggingCatcher, # now let's add the required fields # and run it again, to make sure we don't get # any warning anymore - self.loghandler.flush() metadata = {'home_page': 'xxx', 'author': 'xxx', 'author_email': 'xxx', 'name': 'xxx', 'version': '4.2', @@ -50,8 +49,10 @@ class CheckTestCase(support.LoggingCatcher, self.assertRaises(PackagingSetupError, self._run, {'name': 'xxx', 'version': 'xxx'}, **{'strict': 1}) - # and of course, no error when all metadata fields are present + # clear warnings from the previous calls self.loghandler.flush() + + # and of course, no error when all metadata fields are present cmd = self._run(metadata, strict=True) self.assertEqual([], self.get_logs(logging.WARNING)) @@ -70,7 +71,6 @@ class CheckTestCase(support.LoggingCatcher, 'name': 'xxx', 'version': '4.2', 'requires_python': '2.4', } - self.loghandler.flush() cmd = self._run(metadata) self.assertEqual([], self.get_logs(logging.WARNING)) @@ -85,9 +85,11 @@ class CheckTestCase(support.LoggingCatcher, self.assertRaises(PackagingSetupError, self._run, metadata, **{'strict': 1}) + # clear warnings from the previous calls + self.loghandler.flush() + # now with correct version format again metadata['version'] = '4.2' - self.loghandler.flush() cmd = self._run(metadata, strict=True) self.assertEqual([], self.get_logs(logging.WARNING)) @@ -100,7 +102,6 @@ class CheckTestCase(support.LoggingCatcher, cmd.check_restructuredtext() self.assertEqual(len(self.get_logs(logging.WARNING)), 1) - self.loghandler.flush() pkg_info, dist = self.create_dist(description='title\n=====\n\ntest') cmd = check(dist) cmd.check_restructuredtext() diff --git a/Lib/packaging/tests/test_manifest.py b/Lib/packaging/tests/test_manifest.py index 9fb8b63..e0bcbbc 100644 --- a/Lib/packaging/tests/test_manifest.py +++ b/Lib/packaging/tests/test_manifest.py @@ -50,9 +50,6 @@ class ManifestTestCase(support.TempdirManager, for warning in warnings: self.assertIn('no files found matching', warning) - # reset logs for the next assert - self.loghandler.flush() - # manifest also accepts file-like objects with open(MANIFEST) as f: manifest.read_template(f) -- cgit v0.12 From 8c86ecdab52528ed7457872c99252d0df94c7157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Thu, 2 Jun 2011 14:54:44 +0200 Subject: Fix format of warnings from the packaging check command --- Lib/packaging/command/check.py | 2 +- Lib/packaging/tests/test_command_check.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/packaging/command/check.py b/Lib/packaging/command/check.py index 94c4a97..6715db9 100644 --- a/Lib/packaging/command/check.py +++ b/Lib/packaging/command/check.py @@ -32,7 +32,7 @@ class check(Command): # XXX we could use a special handler for this, but would need to test # if it works even if the logger has a too high level self._warnings.append((msg, args)) - return logger.warning(self.get_command_name() + msg, *args) + return logger.warning('%s: %s' % (self.get_command_name(), msg), *args) def run(self): """Runs the command.""" diff --git a/Lib/packaging/tests/test_command_check.py b/Lib/packaging/tests/test_command_check.py index 0bdd616..271e457 100644 --- a/Lib/packaging/tests/test_command_check.py +++ b/Lib/packaging/tests/test_command_check.py @@ -124,6 +124,17 @@ class CheckTestCase(support.LoggingCatcher, cmd.check_hooks_resolvable() self.assertEqual(len(self.get_logs(logging.WARNING)), 1) + def test_warn(self): + _, dist = self.create_dist() + cmd = check(dist) + self.assertEqual([], self.get_logs()) + cmd.warn('hello') + self.assertEqual(['check: hello'], self.get_logs()) + cmd.warn('hello %s', 'world') + self.assertEqual(['check: hello world'], self.get_logs()) + cmd.warn('hello %s %s', 'beautiful', 'world') + self.assertEqual(['check: hello beautiful world'], self.get_logs()) + def test_suite(): return unittest.makeSuite(CheckTestCase) -- cgit v0.12 From 55729fe7189ab3d21c14cc6be6896ca4f0ebad65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Thu, 2 Jun 2011 15:45:25 +0200 Subject: Packaging doc: Add missing index file, improve main page description. Also promote notices from distutils doc to deprecation boxes. --- Doc/distutils/index.rst | 10 +++---- Doc/install/index.rst | 56 +++++++++++++++++++++++++++++++++++ Doc/library/distutils.rst | 15 ++++++---- Doc/tools/sphinxext/indexcontent.html | 4 +-- 4 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 Doc/install/index.rst diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index 5fa25a6..b3a5231 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -14,6 +14,10 @@ the module developer's point of view, describing how to use the Distutils to make Python modules and extensions easily available to a wider audience with very little overhead for build/release/install mechanics. +.. deprecated:: 3.3 + :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and + :ref:`packaging-install-index`. + .. toctree:: :maxdepth: 2 :numbered: @@ -36,9 +40,3 @@ following the above guidelines: .. toctree:: install.rst - - -.. seealso:: - - :ref:`packaging-index` and :ref:`packaging-install-index` - Documentation of Packaging, the new version of Distutils. diff --git a/Doc/install/index.rst b/Doc/install/index.rst new file mode 100644 index 0000000..58b7835 --- /dev/null +++ b/Doc/install/index.rst @@ -0,0 +1,56 @@ +.. _packaging-install-index: + +****************************** + Installing Python Projects +****************************** + +:Author: Greg Ward and Packaging contributors +:Release: |version| +:Date: |today| + +.. TODO: Fill in XXX comments + +.. The audience for this document includes people who don't know anything + about Python and aren't about to learn the language just in order to + install and maintain it for their users, i.e. system administrators. + Thus, I have to be sure to explain the basics at some point: + sys.path and PYTHONPATH at least. Should probably give pointers to + other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc. + + Finally, it might be useful to include all the material from my "Care + and Feeding of a Python Installation" talk in here somewhere. Yow! + +.. topic:: Abstract + + This document describes Packaging from the end-user's point of view: it + explains how to extend the functionality of a standard Python installation by + building and installing third-party Python modules and applications. + + +This guide is split into a simple overview followed by a longer presentation of +the :program:`pysetup` script, the Python package management tool used to +build, distribute, search for, install, remove and list Python distributions. + +.. TODO integrate install and pysetup instead of duplicating + +.. toctree:: + :maxdepth: 2 + :numbered: + + install + pysetup + pysetup-config + pysetup-servers + + +.. seealso:: + + :ref:`packaging-index` + The manual for developers of Python projects who want to package and + distribute them. This describes how to use :mod:`packaging` to make + projects easily found and added to an existing Python installation. + + :mod:`packaging` + A library reference for developers of packaging tools wanting to use + standalone building blocks like :mod:`~packaging.version` or + :mod:`~packaging.metadata`, or extend Packaging itself. diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst index f1a0f6b..53a69ae 100644 --- a/Doc/library/distutils.rst +++ b/Doc/library/distutils.rst @@ -12,21 +12,24 @@ additional modules into a Python installation. The new modules may be either 100%-pure Python, or may be extension modules written in C, or may be collections of Python packages which include modules coded in both Python and C. -This package is discussed in two separate chapters: +.. deprecated:: 3.3 + :mod:`packaging` replaces Distutils. See :ref:`packaging-index` and + :ref:`packaging-install-index`. +User documentation and API reference are provided in another document: + .. seealso:: :ref:`distutils-index` The manual for developers and packagers of Python modules. This describes how to prepare :mod:`distutils`\ -based packages so that they may be - easily installed into an existing Python installation. + easily installed into an existing Python installation. If also contains + instructions for end-users wanting to install a distutils-based package, + :ref:`install-index`. - :ref:`install-index` - An "administrators" manual which includes information on installing - modules into an existing Python installation. You do not need to be a - Python programmer to read this manual. +.. trick to silence a Sphinx warning .. toctree:: :hidden: diff --git a/Doc/tools/sphinxext/indexcontent.html b/Doc/tools/sphinxext/indexcontent.html index 10f9d26..778346f 100644 --- a/Doc/tools/sphinxext/indexcontent.html +++ b/Doc/tools/sphinxext/indexcontent.html @@ -21,9 +21,9 @@ + finding and installing modules and applications

+ packaging and distributing modules and applications