From 1c1d9a50262b491f755950aa2b3da20c64855581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Fri, 10 Jun 2011 23:26:31 +0200 Subject: Move useful function to packaging.util. Original patch by Erik Bray as part of #11595, changed by me to improve readability. --- Lib/packaging/config.py | 19 +++++++------------ Lib/packaging/util.py | 17 +++++++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Lib/packaging/config.py b/Lib/packaging/config.py index 6df2bab..be75da9 100644 --- a/Lib/packaging/config.py +++ b/Lib/packaging/config.py @@ -9,7 +9,8 @@ from configparser import RawConfigParser from packaging import logger from packaging.errors import PackagingOptionError from packaging.compiler.extension import Extension -from packaging.util import check_environ, iglob, resolve_name, strtobool +from packaging.util import (check_environ, iglob, resolve_name, strtobool, + split_multiline) from packaging.compiler import set_compiler from packaging.command import set_command from packaging.markers import interpret @@ -124,12 +125,6 @@ class Config: # XXX return value - def _multiline(self, value): - value = [v for v in - [v.strip() for v in value.split('\n')] - if v != ''] - return value - def _read_setup_cfg(self, parser, cfg_filename): cfg_directory = os.path.dirname(os.path.abspath(cfg_filename)) content = {} @@ -155,7 +150,7 @@ class Config: for key, value in content['metadata'].items(): key = key.replace('_', '-') if metadata.is_multi_field(key): - value = self._multiline(value) + value = split_multiline(value) if key == 'project-url': value = [(label.strip(), url.strip()) @@ -192,7 +187,7 @@ class Config: files = content['files'] self.dist.package_dir = files.pop('packages_root', None) - files = dict((key, self._multiline(value)) for key, value in + files = dict((key, split_multiline(value)) for key, value in files.items()) self.dist.packages = [] @@ -310,7 +305,7 @@ class Config: opt = opt.replace('-', '_') if opt == 'sub_commands': - val = self._multiline(val) + val = split_multiline(val) if isinstance(val, str): val = [val] @@ -348,14 +343,14 @@ class Config: raise PackagingOptionError(msg) def _load_compilers(self, compilers): - compilers = self._multiline(compilers) + compilers = split_multiline(compilers) if isinstance(compilers, str): compilers = [compilers] for compiler in compilers: set_compiler(compiler.strip()) def _load_commands(self, commands): - commands = self._multiline(commands) + commands = split_multiline(commands) if isinstance(commands, str): commands = [commands] for command in commands: diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index 812dbe3..dddfb3f 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -250,6 +250,14 @@ def split_quoted(s): return words +def split_multiline(value): + """Split a multiline string into a list, excluding blank lines.""" + + return [element for element in + (line.strip() for line in value.split('\n')) + if element] + + def execute(func, args, msg=None, verbose=0, dry_run=False): """Perform some action that affects the outside world. @@ -542,18 +550,15 @@ def write_file(filename, contents): def _is_package(path): - if not os.path.isdir(path): - return False - return os.path.isfile(os.path.join(path, '__init__.py')) + return os.path.isdir(path) and os.path.isfile( + os.path.join(path, '__init__.py')) # Code taken from the pip project def _is_archive_file(name): archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar') ext = splitext(name)[1].lower() - if ext in archives: - return True - return False + return ext in archives def _under(path, root): -- cgit v0.12 From 3605030c9b8b86c805c560ddab6613ddba451de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Fri, 10 Jun 2011 23:52:26 +0200 Subject: Fix assorted bugs in packaging.util.cfg_to_args (#11595). Original patch by Erik Bray. --- Lib/packaging/tests/test_util.py | 48 ++++++++++++++++++++++++++++++++++------ Lib/packaging/util.py | 31 +++++++++++++++++--------- Misc/ACKS | 1 + Misc/NEWS | 4 ++++ 4 files changed, 67 insertions(+), 17 deletions(-) diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py index 5a94a73..68ad8eb 100644 --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -8,16 +8,18 @@ import subprocess from io import StringIO from packaging.tests import support, unittest +from packaging.tests.test_config import SETUP_CFG from packaging.errors import ( PackagingPlatformError, PackagingByteCompileError, PackagingFileError, PackagingExecError, InstallationException) from packaging import util +from packaging.dist import Distribution from packaging.util import ( convert_path, change_root, split_quoted, strtobool, rfc822_escape, get_compiler_versions, _MAC_OS_X_LD_VERSION, byte_compile, find_packages, spawn, get_pypirc_path, generate_pypirc, read_pypirc, resolve_name, iglob, RICH_GLOB, egginfo_to_distinfo, is_setuptools, is_distutils, is_packaging, - get_install_method) + get_install_method, cfg_to_args) PYPIRC = """\ @@ -88,13 +90,15 @@ class UtilTestCase(support.EnvironRestorer, support.LoggingCatcher, unittest.TestCase): - restore_environ = ['HOME'] + restore_environ = ['HOME', 'PLAT'] def setUp(self): super(UtilTestCase, self).setUp() - self.tmp_dir = self.mkdtemp() - self.rc = os.path.join(self.tmp_dir, '.pypirc') - os.environ['HOME'] = self.tmp_dir + self.addCleanup(os.chdir, os.getcwd()) + tempdir = self.mkdtemp() + self.rc = os.path.join(tempdir, '.pypirc') + os.environ['HOME'] = tempdir + os.chdir(tempdir) # saving the environment self.name = os.name self.platform = sys.platform @@ -103,7 +107,6 @@ class UtilTestCase(support.EnvironRestorer, self.join = os.path.join self.isabs = os.path.isabs self.splitdrive = os.path.splitdrive - #self._config_vars = copy(sysconfig._config_vars) # patching os.uname if hasattr(os, 'uname'): @@ -137,7 +140,6 @@ class UtilTestCase(support.EnvironRestorer, os.uname = self.uname else: del os.uname - #sysconfig._config_vars = copy(self._config_vars) util.find_executable = self.old_find_executable subprocess.Popen = self.old_popen sys.old_stdout = self.old_stdout @@ -491,6 +493,38 @@ class UtilTestCase(support.EnvironRestorer, content = f.read() self.assertEqual(content, WANTED) + def test_cfg_to_args(self): + opts = {'description-file': 'README', 'extra-files': '', + 'setup-hook': 'packaging.tests.test_config.hook'} + self.write_file('setup.cfg', SETUP_CFG % opts) + self.write_file('README', 'loooong description') + + args = cfg_to_args() + # use Distribution to get the contents of the setup.cfg file + dist = Distribution() + dist.parse_config_files() + metadata = dist.metadata + + self.assertEqual(args['name'], metadata['Name']) + # + .dev1 because the test SETUP_CFG also tests a hook function in + # test_config.py for appending to the version string + self.assertEqual(args['version'] + '.dev1', metadata['Version']) + self.assertEqual(args['author'], metadata['Author']) + self.assertEqual(args['author_email'], metadata['Author-Email']) + self.assertEqual(args['maintainer'], metadata['Maintainer']) + self.assertEqual(args['maintainer_email'], + metadata['Maintainer-Email']) + self.assertEqual(args['description'], metadata['Summary']) + self.assertEqual(args['long_description'], metadata['Description']) + self.assertEqual(args['classifiers'], metadata['Classifier']) + self.assertEqual(args['requires'], metadata['Requires-Dist']) + self.assertEqual(args['provides'], metadata['Provides-Dist']) + + self.assertEqual(args['package_dir'].get(''), dist.package_dir) + self.assertEqual(args['packages'], dist.packages) + self.assertEqual(args['scripts'], dist.scripts) + self.assertEqual(args['py_modules'], dist.py_modules) + class GlobTestCaseBase(support.TempdirManager, support.LoggingCatcher, diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index dddfb3f..76b8747 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -1015,16 +1015,20 @@ def cfg_to_args(path='setup.cfg'): "requires": ("metadata", "requires_dist"), "provides": ("metadata", "provides_dist"), # ** "obsoletes": ("metadata", "obsoletes_dist"), # ** + "package_dir": ("files", 'packages_root'), "packages": ("files",), "scripts": ("files",), "py_modules": ("files", "modules"), # ** } MULTI_FIELDS = ("classifiers", - "requires", "platforms", + "requires", + "provides", + "obsoletes", "packages", - "scripts") + "scripts", + "py_modules") def has_get_option(config, section, option): if config.has_option(section, option): @@ -1036,9 +1040,9 @@ def cfg_to_args(path='setup.cfg'): # The real code starts here config = RawConfigParser() - if not os.path.exists(file): + if not os.path.exists(path): raise PackagingFileError("file '%s' does not exist" % - os.path.abspath(file)) + os.path.abspath(path)) config.read(path) kwargs = {} @@ -1055,17 +1059,24 @@ def cfg_to_args(path='setup.cfg'): in_cfg_value = has_get_option(config, section, option) if not in_cfg_value: # There is no such option in the setup.cfg - if arg == "long_description": - filename = has_get_option(config, section, "description_file") - if filename: - with open(filename) as fp: - in_cfg_value = fp.read() + if arg == 'long_description': + filenames = has_get_option(config, section, 'description-file') + if filenames: + filenames = split_multiline(filenames) + in_cfg_value = [] + for filename in filenames: + with open(filename) as fp: + in_cfg_value.append(fp.read()) + in_cfg_value = '\n\n'.join(in_cfg_value) else: continue + if arg == 'package_dir' and in_cfg_value: + in_cfg_value = {'': in_cfg_value} + if arg in MULTI_FIELDS: # support multiline options - in_cfg_value = in_cfg_value.strip().split('\n') + in_cfg_value = split_multiline(in_cfg_value) kwargs[arg] = in_cfg_value diff --git a/Misc/ACKS b/Misc/ACKS index de5410c..5380c72 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -116,6 +116,7 @@ Monty Brandenberg Georg Brandl Christopher Brannon Terrence Brannon +Erik Bray Brian Brazil Dave Brennan Tom Bridgman diff --git a/Misc/NEWS b/Misc/NEWS index f2d3ffe..5b01308 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -187,6 +187,10 @@ Core and Builtins Library ------- +- Issue #11595: Fix assorted bugs in packaging.util.cfg_to_args, a + compatibility helper for the distutils-packaging transition. Original patch + by Erik Bray. + - Issue #12246: Warn and fail when trying to install a third-party project from an uninstalled Python (built in a source checkout). Original patch by Tshepang Lekhonkhobe. -- cgit v0.12 From 8474f2901b78cc9afe9487ed9236430f43be18b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sat, 11 Jun 2011 00:21:18 +0200 Subject: setup.cfg: Document that description-file can contain more than one file --- Doc/packaging/setupcfg.rst | 1 + Lib/packaging/config.py | 13 +++++-------- Lib/packaging/tests/test_config.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Doc/packaging/setupcfg.rst b/Doc/packaging/setupcfg.rst index aa8216f..463522b 100644 --- a/Doc/packaging/setupcfg.rst +++ b/Doc/packaging/setupcfg.rst @@ -285,6 +285,7 @@ One extra field not present in PEP 345 is supported: description-file Path to a text file that will be used to fill the ``description`` field. + Multiple values are accepted; they must be separated by whitespace. ``description-file`` and ``description`` are mutually exclusive. *optional* diff --git a/Lib/packaging/config.py b/Lib/packaging/config.py index be75da9..3427d9a 100644 --- a/Lib/packaging/config.py +++ b/Lib/packaging/config.py @@ -163,21 +163,18 @@ class Config: "mutually exclusive") raise PackagingOptionError(msg) - if isinstance(value, list): - filenames = value - else: - filenames = value.split() + filenames = value.split() - # concatenate each files - value = '' + # concatenate all files + value = [] for filename in filenames: # will raise if file not found with open(filename) as description_file: - value += description_file.read().strip() + '\n' + value.append(description_file.read().strip()) # add filename as a required file if filename not in metadata.requires_files: metadata.requires_files.append(filename) - value = value.strip() + value = '\n'.join(value).strip() key = 'description' if metadata.is_metadata_field(key): diff --git a/Lib/packaging/tests/test_config.py b/Lib/packaging/tests/test_config.py index 9198ead..1669862 100644 --- a/Lib/packaging/tests/test_config.py +++ b/Lib/packaging/tests/test_config.py @@ -327,7 +327,7 @@ class ConfigTestCase(support.TempdirManager, self.assertIn('could not import setup_hook', logs[0]) def test_metadata_requires_description_files_missing(self): - self.write_setup({'description-file': 'README\n README2'}) + self.write_setup({'description-file': 'README README2'}) self.write_file('README', 'yeah') self.write_file('README2', 'yeah') os.mkdir('src') -- cgit v0.12 From 643cb7345f66f0d9dbc1066567ffb1d519026388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sat, 11 Jun 2011 00:33:38 +0200 Subject: =?UTF-8?q?Allow=20multiple=20setup=20hooks=20in=20packaging?= =?UTF-8?q?=E2=80=99s=20setup.cfg=20files=20(#12240).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original patch by Erik Bray. --- Doc/packaging/setupcfg.rst | 14 +++++++++----- Lib/packaging/config.py | 37 +++++++++++++++++++------------------ Lib/packaging/tests/test_config.py | 36 +++++++++++++++++++++++++++++------- Lib/packaging/tests/test_util.py | 2 +- Misc/NEWS | 3 +++ 5 files changed, 61 insertions(+), 31 deletions(-) diff --git a/Doc/packaging/setupcfg.rst b/Doc/packaging/setupcfg.rst index 463522b..2b01ffb 100644 --- a/Doc/packaging/setupcfg.rst +++ b/Doc/packaging/setupcfg.rst @@ -176,15 +176,19 @@ compilers compilers = hotcompiler.SmartCCompiler -setup_hook - defines a callable that will be called right after the - :file:`setup.cfg` file is read. The callable receives the configuration - in form of a mapping and can make some changes to it. *optional* +setup_hooks + Defines a list of callables to be called right after the :file:`setup.cfg` + file is read, before any other processing. The callables are executed in the + order they're found in the file; if one of them cannot be found, tools should + not stop, but for example produce a warning and continue with the next line. + Each callable receives the configuration as a dictionary (keys are + :file:`setup.cfg` sections, values are dictionaries of fields) and can make + any changes to it. *optional*, *multi* Example:: [global] - setup_hook = package.setup.customize_dist + setup_hooks = package.setup.customize_dist Metadata diff --git a/Lib/packaging/config.py b/Lib/packaging/config.py index 3427d9a..21bbcf8 100644 --- a/Lib/packaging/config.py +++ b/Lib/packaging/config.py @@ -61,17 +61,15 @@ def get_resources_dests(resources_root, rules): class Config: - """Reads configuration files and work with the Distribution instance - """ + """Class used to work with configuration files""" def __init__(self, dist): self.dist = dist - self.setup_hook = None + self.setup_hooks = [] - def run_hook(self, config): - if self.setup_hook is None: - return - # the hook gets only the config - self.setup_hook(config) + def run_hooks(self, config): + """Run setup hooks in the order defined in the spec.""" + for hook in self.setup_hooks: + hook(config) def find_config_files(self): """Find as many configuration files as should be processed for this @@ -131,17 +129,20 @@ class Config: for section in parser.sections(): content[section] = dict(parser.items(section)) - # global:setup_hook is called *first* + # global setup hooks are called first if 'global' in content: - if 'setup_hook' in content['global']: - setup_hook = content['global']['setup_hook'] - try: - self.setup_hook = resolve_name(setup_hook) - except ImportError as e: - logger.warning('could not import setup_hook: %s', - e.args[0]) - else: - self.run_hook(content) + if 'setup_hooks' in content['global']: + setup_hooks = split_multiline(content['global']['setup_hooks']) + + for line in setup_hooks: + try: + hook = resolve_name(line) + except ImportError as e: + logger.warning('cannot find setup hook: %s', e.args[0]) + else: + self.setup_hooks.append(hook) + + self.run_hooks(content) metadata = self.dist.metadata diff --git a/Lib/packaging/tests/test_config.py b/Lib/packaging/tests/test_config.py index 1669862..6be63eb 100644 --- a/Lib/packaging/tests/test_config.py +++ b/Lib/packaging/tests/test_config.py @@ -90,7 +90,7 @@ commands = compilers = packaging.tests.test_config.DCompiler -setup_hook = %(setup-hook)s +setup_hooks = %(setup-hooks)s @@ -135,8 +135,16 @@ class DCompiler: pass -def hook(content): - content['metadata']['version'] += '.dev1' +def version_hook(config): + config['metadata']['version'] += '.dev1' + + +def first_hook(config): + config['files']['modules'] += '\n first' + + +def third_hook(config): + config['files']['modules'] += '\n third' class FooBarBazTest: @@ -186,7 +194,7 @@ class ConfigTestCase(support.TempdirManager, def write_setup(self, kwargs=None): opts = {'description-file': 'README', 'extra-files': '', - 'setup-hook': 'packaging.tests.test_config.hook'} + 'setup-hooks': 'packaging.tests.test_config.version_hook'} if kwargs: opts.update(kwargs) self.write_file('setup.cfg', SETUP_CFG % opts, encoding='utf-8') @@ -318,13 +326,27 @@ class ConfigTestCase(support.TempdirManager, self.assertEqual(ext.extra_compile_args, cargs) self.assertEqual(ext.language, 'cxx') - def test_missing_setuphook_warns(self): - self.write_setup({'setup-hook': 'this.does._not.exist'}) + def test_missing_setup_hook_warns(self): + self.write_setup({'setup-hooks': 'this.does._not.exist'}) self.write_file('README', 'yeah') dist = self.get_dist() logs = self.get_logs(logging.WARNING) self.assertEqual(1, len(logs)) - self.assertIn('could not import setup_hook', logs[0]) + self.assertIn('cannot find setup hook', logs[0]) + + def test_multiple_setup_hooks(self): + self.write_setup({ + 'setup-hooks': '\n packaging.tests.test_config.first_hook' + '\n packaging.tests.test_config.missing_hook' + '\n packaging.tests.test_config.third_hook' + }) + self.write_file('README', 'yeah') + dist = self.get_dist() + + self.assertEqual(['haven', 'first', 'third'], dist.py_modules) + logs = self.get_logs(logging.WARNING) + self.assertEqual(1, len(logs)) + self.assertIn('cannot find setup hook', logs[0]) def test_metadata_requires_description_files_missing(self): self.write_setup({'description-file': 'README README2'}) diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py index 68ad8eb..f657ab2 100644 --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -495,7 +495,7 @@ class UtilTestCase(support.EnvironRestorer, def test_cfg_to_args(self): opts = {'description-file': 'README', 'extra-files': '', - 'setup-hook': 'packaging.tests.test_config.hook'} + 'setup-hooks': 'packaging.tests.test_config.version_hook'} self.write_file('setup.cfg', SETUP_CFG % opts) self.write_file('README', 'loooong description') diff --git a/Misc/NEWS b/Misc/NEWS index 5b01308..47f3b0e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -187,6 +187,9 @@ Core and Builtins Library ------- +- Issue #12240: Allow multiple setup hooks in packaging's setup.cfg files. + Original patch by Erik Bray. + - Issue #11595: Fix assorted bugs in packaging.util.cfg_to_args, a compatibility helper for the distutils-packaging transition. Original patch by Erik Bray. -- cgit v0.12 From 7ebadd5d8da049d3ca23737db0312bde6fdaa79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sat, 11 Jun 2011 03:27:03 +0200 Subject: Use correct directive to document one method --- Doc/library/collections.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 6c9b1e5..39a03dd 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -83,7 +83,7 @@ The class can be used to simulate nested scopes and is useful in templating. creating subcontexts that can be updated without altering values in any of the parent mappings. - .. attribute:: parents() + .. method:: parents() Returns a new :class:`ChainMap` containing all of the maps in the current instance except the first one. This is useful for skipping the first map -- cgit v0.12 From 6280606a5781147158a607b409ec7ebf4732bd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89ric=20Araujo?= Date: Sat, 11 Jun 2011 09:46:07 +0200 Subject: Adjust logging in packaging.util.spawn (related to #11599) --- Lib/packaging/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index 76b8747..29994c0 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -777,12 +777,13 @@ def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None): Raise PackagingExecError if running the program fails in any way; just return on success. """ - logger.info(' '.join(cmd)) + logger.debug('spawn: running %r', cmd) if dry_run: + logging.debug('dry run, no process actually spawned') return exit_status = subprocess.call(cmd, env=env) if exit_status != 0: - msg = "command '%s' failed with exit status %d" + msg = "command %r failed with exit status %d" raise PackagingExecError(msg % (cmd, exit_status)) -- cgit v0.12