diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/compileall.py | 14 | ||||
| -rw-r--r-- | Lib/distutils/tests/support.py | 7 | ||||
| -rw-r--r-- | Lib/packaging/command/build_py.py | 2 | ||||
| -rw-r--r-- | Lib/packaging/command/install_lib.py | 2 | ||||
| -rw-r--r-- | Lib/packaging/command/register.py | 1 | ||||
| -rw-r--r-- | Lib/packaging/command/upload.py | 1 | ||||
| -rw-r--r-- | Lib/packaging/config.py | 31 | ||||
| -rw-r--r-- | Lib/packaging/create.py | 6 | ||||
| -rw-r--r-- | Lib/packaging/tests/support.py | 40 | ||||
| -rw-r--r-- | Lib/packaging/tests/test_config.py | 47 | ||||
| -rw-r--r-- | Lib/packaging/tests/test_util.py | 9 | ||||
| -rw-r--r-- | Lib/packaging/util.py | 163 | ||||
| -rw-r--r-- | Lib/pipes.py | 2 | ||||
| -rw-r--r-- | Lib/shutil.py | 3 |
14 files changed, 145 insertions, 183 deletions
diff --git a/Lib/compileall.py b/Lib/compileall.py index 743be27..d3cff6a 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -142,7 +142,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, Arguments (all optional): - skip_curdir: if true, skip current directory (default true) + skip_curdir: if true, skip current directory (default True) maxlevels: max recursion level (default 0) force: as for compile_dir() (default False) quiet: as for compile_dir() (default False) @@ -177,17 +177,17 @@ def main(): help='use legacy (pre-PEP3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' - 'compile time tracebacks and in runtime ' + 'compile-time tracebacks and in runtime ' 'tracebacks in cases where the source file is ' 'unavailable')) parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, - help=('skip files matching the regular expression. ' - 'The regexp is searched for in the full path ' - 'to each file considered for compilation.')) + help=('skip files matching the regular expression; ' + 'the regexp is searched for in the full path ' + 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation. ' - 'If "-", names are read from stdin.')) + 'FILE to the list considered for compilation; ' + 'if "-", names are read from stdin')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' diff --git a/Lib/distutils/tests/support.py b/Lib/distutils/tests/support.py index 44fcd6b..d77bbee 100644 --- a/Lib/distutils/tests/support.py +++ b/Lib/distutils/tests/support.py @@ -175,10 +175,9 @@ def _get_xxmodule_path(): def fixup_build_ext(cmd): """Function needed to make build_ext tests pass. - When Python was build with --enable-shared on Unix, -L. is not good - enough to find the libpython<blah>.so. This is because regrtest runs - it under a tempdir, not in the top level where the .so lives. By the - time we've gotten here, Python's already been chdir'd to the tempdir. + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython<blah>.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. When Python was built with in debug mode on Windows, build_ext commands need their debug attribute set, and it is not done automatically for diff --git a/Lib/packaging/command/build_py.py b/Lib/packaging/command/build_py.py index 360f4c9..7baa6e4 100644 --- a/Lib/packaging/command/build_py.py +++ b/Lib/packaging/command/build_py.py @@ -393,7 +393,7 @@ class build_py(Command, Mixin2to3): self.get_command_name()) return - from packaging.util import byte_compile + from packaging.util import byte_compile # FIXME use compileall prefix = self.build_lib if prefix[-1] != os.sep: prefix = prefix + os.sep diff --git a/Lib/packaging/command/install_lib.py b/Lib/packaging/command/install_lib.py index 5ff9cee..978f0ef 100644 --- a/Lib/packaging/command/install_lib.py +++ b/Lib/packaging/command/install_lib.py @@ -122,7 +122,7 @@ class install_lib(Command): self.get_command_name()) return - from packaging.util import byte_compile + from packaging.util import byte_compile # FIXME use compileall # Get the "--root" directory supplied to the "install_dist" command, # and use it as a prefix to strip off the purported filename diff --git a/Lib/packaging/command/register.py b/Lib/packaging/command/register.py index 006dfdf..67cda80 100644 --- a/Lib/packaging/command/register.py +++ b/Lib/packaging/command/register.py @@ -2,7 +2,6 @@ # Contributed by Richard Jones -import io import getpass import urllib.error import urllib.parse diff --git a/Lib/packaging/command/upload.py b/Lib/packaging/command/upload.py index e39016c..f56d2c6 100644 --- a/Lib/packaging/command/upload.py +++ b/Lib/packaging/command/upload.py @@ -5,7 +5,6 @@ import socket import logging import platform import urllib.parse -from io import BytesIO from base64 import standard_b64encode from hashlib import md5 from urllib.error import HTTPError diff --git a/Lib/packaging/config.py b/Lib/packaging/config.py index 43263f7..e02800e 100644 --- a/Lib/packaging/config.py +++ b/Lib/packaging/config.py @@ -16,6 +16,19 @@ from packaging.command import set_command from packaging.markers import interpret +def _check_name(name, packages): + if '.' not in name: + return + parts = name.split('.') + modname = parts[-1] + parent = '.'.join(parts[:-1]) + if parent not in packages: + # we could log a warning instead of raising, but what's the use + # of letting people build modules they can't import? + raise PackagingOptionError( + 'parent package for extension %r not found' % name) + + def _pop_values(values_dct, key): """Remove values from the dictionary and convert them as a list""" vals_str = values_dct.pop(key, '') @@ -142,7 +155,8 @@ class Config: try: hook = resolve_name(line) except ImportError as e: - logger.warning('cannot find setup hook: %s', e.args[0]) + logger.warning('cannot find setup hook: %s', + e.args[0]) else: self.setup_hooks.append(hook) self.run_hooks(content) @@ -216,7 +230,7 @@ class Config: for data in files.get('package_data', []): data = data.split('=') if len(data) != 2: - continue # XXX error should never pass silently + continue # FIXME errors should never pass silently key, value = data self.dist.package_data[key.strip()] = value.strip() @@ -251,13 +265,18 @@ class Config: ext_modules = self.dist.ext_modules for section_key in content: - labels = section_key.split('=') + # no str.partition in 2.4 :( + labels = section_key.split(':') if len(labels) == 2 and labels[0] == 'extension': - # labels[1] not used from now but should be implemented - # for extension build dependency values_dct = content[section_key] + if 'name' in values_dct: + raise PackagingOptionError( + 'extension name should be given as [extension: name], ' + 'not as key') + name = labels[1].strip() + _check_name(name, self.dist.packages) ext_modules.append(Extension( - values_dct.pop('name'), + name, _pop_values(values_dct, 'sources'), _pop_values(values_dct, 'include_dirs'), _pop_values(values_dct, 'define_macros'), diff --git a/Lib/packaging/create.py b/Lib/packaging/create.py index 8f39172..ecabca0 100644 --- a/Lib/packaging/create.py +++ b/Lib/packaging/create.py @@ -36,7 +36,7 @@ from packaging._trove import all_classifiers as _CLASSIFIERS_LIST from packaging.version import is_valid_version _FILENAME = 'setup.cfg' -_DEFAULT_CFG = '.pypkgcreate' +_DEFAULT_CFG = '.pypkgcreate' # FIXME use a section in user .pydistutils.cfg _helptext = { 'name': ''' @@ -127,6 +127,10 @@ def ask_yn(question, default=None, helptext=None): print('\nERROR: You must select "Y" or "N".\n') +# XXX use util.ask +# FIXME: if prompt ends with '?', don't add ':' + + def ask(question, default=None, helptext=None, required=True, lengthy=False, multiline=False): prompt = '%s: ' % (question,) diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py index df6195a..33d5966 100644 --- a/Lib/packaging/tests/support.py +++ b/Lib/packaging/tests/support.py @@ -1,5 +1,8 @@ """Support code for packaging test cases. +*This module should not be considered public: its content and API may +change in incompatible ways.* + A few helper classes are provided: LoggingCatcher, TempdirManager and EnvironRestorer. They are written to be used as mixins:: @@ -7,6 +10,7 @@ EnvironRestorer. They are written to be used as mixins:: from packaging.tests.support import LoggingCatcher class SomeTestCase(LoggingCatcher, unittest.TestCase): + ... If you need to define a setUp method on your test class, you have to call the mixin class' setUp method or it won't work (same thing for @@ -14,17 +18,18 @@ tearDown): def setUp(self): super(SomeTestCase, self).setUp() - ... # other setup code + ... # other setup code Also provided is a DummyCommand class, useful to mock commands in the -tests of another command that needs them, a create_distribution function -and a skip_unless_symlink decorator. +tests of another command that needs them, for example to fake +compilation in build_ext (this requires that the mock build_ext command +be injected into the distribution object's command_obj dictionary). -Also provided is a DummyCommand class, useful to mock commands in the -tests of another command that needs them, a create_distribution function -and a skip_unless_symlink decorator. +For tests that need to compile an extension module, use the +copy_xxmodule_c and fixup_build_ext functions. Each class or function has a docstring to explain its purpose and usage. +Existing tests should also be used as examples. """ import os @@ -39,9 +44,17 @@ from packaging.dist import Distribution from packaging.tests import unittest from test.support import requires_zlib, unlink -__all__ = ['LoggingCatcher', 'TempdirManager', 'EnvironRestorer', - 'DummyCommand', 'unittest', 'create_distribution', - 'skip_unless_symlink', 'requires_zlib', 'copy_xxmodule_c'] +# define __all__ to make pydoc more useful +__all__ = [ + # TestCase mixins + 'LoggingCatcher', 'TempdirManager', 'EnvironRestorer', + # mocks + 'DummyCommand', 'TestDistribution', + # misc. functions and decorators + 'fake_dec', 'create_distribution', 'copy_xxmodule_c', 'fixup_build_ext', + # imported from this module for backport purposes + 'unittest', 'requires_zlib', 'skip_unless_symlink', +] logger = logging.getLogger('packaging') @@ -233,6 +246,8 @@ class DummyCommand: Useful for mocking one dependency command in the tests for another command, see e.g. the dummy build command in test_build_scripts. """ + # XXX does not work with dist.get_reinitialized_command, which typechecks + # and wants a finalized attribute def __init__(self, **kwargs): for kw, val in kwargs.items(): @@ -308,10 +323,9 @@ def _get_xxmodule_path(): def fixup_build_ext(cmd): """Function needed to make build_ext tests pass. - When Python was build with --enable-shared on Unix, -L. is not good - enough to find the libpython<blah>.so. This is because regrtest runs - it under a tempdir, not in the top level where the .so lives. By the - time we've gotten here, Python's already been chdir'd to the tempdir. + When Python was built with --enable-shared on Unix, -L. is not enough to + find libpython<blah>.so, because regrtest runs in a tempdir, not in the + source directory where the .so lives. When Python was built with in debug mode on Windows, build_ext commands need their debug attribute set, and it is not done automatically for diff --git a/Lib/packaging/tests/test_config.py b/Lib/packaging/tests/test_config.py index 43ab2c8..bcb55fb 100644 --- a/Lib/packaging/tests/test_config.py +++ b/Lib/packaging/tests/test_config.py @@ -6,7 +6,7 @@ from io import StringIO from packaging import command from packaging.dist import Distribution -from packaging.errors import PackagingFileError +from packaging.errors import PackagingFileError, PackagingOptionError from packaging.compiler import new_compiler, _COMPILERS from packaging.command.sdist import sdist @@ -100,21 +100,21 @@ sub_commands = foo # Can not be merged with SETUP_CFG else install_dist # command will fail when trying to compile C sources +# TODO use a DummyCommand to mock build_ext EXT_SETUP_CFG = """ [files] packages = one two + parent.undeclared -[extension=speed_coconuts] -name = one.speed_coconuts +[extension:one.speed_coconuts] sources = c_src/speed_coconuts.c extra_link_args = "`gcc -print-file-name=libgcc.a`" -shared define_macros = HAVE_CAIRO HAVE_GTK2 libraries = gecodeint gecodekernel -- sys.platform != 'win32' GecodeInt GecodeKernel -- sys.platform == 'win32' -[extension=fast_taunt] -name = two.fast_taunt +[extension: two.fast_taunt] sources = cxx_src/utils_taunt.cxx cxx_src/python_module.cxx include_dirs = /usr/include/gecode @@ -124,6 +124,30 @@ extra_compile_args = -fPIC -O2 /DGECODE_VERSION='win32' -- sys.platform == 'win32' language = cxx +# corner case: if the parent package of an extension is declared but +# not its grandparent, it's legal +[extension: parent.undeclared._speed] +sources = parent/undeclared/_speed.c +""" + +EXT_SETUP_CFG_BUGGY_1 = """ +[extension: realname] +name = crash_here +""" + +EXT_SETUP_CFG_BUGGY_2 = """ +[files] +packages = ham + +[extension: spam.eggs] +""" + +EXT_SETUP_CFG_BUGGY_3 = """ +[files] +packages = ok + ok.works + +[extension: ok.works.breaks._ext] """ HOOKS_MODULE = """ @@ -311,7 +335,7 @@ class ConfigTestCase(support.TempdirManager, dist = self.get_dist() ext_modules = dict((mod.name, mod) for mod in dist.ext_modules) - self.assertEqual(len(ext_modules), 2) + self.assertEqual(len(ext_modules), 3) ext = ext_modules.get('one.speed_coconuts') self.assertEqual(ext.sources, ['c_src/speed_coconuts.c']) self.assertEqual(ext.define_macros, ['HAVE_CAIRO', 'HAVE_GTK2']) @@ -335,6 +359,15 @@ class ConfigTestCase(support.TempdirManager, self.assertEqual(ext.extra_compile_args, cargs) self.assertEqual(ext.language, 'cxx') + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_1) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_2) + self.assertRaises(PackagingOptionError, self.get_dist) + + self.write_file('setup.cfg', EXT_SETUP_CFG_BUGGY_3) + self.assertRaises(PackagingOptionError, self.get_dist) + def test_project_setup_hook_works(self): # Bug #11637: ensure the project directory is on sys.path to allow # project-specific hooks @@ -358,7 +391,7 @@ class ConfigTestCase(support.TempdirManager, self.write_setup({ 'setup-hooks': '\n packaging.tests.test_config.first_hook' '\n packaging.tests.test_config.missing_hook' - '\n packaging.tests.test_config.third_hook' + '\n packaging.tests.test_config.third_hook', }) self.write_file('README', 'yeah') dist = self.get_dist() diff --git a/Lib/packaging/tests/test_util.py b/Lib/packaging/tests/test_util.py index 21ac4e2..5e804c2 100644 --- a/Lib/packaging/tests/test_util.py +++ b/Lib/packaging/tests/test_util.py @@ -15,7 +15,7 @@ from packaging.errors import ( from packaging import util from packaging.dist import Distribution from packaging.util import ( - convert_path, change_root, split_quoted, strtobool, rfc822_escape, + convert_path, change_root, split_quoted, strtobool, get_compiler_versions, _MAC_OS_X_LD_VERSION, byte_compile, find_packages, spawn, get_pypirc_path, generate_pypirc, read_pypirc, resolve_name, iglob, RICH_GLOB, egginfo_to_distinfo, is_setuptools, is_distutils, is_packaging, @@ -255,13 +255,6 @@ class UtilTestCase(support.EnvironRestorer, for n in no: self.assertFalse(strtobool(n)) - def test_rfc822_escape(self): - header = 'I am a\npoor\nlonesome\nheader\n' - res = rfc822_escape(header) - wanted = ('I am a%(8s)spoor%(8s)slonesome%(8s)s' - 'header%(8s)s') % {'8s': '\n' + 8 * ' '} - self.assertEqual(res, wanted) - def test_find_exe_version(self): # the ld version scheme under MAC OS is: # ^@(#)PROGRAM:ld PROJECT:ld64-VERSION diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py index f94be5d..49c5991 100644 --- a/Lib/packaging/util.py +++ b/Lib/packaging/util.py @@ -8,8 +8,6 @@ import errno import shutil import string import hashlib -import tarfile -import zipfile import posixpath import subprocess import sysconfig @@ -23,6 +21,30 @@ from packaging.errors import (PackagingPlatformError, PackagingFileError, PackagingByteCompileError, PackagingExecError, InstallationException, PackagingInternalError) +__all__ = [ + # file dependencies + 'newer', 'newer_group', + # helpers for commands (dry-run system) + 'execute', 'write_file', + # spawning programs + 'find_executable', 'spawn', + # path manipulation + 'convert_path', 'change_root', + # 2to3 conversion + 'Mixin2to3', 'run_2to3', + # packaging compatibility helpers + 'cfg_to_args', 'generate_setup_py', + 'egginfo_to_distinfo', + 'get_install_method', + # misc + 'ask', 'check_environ', 'encode_multipart', 'resolve_name', + # querying for information TODO move to sysconfig + 'get_compiler_versions', 'get_platform', 'set_platform', + # configuration TODO move to packaging.config + 'get_pypirc_path', 'read_pypirc', 'generate_pypirc', + 'strtobool', 'split_multiline', +] + _PLATFORM = None _DEFAULT_INSTALLER = 'packaging' @@ -152,31 +174,6 @@ def check_environ(): _environ_checked = True -def subst_vars(s, local_vars): - """Perform shell/Perl-style variable substitution on 'string'. - - Every occurrence of '$' followed by a name is considered a variable, and - variable is substituted by the value found in the 'local_vars' - dictionary, or in 'os.environ' if it's not in 'local_vars'. - 'os.environ' is first checked/augmented to guarantee that it contains - certain values: see 'check_environ()'. Raise ValueError for any - variables not found in either 'local_vars' or 'os.environ'. - """ - check_environ() - - def _subst(match, local_vars=local_vars): - var_name = match.group(1) - if var_name in local_vars: - return str(local_vars[var_name]) - else: - return os.environ[var_name] - - try: - return re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s) - except KeyError as var: - raise ValueError("invalid variable '$%s'" % var) - - # Needed by 'split_quoted()' _wordchars_re = _squote_re = _dquote_re = None @@ -188,6 +185,8 @@ def _init_regex(): _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') +# TODO replace with shlex.split after testing + def split_quoted(s): """Split a string up according to Unix shell-like rules for quotes and backslashes. @@ -435,15 +434,6 @@ byte_compile(files, optimize=%r, force=%r, file, cfile_base) -def rfc822_escape(header): - """Return a form of *header* suitable for inclusion in an RFC 822-header. - - This function ensures there are 8 spaces after each newline. - """ - lines = header.split('\n') - sep = '\n' + 8 * ' ' - return sep.join(lines) - _RE_VERSION = re.compile('(\d+\.\d+(\.\d+)*)') _MAC_OS_X_LD_VERSION = re.compile('^@\(#\)PROGRAM:ld ' 'PROJECT:ld64-((\d+)(\.\d+)*)') @@ -543,6 +533,10 @@ def write_file(filename, contents): """Create *filename* and write *contents* to it. *contents* is a sequence of strings without line terminators. + + This functions is not intended to replace the usual with open + write + idiom in all cases, only with Command.execute, which runs depending on + the dry_run argument and also logs its arguments). """ with open(filename, "w") as f: for line in contents: @@ -562,6 +556,7 @@ def _is_archive_file(name): def _under(path, root): + # XXX use os.path path = path.split(os.sep) root = root.split(os.sep) if len(root) > len(path): @@ -664,103 +659,11 @@ def splitext(path): return base, ext -def unzip_file(filename, location, flatten=True): - """Unzip the file *filename* into the *location* directory.""" - if not os.path.exists(location): - os.makedirs(location) - with open(filename, 'rb') as zipfp: - zip = zipfile.ZipFile(zipfp) - leading = has_leading_dir(zip.namelist()) and flatten - for name in zip.namelist(): - data = zip.read(name) - fn = name - if leading: - fn = split_leading_dir(name)[1] - fn = os.path.join(location, fn) - dir = os.path.dirname(fn) - if not os.path.exists(dir): - os.makedirs(dir) - if fn.endswith('/') or fn.endswith('\\'): - # A directory - if not os.path.exists(fn): - os.makedirs(fn) - else: - with open(fn, 'wb') as fp: - fp.write(data) - - -def untar_file(filename, location): - """Untar the file *filename* into the *location* directory.""" - if not os.path.exists(location): - os.makedirs(location) - if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): - mode = 'r:gz' - elif (filename.lower().endswith('.bz2') - or filename.lower().endswith('.tbz')): - mode = 'r:bz2' - elif filename.lower().endswith('.tar'): - mode = 'r' - else: - mode = 'r:*' - with tarfile.open(filename, mode) as tar: - leading = has_leading_dir(member.name for member in tar.getmembers()) - for member in tar.getmembers(): - fn = member.name - if leading: - fn = split_leading_dir(fn)[1] - path = os.path.join(location, fn) - if member.isdir(): - if not os.path.exists(path): - os.makedirs(path) - else: - try: - fp = tar.extractfile(member) - except (KeyError, AttributeError): - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - continue - try: - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - with open(path, 'wb') as destfp: - shutil.copyfileobj(fp, destfp) - finally: - fp.close() - - -def has_leading_dir(paths): - """Return true if all the paths have the same leading path name. - - In other words, check that everything is in one subdirectory in an - archive. - """ - common_prefix = None - for path in paths: - prefix, rest = split_leading_dir(path) - if not prefix: - return False - elif common_prefix is None: - common_prefix = prefix - elif prefix != common_prefix: - return False - return True - - -def split_leading_dir(path): - path = str(path) - path = path.lstrip('/').lstrip('\\') - if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) - or '\\' not in path): - return path.split('/', 1) - elif '\\' in path: - return path.split('\\', 1) - else: - return path, '' - if sys.platform == 'darwin': _cfg_target = None _cfg_target_split = None + def spawn(cmd, search_path=True, verbose=0, dry_run=False, env=None): """Run another program specified as a command list 'cmd' in a new process. @@ -1510,7 +1413,7 @@ def encode_multipart(fields, files, boundary=None): for key, values in fields: # handle multiple entries for the same name if not isinstance(values, (tuple, list)): - values=[values] + values = [values] for value in values: l.extend(( diff --git a/Lib/pipes.py b/Lib/pipes.py index 693309f..f1a16f6 100644 --- a/Lib/pipes.py +++ b/Lib/pipes.py @@ -54,8 +54,6 @@ for the built-in function open() or for os.popen(). To create a new template object initialized to a given one: t2 = t.clone() - -For an example, see the function test() at the end of the file. """ # ' diff --git a/Lib/shutil.py b/Lib/shutil.py index 4b75262..0c096cb 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -35,7 +35,7 @@ __all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2", "register_archive_format", "unregister_archive_format", "get_unpack_formats", "register_unpack_format", "unregister_unpack_format", "unpack_archive", - "ignore_patterns"] + "ignore_patterns", "chown"] # disk_usage is added later, if available on the platform class Error(EnvironmentError): @@ -791,6 +791,7 @@ elif os.name == 'nt': used = total - free return _ntuple_diskusage(total, used, free) + def chown(path, user=None, group=None): """Change owner user and group of the given path. |