Branch merge

11 files changed, 167 insertions, 253 deletions

diff --git a/Lib/packaging/command/__init__.py b/Lib/packaging/command/__init__.py
index cd34fc8..87227c0 100644
--- a/Lib/packaging/command/__init__.py
+++ b/Lib/packaging/command/__init__.py
@@ -6,38 +6,28 @@ from packaging.util import resolve_name
 __all__ = ['get_command_names', 'set_command', 'get_command_class',
            'STANDARD_COMMANDS']
 
-_COMMANDS = {
-    'check': 'packaging.command.check.check',
-    'test': 'packaging.command.test.test',
-    'build': 'packaging.command.build.build',
-    'build_py': 'packaging.command.build_py.build_py',
-    'build_ext': 'packaging.command.build_ext.build_ext',
-    'build_clib': 'packaging.command.build_clib.build_clib',
-    'build_scripts': 'packaging.command.build_scripts.build_scripts',
-    'clean': 'packaging.command.clean.clean',
-    'install_dist': 'packaging.command.install_dist.install_dist',
-    'install_lib': 'packaging.command.install_lib.install_lib',
-    'install_headers': 'packaging.command.install_headers.install_headers',
-    'install_scripts': 'packaging.command.install_scripts.install_scripts',
-    'install_data': 'packaging.command.install_data.install_data',
-    'install_distinfo':
-        'packaging.command.install_distinfo.install_distinfo',
-    'sdist': 'packaging.command.sdist.sdist',
-    'bdist': 'packaging.command.bdist.bdist',
-    'bdist_dumb': 'packaging.command.bdist_dumb.bdist_dumb',
-    'bdist_wininst': 'packaging.command.bdist_wininst.bdist_wininst',
-    'register': 'packaging.command.register.register',
-    'upload': 'packaging.command.upload.upload',
-    'upload_docs': 'packaging.command.upload_docs.upload_docs',
-}
-
-# XXX this is crappy
+
+STANDARD_COMMANDS = [
+    # packaging
+    'check', 'test',
+    # building
+    'build', 'build_py', 'build_ext', 'build_clib', 'build_scripts', 'clean',
+    # installing
+    'install_dist', 'install_lib', 'install_headers', 'install_scripts',
+    'install_data', 'install_distinfo',
+    # distributing
+    'sdist', 'bdist', 'bdist_dumb', 'bdist_wininst',
+    'register', 'upload', 'upload_docs',
+    ]
+
 if os.name == 'nt':
-    _COMMANDS['bdist_msi'] = 'packaging.command.bdist_msi.bdist_msi'
+    STANDARD_COMMANDS.insert(STANDARD_COMMANDS.index('bdist_wininst'),
+                             'bdist_msi')
 
-# XXX use OrderedDict to preserve the grouping (build-related, install-related,
-# distribution-related)
-STANDARD_COMMANDS = set(_COMMANDS)
+# XXX maybe we need more than one registry, so that --list-comands can display
+# standard, custom and overriden standard commands differently
+_COMMANDS = dict((name, 'packaging.command.%s.%s' % (name, name))
+                 for name in STANDARD_COMMANDS)
 
 
 def get_command_names():
diff --git a/Lib/packaging/compat.py b/Lib/packaging/compat.py
index dcb58f5..bfce92d 100644
--- a/Lib/packaging/compat.py
+++ b/Lib/packaging/compat.py
@@ -1,4 +1,4 @@
-"""Compatibility helpers."""
+"""Support for build-time 2to3 conversion."""
 
 from packaging import logger
 
@@ -25,7 +25,7 @@ class Mixin2to3(_KLASS):
     """
     if _CONVERT:
 
-        def _run_2to3(self, files, doctests=[], fixers=[]):
+        def _run_2to3(self, files=[], doctests=[], fixers=[]):
             """ Takes a list of files and doctests, and performs conversion
             on those.
               - First, the files which contain the code(`files`) are converted.
@@ -35,17 +35,16 @@ class Mixin2to3(_KLASS):
             if fixers:
                 self.fixer_names = fixers
 
-            logger.info('converting Python code')
-            _KLASS.run_2to3(self, files)
+            if files:
+                logger.info('converting Python code and doctests')
+                _KLASS.run_2to3(self, files)
+                _KLASS.run_2to3(self, files, doctests_only=True)
 
-            logger.info('converting doctests in Python files')
-            _KLASS.run_2to3(self, files, doctests_only=True)
-
-            if doctests != []:
-                logger.info('converting doctest in text files')
+            if doctests:
+                logger.info('converting doctests in text files')
                 _KLASS.run_2to3(self, doctests, doctests_only=True)
     else:
         # If run on Python 2.x, there is nothing to do.
 
-        def _run_2to3(self, files, doctests=[], fixers=[]):
+        def _run_2to3(self, files=[], doctests=[], fixers=[]):
             pass
diff --git a/Lib/packaging/run.py b/Lib/packaging/run.py
index 4756f7c..c3600a7 100644
--- a/Lib/packaging/run.py
+++ b/Lib/packaging/run.py
@@ -254,16 +254,13 @@ def _run(dispatcher, args, **kw):
     parser = dispatcher.parser
     args = args[1:]
 
-    commands = STANDARD_COMMANDS  # + extra commands
+    commands = STANDARD_COMMANDS  # FIXME display extra commands
 
     if args == ['--list-commands']:
         print('List of available commands:')
-        cmds = sorted(commands)
-
-        for cmd in cmds:
+        for cmd in commands:
             cls = dispatcher.cmdclass.get(cmd) or get_command_class(cmd)
-            desc = getattr(cls, 'description',
-                            '(no description available)')
+            desc = getattr(cls, 'description', '(no description available)')
             print('  %s: %s' % (cmd, desc))
         return
 
diff --git a/Lib/packaging/tests/fixer/fix_echo.py b/Lib/packaging/tests/fixer/fix_echo.py
new file mode 100644
index 0000000..8daae3e
--- /dev/null
+++ b/Lib/packaging/tests/fixer/fix_echo.py
@@ -0,0 +1,16 @@
+# Example custom fixer, derived from fix_raw_input by Andre Roberge
+
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name
+
+
+class FixEcho(fixer_base.BaseFix):
+
+    BM_compatible = True
+    PATTERN = """
+              power< name='echo' trailer< '(' [any] ')' > any* >
+              """
+
+    def transform(self, node, results):
+        name = results['name']
+        name.replace(Name('print', prefix=name.prefix))
diff --git a/Lib/packaging/tests/fixer/fix_echo2.py b/Lib/packaging/tests/fixer/fix_echo2.py
new file mode 100644
index 0000000..1b92891
--- /dev/null
+++ b/Lib/packaging/tests/fixer/fix_echo2.py
@@ -0,0 +1,16 @@
+# Example custom fixer, derived from fix_raw_input by Andre Roberge
+
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import Name
+
+
+class FixEcho2(fixer_base.BaseFix):
+
+    BM_compatible = True
+    PATTERN = """
+              power< name='echo2' trailer< '(' [any] ')' > any* >
+              """
+
+    def transform(self, node, results):
+        name = results['name']
+        name.replace(Name('print', prefix=name.prefix))
diff --git a/Lib/packaging/tests/fixer/fix_idioms.py b/Lib/packaging/tests/fixer/fix_idioms.py
deleted file mode 100644
index 64f5ea0..0000000
--- a/Lib/packaging/tests/fixer/fix_idioms.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""Adjust some old Python 2 idioms to their modern counterparts.
-
-* Change some type comparisons to isinstance() calls:
-    type(x) == T -> isinstance(x, T)
-    type(x) is T -> isinstance(x, T)
-    type(x) != T -> not isinstance(x, T)
-    type(x) is not T -> not isinstance(x, T)
-
-* Change "while 1:" into "while True:".
-
-* Change both
-
-    v = list(EXPR)
-    v.sort()
-    foo(v)
-
-and the more general
-
-    v = EXPR
-    v.sort()
-    foo(v)
-
-into
-
-    v = sorted(EXPR)
-    foo(v)
-"""
-# Author: Jacques Frechet, Collin Winter
-
-# Local imports
-from lib2to3 import fixer_base
-from lib2to3.fixer_util import Call, Comma, Name, Node, syms
-
-CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)"
-TYPE = "power< 'type' trailer< '(' x=any ')' > >"
-
-class FixIdioms(fixer_base.BaseFix):
-
-    explicit = False # The user must ask for this fixer
-
-    PATTERN = r"""
-        isinstance=comparison< %s %s T=any >
-        |
-        isinstance=comparison< T=any %s %s >
-        |
-        while_stmt< 'while' while='1' ':' any+ >
-        |
-        sorted=any<
-            any*
-            simple_stmt<
-              expr_stmt< id1=any '='
-                         power< list='list' trailer< '(' (not arglist<any+>) any ')' > >
-              >
-              '\n'
-            >
-            sort=
-            simple_stmt<
-              power< id2=any
-                     trailer< '.' 'sort' > trailer< '(' ')' >
-              >
-              '\n'
-            >
-            next=any*
-        >
-        |
-        sorted=any<
-            any*
-            simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' >
-            sort=
-            simple_stmt<
-              power< id2=any
-                     trailer< '.' 'sort' > trailer< '(' ')' >
-              >
-              '\n'
-            >
-            next=any*
-        >
-    """ % (TYPE, CMP, CMP, TYPE)
-
-    def match(self, node):
-        r = super(FixIdioms, self).match(node)
-        # If we've matched one of the sort/sorted subpatterns above, we
-        # want to reject matches where the initial assignment and the
-        # subsequent .sort() call involve different identifiers.
-        if r and "sorted" in r:
-            if r["id1"] == r["id2"]:
-                return r
-            return None
-        return r
-
-    def transform(self, node, results):
-        if "isinstance" in results:
-            return self.transform_isinstance(node, results)
-        elif "while" in results:
-            return self.transform_while(node, results)
-        elif "sorted" in results:
-            return self.transform_sort(node, results)
-        else:
-            raise RuntimeError("Invalid match")
-
-    def transform_isinstance(self, node, results):
-        x = results["x"].clone() # The thing inside of type()
-        T = results["T"].clone() # The type being compared against
-        x.prefix = ""
-        T.prefix = " "
-        test = Call(Name("isinstance"), [x, Comma(), T])
-        if "n" in results:
-            test.prefix = " "
-            test = Node(syms.not_test, [Name("not"), test])
-        test.prefix = node.prefix
-        return test
-
-    def transform_while(self, node, results):
-        one = results["while"]
-        one.replace(Name("True", prefix=one.prefix))
-
-    def transform_sort(self, node, results):
-        sort_stmt = results["sort"]
-        next_stmt = results["next"]
-        list_call = results.get("list")
-        simple_expr = results.get("expr")
-
-        if list_call:
-            list_call.replace(Name("sorted", prefix=list_call.prefix))
-        elif simple_expr:
-            new = simple_expr.clone()
-            new.prefix = ""
-            simple_expr.replace(Call(Name("sorted"), [new],
-                                     prefix=simple_expr.prefix))
-        else:
-            raise RuntimeError("should not have reached here")
-        sort_stmt.remove()
-        if next_stmt:
-            next_stmt[0].prefix = sort_stmt._prefix
diff --git a/Lib/packaging/tests/support.py b/Lib/packaging/tests/support.py
index 4848bcc..06f06c9 100644
--- a/Lib/packaging/tests/support.py
+++ b/Lib/packaging/tests/support.py
@@ -56,8 +56,9 @@ __all__ = [
     # misc. functions and decorators
     'fake_dec', 'create_distribution', 'use_command',
     'copy_xxmodule_c', 'fixup_build_ext',
+    'skip_2to3_optimize',
     # imported from this module for backport purposes
-    'unittest', 'requires_zlib', 'skip_2to3_optimize', 'skip_unless_symlink',
+    'unittest', 'requires_zlib', 'skip_unless_symlink',
 ]
 
 
@@ -332,22 +333,18 @@ def copy_xxmodule_c(directory):
     """
     filename = _get_xxmodule_path()
     if filename is None:
-        raise unittest.SkipTest('cannot find xxmodule.c (test must run in '
-                                'the python build dir)')
+        raise unittest.SkipTest('cannot find xxmodule.c')
     shutil.copy(filename, directory)
 
 
 def _get_xxmodule_path():
-    srcdir = sysconfig.get_config_var('srcdir')
-    candidates = [
-        # use installed copy if available
-        os.path.join(os.path.dirname(__file__), 'xxmodule.c'),
-        # otherwise try using copy from build directory
-        os.path.join(srcdir, 'Modules', 'xxmodule.c'),
-    ]
-    for path in candidates:
-        if os.path.exists(path):
-            return path
+    if sysconfig.is_python_build():
+        srcdir = sysconfig.get_config_var('projectbase')
+        path = os.path.join(os.getcwd(), srcdir, 'Modules', 'xxmodule.c')
+    else:
+        os.path.join(os.path.dirname(__file__), 'xxmodule.c')
+    if os.path.exists(path):
+        return path
 
 
 def fixup_build_ext(cmd):
@@ -355,20 +352,21 @@ def fixup_build_ext(cmd):
 
     When Python was built with --enable-shared on Unix, -L. is not enough to
     find libpython<blah>.so, because regrtest runs in a tempdir, not in the
-    source directory where the .so lives.
+    source directory where the .so lives.  (Mac OS X embeds absolute paths
+    to shared libraries into executables, so the fixup is a no-op on that
+    platform.)
 
     When Python was built with in debug mode on Windows, build_ext commands
     need their debug attribute set, and it is not done automatically for
     some reason.
 
-    This function handles both of these things.  Example use:
+    This function handles both of these things, and also fixes
+    cmd.distribution.include_dirs if the running Python is an uninstalled
+    build.  Example use:
 
         cmd = build_ext(dist)
         support.fixup_build_ext(cmd)
         cmd.ensure_finalized()
-
-    Unlike most other Unix platforms, Mac OS X embeds absolute paths
-    to shared libraries into executables, so the fixup is not needed there.
     """
     if os.name == 'nt':
         cmd.debug = sys.executable.endswith('_d.exe')
@@ -386,12 +384,17 @@ def fixup_build_ext(cmd):
                 name, equals, value = runshared.partition('=')
                 cmd.library_dirs = value.split(os.pathsep)
 
+    # Allow tests to run with an uninstalled Python
+    if sysconfig.is_python_build():
+        pysrcdir = sysconfig.get_config_var('projectbase')
+        cmd.distribution.include_dirs.append(os.path.join(pysrcdir, 'Include'))
+
+
 try:
     from test.support import skip_unless_symlink
 except ImportError:
     skip_unless_symlink = unittest.skip(
         'requires test.support.skip_unless_symlink')
 
-
 skip_2to3_optimize = unittest.skipIf(sys.flags.optimize,
                                      "2to3 doesn't work under -O")
diff --git a/Lib/packaging/tests/test_markers.py b/Lib/packaging/tests/test_markers.py
index dec0429..a494c6b 100644
--- a/Lib/packaging/tests/test_markers.py
+++ b/Lib/packaging/tests/test_markers.py
@@ -21,8 +21,6 @@ class MarkersTestCase(LoggingCatcher,
 
         self.assertTrue(interpret("sys.platform == '%s'" % sys_platform))
         self.assertTrue(interpret(
-            "sys.platform == '%s' or python_version == '2.4'" % sys_platform))
-        self.assertTrue(interpret(
             "sys.platform == '%s' and python_full_version == '%s'" %
             (sys_platform, version)))
         self.assertTrue(interpret("'%s' == sys.platform" % sys_platform))
@@ -41,12 +39,18 @@ class MarkersTestCase(LoggingCatcher,
 
         # combined operations
         OP = 'os.name == "%s"' % os_name
+        FALSEOP = 'os.name == "buuuu"'
         AND = ' and '
         OR = ' or '
         self.assertTrue(interpret(OP + AND + OP))
         self.assertTrue(interpret(OP + AND + OP + AND + OP))
         self.assertTrue(interpret(OP + OR + OP))
-        self.assertTrue(interpret(OP + OR + OP + OR + OP))
+        self.assertTrue(interpret(OP + OR + FALSEOP))
+        self.assertTrue(interpret(OP + OR + OP + OR + FALSEOP))
+        self.assertTrue(interpret(OP + OR + FALSEOP + OR + FALSEOP))
+        self.assertTrue(interpret(FALSEOP + OR + OP))
+        self.assertFalse(interpret(FALSEOP + AND + FALSEOP))
+        self.assertFalse(interpret(FALSEOP + OR + FALSEOP))
 
         # other operators
         self.assertTrue(interpret("os.name != 'buuuu'"))
diff --git a/Lib/packaging/tests/test_mixin2to3.py b/Lib/packaging/tests/test_mixin2to3.py
index c439bcb..08a102b 100644
--- a/Lib/packaging/tests/test_mixin2to3.py
+++ b/Lib/packaging/tests/test_mixin2to3.py
@@ -8,70 +8,76 @@ class Mixin2to3TestCase(support.TempdirManager,
                         support.LoggingCatcher,
                         unittest.TestCase):
 
-    @support.skip_2to3_optimize
-    def test_convert_code_only(self):
-        # used to check if code gets converted properly.
-        code = "print 'test'"
+    def setUp(self):
+        super(Mixin2to3TestCase, self).setUp()
+        self.filename = self.mktempfile().name
 
-        with self.mktempfile() as fp:
-            fp.write(code)
+    def check(self, source, wanted, **kwargs):
+        source = textwrap.dedent(source)
+        with open(self.filename, 'w') as fp:
+            fp.write(source)
 
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3([fp.name])
-        expected = "print('test')"
+        Mixin2to3()._run_2to3(**kwargs)
 
-        with open(fp.name) as fp:
+        wanted = textwrap.dedent(wanted)
+        with open(self.filename) as fp:
             converted = fp.read()
+        self.assertMultiLineEqual(converted, wanted)
 
-        self.assertEqual(expected, converted)
-
-    def test_doctests_only(self):
-        # used to check if doctests gets converted properly.
-        doctest = textwrap.dedent('''\
+    def test_conversion(self):
+        # check that code and doctests get converted
+        self.check('''\
             """Example docstring.
 
             >>> print test
             test
 
             It works.
-            """''')
-
-        with self.mktempfile() as fp:
-            fp.write(doctest)
-
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3([fp.name])
-        expected = textwrap.dedent('''\
+            """
+            print 'test'
+            ''',
+            '''\
             """Example docstring.
 
             >>> print(test)
             test
 
             It works.
-            """\n''')
+            """
+            print('test')
 
-        with open(fp.name) as fp:
-            converted = fp.read()
-
-        self.assertEqual(expected, converted)
-
-    def test_additional_fixers(self):
-        # used to check if use_2to3_fixers works
-        code = 'type(x) is not T'
+            ''',  # 2to3 adds a newline here
+            files=[self.filename])
 
-        with self.mktempfile() as fp:
-            fp.write(code)
+    def test_doctests_conversion(self):
+        # check that doctest files are converted
+        self.check('''\
+            Welcome to the doc.
 
-        mixin2to3 = Mixin2to3()
-        mixin2to3._run_2to3(files=[fp.name], doctests=[fp.name],
-                            fixers=['packaging.tests.fixer'])
+            >>> print test
+            test
+            ''',
+            '''\
+            Welcome to the doc.
 
-        expected = 'not isinstance(x, T)'
+            >>> print(test)
+            test
 
-        with open(fp.name) as fp:
-            converted = fp.read()
+            ''',
+            doctests=[self.filename])
 
-        self.assertEqual(expected, converted)
+    def test_additional_fixers(self):
+        # make sure the fixers argument works
+        self.check("""\
+            echo('42')
+            echo2('oh no')
+            """,
+            """\
+            print('42')
+            print('oh no')
+            """,
+            files=[self.filename],
+            fixers=['packaging.tests.fixer'])
 
 
 def test_suite():
diff --git a/Lib/packaging/tests/test_run.py b/Lib/packaging/tests/test_run.py
index 84b9bf6..14e7b07 100644
--- a/Lib/packaging/tests/test_run.py
+++ b/Lib/packaging/tests/test_run.py
@@ -67,6 +67,23 @@ class RunTestCase(support.TempdirManager,
         self.assertGreater(out, b'')
         self.assertEqual(err, b'')
 
+    def test_list_commands(self):
+        status, out, err = assert_python_ok('-m', 'packaging.run', 'run',
+                                            '--list-commands')
+        # check that something is displayed
+        self.assertEqual(status, 0)
+        self.assertGreater(out, b'')
+        self.assertEqual(err, b'')
+
+        # make sure the manual grouping of commands is respected
+        check_position = out.find(b'  check: ')
+        build_position = out.find(b'  build: ')
+        self.assertTrue(check_position, out)  # "out" printed as debugging aid
+        self.assertTrue(build_position, out)
+        self.assertLess(check_position, build_position, out)
+
+        # TODO test that custom commands don't break --list-commands
+
 
 def test_suite():
     return unittest.makeSuite(RunTestCase)
diff --git a/Lib/packaging/util.py b/Lib/packaging/util.py
index 5b651b1..a1f6782 100644
--- a/Lib/packaging/util.py
+++ b/Lib/packaging/util.py
@@ -853,13 +853,11 @@ def run_2to3(files, doctests_only=False, fixer_names=None,
 
     # Make this class local, to delay import of 2to3
     from lib2to3.refactor import get_fixers_from_package, RefactoringTool
-    fixers = []
     fixers = get_fixers_from_package('lib2to3.fixes')
 
     if fixer_names:
         for fixername in fixer_names:
-            fixers.extend(fixer for fixer in
-                          get_fixers_from_package(fixername))
+            fixers.extend(get_fixers_from_package(fixername))
     r = RefactoringTool(fixers, options=options)
     r.refactor(files, write=True, doctests_only=doctests_only)
 
@@ -870,21 +868,23 @@ class Mixin2to3:
     the class variables, or inherit from this class
     to override how 2to3 is invoked.
     """
-    # provide list of fixers to run.
-    # defaults to all from lib2to3.fixers
+    # list of fixers to run; defaults to all implicit from lib2to3.fixers
     fixer_names = None
-
-    # options dictionary
+    # dict of options
     options = None
-
-    # list of fixers to invoke even though they are marked as explicit
+    # list of extra fixers to invoke
     explicit = None
+    # TODO need a better way to add just one fixer from a package
+    # TODO need a way to exclude individual fixers
 
     def run_2to3(self, files, doctests_only=False):
         """ Issues a call to util.run_2to3. """
         return run_2to3(files, doctests_only, self.fixer_names,
                         self.options, self.explicit)
 
+    # TODO provide initialize/finalize_options
+
+
 RICH_GLOB = re.compile(r'\{([^}]*)\}')
 _CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]')
 _CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$')
@@ -1049,7 +1049,6 @@ def cfg_to_args(path='setup.cfg'):
 
 SETUP_TEMPLATE = """\
 # This script was automatically generated by packaging
-import os
 import codecs
 from distutils.core import setup
 try:
@@ -1057,6 +1056,7 @@ try:
 except ImportError:
     from configparser import RawConfigParser
 
+
 %(split_multiline)s
 
 %(cfg_to_args)s