diff options
author | Nick Coghlan <ncoghlan@gmail.com> | 2012-07-15 11:19:18 (GMT) |
---|---|---|
committer | Nick Coghlan <ncoghlan@gmail.com> | 2012-07-15 11:19:18 (GMT) |
commit | 8ecf50474ce3d0ef34fbcce940566c70370e57ad (patch) | |
tree | 51f2817b0238f406b5e673e703996ec7bc2a8c8b | |
parent | 3f94cbf9eba7adef027cfc5d087b3660800df9d7 (diff) | |
download | cpython-8ecf50474ce3d0ef34fbcce940566c70370e57ad.zip cpython-8ecf50474ce3d0ef34fbcce940566c70370e57ad.tar.gz cpython-8ecf50474ce3d0ef34fbcce940566c70370e57ad.tar.bz2 |
Issue #15343: Handle importlib.machinery.FileFinder instances in pkgutil.walk_packages (et al)
-rw-r--r-- | Doc/library/pkgutil.rst | 35 | ||||
-rw-r--r-- | Lib/pkgutil.py | 43 | ||||
-rw-r--r-- | Lib/test/test_pkgutil.py | 6 | ||||
-rw-r--r-- | Lib/test/test_runpy.py | 37 | ||||
-rw-r--r-- | Misc/NEWS | 4 |
5 files changed, 106 insertions, 19 deletions
diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index bcd5d91..22d44eb 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -81,7 +81,7 @@ support. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying - on a package internal PEP 302 import emulation. + on the package internal PEP 302 import emulation. .. function:: get_importer(path_item) @@ -96,7 +96,7 @@ support. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying - on a package internal PEP 302 import emulation. + on the package internal PEP 302 import emulation. .. function:: get_loader(module_or_name) @@ -115,7 +115,7 @@ support. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying - on a package internal PEP 302 import emulation. + on the package internal PEP 302 import emulation. .. function:: iter_importers(fullname='') @@ -133,12 +133,12 @@ support. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying - on a package internal PEP 302 import emulation. + on the package internal PEP 302 import emulation. .. function:: iter_modules(path=None, prefix='') - Yields ``(module_loader, name, ispkg)`` for all submodules on *path*, or, if + Yields ``(module_finder, name, ispkg)`` for all submodules on *path*, or, if path is ``None``, all top-level modules on ``sys.path``. *path* should be either ``None`` or a list of paths to look for modules in. @@ -146,19 +146,19 @@ support. *prefix* is a string to output on the front of every module name on output. .. note:: - Only works with a :term:`finder` which defines an ``iter_modules()`` - method, which is non-standard but implemented by classes defined in this - module. + Only works for a :term:`finder` which defines an ``iter_modules()`` + method. This interface is non-standard, so the module also provides + implementations for :class:`importlib.machinery.FileFinder` and + :class:`zipimport.zipimporter`. .. versionchanged:: 3.3 - As of Python 3.3, the import system provides finders by default, but they - do not include the non-standard ``iter_modules()`` method required by this - function. + Updated to be based directly on :mod:`importlib` rather than relying + on the package internal PEP 302 import emulation. .. function:: walk_packages(path=None, prefix='', onerror=None) - Yields ``(module_loader, name, ispkg)`` for all modules recursively on + Yields ``(module_finder, name, ispkg)`` for all modules recursively on *path*, or, if path is ``None``, all accessible modules. *path* should be either ``None`` or a list of paths to look for modules in. @@ -184,13 +184,14 @@ support. walk_packages(ctypes.__path__, ctypes.__name__ + '.') .. note:: - Only works for a :term:`finder` which define an ``iter_modules()`` method, - which is non-standard but implemented by classes defined in this module. + Only works for a :term:`finder` which defines an ``iter_modules()`` + method. This interface is non-standard, so the module also provides + implementations for :class:`importlib.machinery.FileFinder` and + :class:`zipimport.zipimporter`. .. versionchanged:: 3.3 - As of Python 3.3, the import system provides finders by default, but they - do not include the non-standard ``iter_modules()`` method required by this - function. + Updated to be based directly on :mod:`importlib` rather than relying + on the package internal PEP 302 import emulation. .. function:: get_data(package, resource) diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 487c8cd..8407b6d 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -157,6 +157,49 @@ def iter_importer_modules(importer, prefix=''): iter_importer_modules = simplegeneric(iter_importer_modules) +# Implement a file walker for the normal importlib path hook +def _iter_file_finder_modules(importer, prefix=''): + if importer.path is None or not os.path.isdir(importer.path): + return + + yielded = {} + import inspect + try: + filenames = os.listdir(importer.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + modname = inspect.getmodulename(fn) + if modname=='__init__' or modname in yielded: + continue + + path = os.path.join(importer.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + for fn in dircontents: + subname = inspect.getmodulename(fn) + if subname=='__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + +iter_importer_modules.register( + importlib.machinery.FileFinder, _iter_file_finder_modules) class ImpImporter: """PEP 302 Importer that wraps Python's "classic" import algorithm diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 51f5dee..73c6869 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -9,7 +9,11 @@ import tempfile import shutil import zipfile - +# Note: pkgutil.walk_packages is currently tested in test_runpy. This is +# a hack to get a major issue resolved for 3.3b2. Longer term, it should +# be moved back here, perhaps by factoring out the helper code for +# creating interesting package layouts to a separate module. +# Issue #15348 declares this is indeed a dodgy hack ;) class PkgutilTests(unittest.TestCase): diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index c39e281..abb7dd9 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -13,6 +13,7 @@ from test.support import ( from test.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir) + import runpy from runpy import _run_code, _run_module_code, run_module, run_path # Note: This module can't safely test _run_module_as_main as it @@ -148,7 +149,7 @@ class ExecutionLayerTestCase(unittest.TestCase, CodeExecutionMixin): mod_package) self.check_code_execution(create_ns, expected_ns) - +# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin): """Unit tests for runpy.run_module""" @@ -413,6 +414,40 @@ from ..uncle.cousin import nephew finally: self._del_pkg(pkg_dir, depth, mod_name) + def test_pkgutil_walk_packages(self): + # This is a dodgy hack to use the test_runpy infrastructure to test + # issue #15343. Issue #15348 declares this is indeed a dodgy hack ;) + import pkgutil + max_depth = 4 + base_name = "__runpy_pkg__" + package_suffixes = ["uncle", "uncle.cousin"] + module_suffixes = ["uncle.cousin.nephew", base_name + ".sibling"] + expected_packages = set() + expected_modules = set() + for depth in range(1, max_depth): + pkg_name = ".".join([base_name] * depth) + expected_packages.add(pkg_name) + for name in package_suffixes: + expected_packages.add(pkg_name + "." + name) + for name in module_suffixes: + expected_modules.add(pkg_name + "." + name) + pkg_name = ".".join([base_name] * max_depth) + expected_packages.add(pkg_name) + expected_modules.add(pkg_name + ".runpy_test") + pkg_dir, mod_fname, mod_name = ( + self._make_pkg("", max_depth)) + self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name) + for depth in range(2, max_depth+1): + self._add_relative_modules(pkg_dir, "", depth) + for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]): + self.assertIsInstance(finder, + importlib.machinery.FileFinder) + if ispkg: + expected_packages.remove(mod_name) + else: + expected_modules.remove(mod_name) + self.assertEqual(len(expected_packages), 0, expected_packages) + self.assertEqual(len(expected_modules), 0, expected_modules) class RunPathTestCase(unittest.TestCase, CodeExecutionMixin): """Unit tests for runpy.run_path""" @@ -38,6 +38,10 @@ Core and Builtins Library ------- +- Issue #15343: pkgutil now includes an iter_importer_modules implementation + for importlib.machinery.FileFinder (similar to the way it already handled + zipimport.zipimporter) + - Issue #15314: runpy now sets __main__.__loader__ correctly - Issue #15357: The import emulation in pkgutil is now deprecated. pkgutil |