diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/pkgutil.py | 94 | ||||
| -rw-r--r-- | Lib/runpy.py | 56 | ||||
| -rw-r--r-- | Lib/test/test_cmd_line_script.py | 73 | ||||
| -rw-r--r-- | Lib/test/test_pkgutil.py | 43 |
4 files changed, 159 insertions, 107 deletions
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 59c99bf..487c8cd 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -3,7 +3,9 @@ import os import sys import imp +import importlib import os.path +from warnings import warn from types import ModuleType __all__ = [ @@ -168,6 +170,8 @@ class ImpImporter: """ def __init__(self, path=None): + warn("This emulation is deprecated, use 'importlib' instead", + DeprecationWarning) self.path = path def find_module(self, fullname, path=None): @@ -232,6 +236,8 @@ class ImpLoader: code = source = None def __init__(self, fullname, file, filename, etc): + warn("This emulation is deprecated, use 'importlib' instead", + DeprecationWarning) self.file = file self.filename = filename self.fullname = fullname @@ -366,10 +372,6 @@ def get_importer(path_item): The returned importer is cached in sys.path_importer_cache if it was newly created by a path hook. - If there is no importer, a wrapper around the basic import - machinery is returned. This wrapper is never inserted into - the importer cache (None is inserted instead). - The cache (or part of it) can be cleared manually if a rescan of sys.path_hooks is necessary. """ @@ -384,10 +386,7 @@ def get_importer(path_item): except ImportError: pass else: - try: - importer = ImpImporter(path_item) - except ImportError: - importer = None + importer = None return importer @@ -395,55 +394,37 @@ def iter_importers(fullname=""): """Yield PEP 302 importers for the given module name If fullname contains a '.', the importers will be for the package - containing fullname, otherwise they will be importers for sys.meta_path, - sys.path, and Python's "classic" import machinery, in that order. If - the named module is in a package, that package is imported as a side - effect of invoking this function. + containing fullname, otherwise they will be all registered top level + importers (i.e. those on both sys.meta_path and sys.path_hooks). - Non PEP 302 mechanisms (e.g. the Windows registry) used by the - standard import machinery to find files in alternative locations - are partially supported, but are searched AFTER sys.path. Normally, - these locations are searched BEFORE sys.path, preventing sys.path - entries from shadowing them. - - For this to cause a visible difference in behaviour, there must - be a module or package name that is accessible via both sys.path - and one of the non PEP 302 file system mechanisms. In this case, - the emulation will find the former version, while the builtin - import mechanism will find the latter. + If the named module is in a package, that package is imported as a side + effect of invoking this function. - Items of the following types can be affected by this discrepancy: - imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY + If no module name is specified, all top level importers are produced. """ if fullname.startswith('.'): - raise ImportError("Relative module names not supported") + msg = "Relative module name {!r} not supported".format(fullname) + raise ImportError(msg) if '.' in fullname: # Get the containing package's __path__ - pkg = '.'.join(fullname.split('.')[:-1]) - if pkg not in sys.modules: - __import__(pkg) - path = getattr(sys.modules[pkg], '__path__', None) or [] + pkg_name = fullname.rpartition(".")[0] + pkg = importlib.import_module(pkg) + path = getattr(sys.modules[pkg], '__path__', None) + if path is None: + return else: for importer in sys.meta_path: yield importer path = sys.path for item in path: yield get_importer(item) - if '.' not in fullname: - yield ImpImporter() def get_loader(module_or_name): """Get a PEP 302 "loader" object for module_or_name - If the module or package is accessible via the normal import - mechanism, a wrapper around the relevant part of that machinery - is returned. Returns None if the module cannot be found or imported. + Returns None if the module cannot be found or imported. If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. - - This function uses iter_importers(), and is thus subject to the same - limitations regarding platform-specific special import locations such - as the Windows registry. """ if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] @@ -457,22 +438,33 @@ def get_loader(module_or_name): fullname = module_or_name return find_loader(fullname) + def find_loader(fullname): """Find a PEP 302 "loader" object for fullname - If fullname contains dots, path must be the containing package's __path__. - Returns None if the module cannot be found or imported. This function uses - iter_importers(), and is thus subject to the same limitations regarding - platform-specific special import locations such as the Windows registry. + This is s convenience wrapper around :func:`importlib.find_loader` that + sets the *path* argument correctly when searching for submodules, and + also ensures parent packages (if any) are imported before searching for + submodules. """ - for importer in iter_importers(fullname): - if importer is None: - continue - loader = importer.find_module(fullname) - if loader is not None: - return loader - - return None + if fullname.startswith('.'): + msg = "Relative module name {!r} not supported".format(fullname) + raise ImportError(msg) + path = None + pkg_name = fullname.rpartition(".")[0] + if pkg_name: + pkg = importlib.import_module(pkg_name) + path = getattr(pkg, "__path__", None) + if path is None: + return None + try: + return importlib.find_loader(fullname, path) + except (ImportError, AttributeError, TypeError, ValueError) as ex: + # This hack fixes an impedance mismatch between pkgutil and + # importlib, where the latter throws other errors for cases where + # pkgutil previously threw ImportError + msg = "Error while finding loader for {!r} ({}: {})" + raise ImportError(msg.format(fullname, type(ex), ex)) from ex def extend_path(path, name): diff --git a/Lib/runpy.py b/Lib/runpy.py index 8ca325b..5823943 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -13,11 +13,8 @@ importers when locating support scripts as well as when importing modules. import os import sys import imp -from pkgutil import read_code -try: - from imp import get_loader -except ImportError: - from pkgutil import get_loader +import importlib.machinery +from pkgutil import read_code, get_loader, get_importer __all__ = [ "run_module", "run_path", @@ -154,6 +151,7 @@ def _run_module_as_main(mod_name, alter_argv=True): # know what the code was looking for info = "can't find '__main__' module in %r" % sys.argv[0] msg = "%s: %s" % (sys.executable, info) + raise sys.exit(msg) pkg_name = mod_name.rpartition('.')[0] main_globals = sys.modules["__main__"].__dict__ @@ -183,36 +181,23 @@ def run_module(mod_name, init_globals=None, def _get_main_module_details(): # Helper that gives a nicer error message when attempting to # execute a zipfile or directory by invoking __main__.py + # Also moves the standard __main__ out of the way so that the + # preexisting __loader__ entry doesn't cause issues main_name = "__main__" + saved_main = sys.modules[main_name] + del sys.modules[main_name] try: return _get_module_details(main_name) except ImportError as exc: if main_name in str(exc): raise ImportError("can't find %r module in %r" % - (main_name, sys.path[0])) + (main_name, sys.path[0])) from exc raise + finally: + sys.modules[main_name] = saved_main -# XXX (ncoghlan): Perhaps expose the C API function -# as imp.get_importer instead of reimplementing it in Python? -def _get_importer(path_name): - """Python version of PyImport_GetImporter C API function""" - cache = sys.path_importer_cache - try: - importer = cache[path_name] - except KeyError: - for hook in sys.path_hooks: - try: - importer = hook(path_name) - break - except ImportError: - pass - else: - importer = None - cache[path_name] = importer - return importer - -def _get_code_from_file(fname): +def _get_code_from_file(run_name, fname): # Check for a compiled file first with open(fname, "rb") as f: code = read_code(f) @@ -220,7 +205,10 @@ def _get_code_from_file(fname): # That didn't work, so try it as normal source code with open(fname, "rb") as f: code = compile(f.read(), fname, 'exec') - return code + loader = importlib.machinery.SourceFileLoader(run_name, fname) + else: + loader = importlib.machinery.SourcelessFileLoader(run_name, fname) + return code, loader def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location @@ -235,13 +223,13 @@ def run_path(path_name, init_globals=None, run_name=None): if run_name is None: run_name = "<run_path>" pkg_name = run_name.rpartition(".")[0] - importer = _get_importer(path_name) + importer = get_importer(path_name) if isinstance(importer, (type(None), imp.NullImporter)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code, mod_loader = _get_code_from_file(run_name, path_name) return _run_module_code(code, init_globals, run_name, path_name, - pkg_name=pkg_name) + mod_loader, pkg_name) else: # Importer is defined for path, so add it to # the start of sys.path @@ -253,13 +241,7 @@ def run_path(path_name, init_globals=None, run_name=None): # have no choice and we have to remove it even while we read the # code. If we don't do this, a __loader__ attribute in the # existing __main__ module may prevent location of the new module. - main_name = "__main__" - saved_main = sys.modules[main_name] - del sys.modules[main_name] - try: - mod_name, loader, code, fname = _get_main_module_details() - finally: - sys.modules[main_name] = saved_main + mod_name, loader, code, fname = _get_main_module_details() with _TempModule(run_name) as temp_module, \ _ModifiedArgv0(path_name): mod_globals = temp_module.module.__dict__ diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d8bc263..a08cabd 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -1,6 +1,8 @@ # tests command line execution of scripts import importlib +import importlib.machinery +import zipimport import unittest import sys import os @@ -11,7 +13,8 @@ import textwrap from test import support from test.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, - assert_python_ok, assert_python_failure, temp_dir) + assert_python_ok, assert_python_failure, temp_dir, + spawn_python, kill_python) verbose = support.verbose @@ -34,6 +37,8 @@ f() assertEqual(result, ['Top level assignment', 'Lower level reference']) # Check population of magic variables assertEqual(__name__, '__main__') +_loader = __loader__ if isinstance(__loader__, type) else type(__loader__) +print('__loader__==%a' % _loader) print('__file__==%a' % __file__) assertEqual(__cached__, None) print('__package__==%r' % __package__) @@ -85,11 +90,13 @@ def _make_launch_script(script_dir, script_basename, module_name, path=None): class CmdLineTest(unittest.TestCase): def _check_output(self, script_name, exit_code, data, expected_file, expected_argv0, - expected_path0, expected_package): + expected_path0, expected_package, + expected_loader): if verbose > 1: print("Output from test script %r:" % script_name) print(data) self.assertEqual(exit_code, 0) + printed_loader = '__loader__==%a' % expected_loader printed_file = '__file__==%a' % expected_file printed_package = '__package__==%r' % expected_package printed_argv0 = 'sys.argv[0]==%a' % expected_argv0 @@ -101,6 +108,7 @@ class CmdLineTest(unittest.TestCase): print(printed_package) print(printed_argv0) print(printed_cwd) + self.assertIn(printed_loader.encode('utf-8'), data) self.assertIn(printed_file.encode('utf-8'), data) self.assertIn(printed_package.encode('utf-8'), data) self.assertIn(printed_argv0.encode('utf-8'), data) @@ -109,14 +117,15 @@ class CmdLineTest(unittest.TestCase): def _check_script(self, script_name, expected_file, expected_argv0, expected_path0, - expected_package, + expected_package, expected_loader, *cmd_line_switches): if not __debug__: cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) run_args = cmd_line_switches + (script_name,) + tuple(example_args) rc, out, err = assert_python_ok(*run_args) self._check_output(script_name, rc, out + err, expected_file, - expected_argv0, expected_path0, expected_package) + expected_argv0, expected_path0, + expected_package, expected_loader) def _check_import_error(self, script_name, expected_msg, *cmd_line_switches): @@ -128,11 +137,27 @@ class CmdLineTest(unittest.TestCase): print('Expected output: %r' % expected_msg) self.assertIn(expected_msg.encode('utf-8'), err) + def test_dash_c_loader(self): + rc, out, err = assert_python_ok("-c", "print(__loader__)") + expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8") + self.assertIn(expected, out) + + def test_stdin_loader(self): + p = spawn_python() + try: + p.stdin.write(b"print(__loader__)\n") + p.stdin.flush() + finally: + out = kill_python(p) + expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8") + self.assertIn(expected, out) + def test_basic_script(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') self._check_script(script_name, script_name, script_name, - script_dir, None) + script_dir, None, + importlib.machinery.SourceFileLoader) def test_script_compiled(self): with temp_dir() as script_dir: @@ -141,13 +166,15 @@ class CmdLineTest(unittest.TestCase): os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) self._check_script(pyc_file, pyc_file, - pyc_file, script_dir, None) + pyc_file, script_dir, None, + importlib.machinery.SourcelessFileLoader) def test_directory(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') self._check_script(script_dir, script_name, script_dir, - script_dir, '') + script_dir, '', + importlib.machinery.SourceFileLoader) def test_directory_compiled(self): with temp_dir() as script_dir: @@ -156,7 +183,8 @@ class CmdLineTest(unittest.TestCase): os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) self._check_script(script_dir, pyc_file, script_dir, - script_dir, '') + script_dir, '', + importlib.machinery.SourcelessFileLoader) def test_directory_error(self): with temp_dir() as script_dir: @@ -167,14 +195,16 @@ class CmdLineTest(unittest.TestCase): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name) - self._check_script(zip_name, run_name, zip_name, zip_name, '') + self._check_script(zip_name, run_name, zip_name, zip_name, '', + zipimport.zipimporter) def test_zipfile_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') compiled_name = py_compile.compile(script_name, doraise=True) zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) - self._check_script(zip_name, run_name, zip_name, zip_name, '') + self._check_script(zip_name, run_name, zip_name, zip_name, '', + zipimport.zipimporter) def test_zipfile_error(self): with temp_dir() as script_dir: @@ -189,19 +219,24 @@ class CmdLineTest(unittest.TestCase): make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, 'script') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script') - self._check_script(launch_name, script_name, script_name, script_dir, 'test_pkg') + self._check_script(launch_name, script_name, script_name, + script_dir, 'test_pkg', + importlib.machinery.SourceFileLoader) def test_module_in_package_in_zipfile(self): with temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg') + self._check_script(launch_name, run_name, run_name, + zip_name, 'test_pkg', zipimport.zipimporter) def test_module_in_subpackage_in_zipfile(self): with temp_dir() as script_dir: zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2) launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name) - self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg.test_pkg') + self._check_script(launch_name, run_name, run_name, + zip_name, 'test_pkg.test_pkg', + zipimport.zipimporter) def test_package(self): with temp_dir() as script_dir: @@ -210,7 +245,8 @@ class CmdLineTest(unittest.TestCase): script_name = _make_test_script(pkg_dir, '__main__') launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') self._check_script(launch_name, script_name, - script_name, script_dir, 'test_pkg') + script_name, script_dir, 'test_pkg', + importlib.machinery.SourceFileLoader) def test_package_compiled(self): with temp_dir() as script_dir: @@ -222,7 +258,8 @@ class CmdLineTest(unittest.TestCase): pyc_file = support.make_legacy_pyc(script_name) launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') self._check_script(launch_name, pyc_file, - pyc_file, script_dir, 'test_pkg') + pyc_file, script_dir, 'test_pkg', + importlib.machinery.SourcelessFileLoader) def test_package_error(self): with temp_dir() as script_dir: @@ -259,7 +296,8 @@ class CmdLineTest(unittest.TestCase): expected = "init_argv0==%r" % '-m' self.assertIn(expected.encode('utf-8'), out) self._check_output(script_name, rc, out, - script_name, script_name, '', 'test_pkg') + script_name, script_name, '', 'test_pkg', + importlib.machinery.SourceFileLoader) def test_issue8202_dash_c_file_ignored(self): # Make sure a "-c" file in the current directory @@ -285,7 +323,8 @@ class CmdLineTest(unittest.TestCase): f.write("data") rc, out, err = assert_python_ok('-m', 'other', *example_args) self._check_output(script_name, rc, out, - script_name, script_name, '', '') + script_name, script_name, '', '', + importlib.machinery.SourceFileLoader) def test_dash_m_error_code_is_one(self): # If a module is invoked with the -m command line flag diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index e46731a..51f5dee 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,4 +1,4 @@ -from test.support import run_unittest, unload +from test.support import run_unittest, unload, check_warnings import unittest import sys import imp @@ -255,12 +255,51 @@ class NestedNamespacePackageTest(unittest.TestCase): self.assertEqual(d, 2) +class ImportlibMigrationTests(unittest.TestCase): + # With full PEP 302 support in the standard import machinery, the + # PEP 302 emulation in this module is in the process of being + # deprecated in favour of importlib proper + + def check_deprecated(self): + return check_warnings( + ("This emulation is deprecated, use 'importlib' instead", + DeprecationWarning)) + + def test_importer_deprecated(self): + with self.check_deprecated(): + x = pkgutil.ImpImporter("") + + def test_loader_deprecated(self): + with self.check_deprecated(): + x = pkgutil.ImpLoader("", "", "", "") + + def test_get_loader_avoids_emulation(self): + with check_warnings() as w: + self.assertIsNotNone(pkgutil.get_loader("sys")) + self.assertIsNotNone(pkgutil.get_loader("os")) + self.assertIsNotNone(pkgutil.get_loader("test.support")) + self.assertEqual(len(w.warnings), 0) + + def test_get_importer_avoids_emulation(self): + with check_warnings() as w: + self.assertIsNotNone(pkgutil.get_importer(sys.path[0])) + self.assertEqual(len(w.warnings), 0) + + def test_iter_importers_avoids_emulation(self): + with check_warnings() as w: + for importer in pkgutil.iter_importers(): pass + self.assertEqual(len(w.warnings), 0) + + def test_main(): run_unittest(PkgutilTests, PkgutilPEP302Tests, ExtendPathTests, - NestedNamespacePackageTest) + NestedNamespacePackageTest, ImportlibMigrationTests) # this is necessary if test is run repeated (like when finding leaks) import zipimport + import importlib zipimport._zip_directory_cache.clear() + importlib.invalidate_caches() + if __name__ == '__main__': test_main() |