diff options
author | Brett Cannon <brett@python.org> | 2013-05-31 22:56:47 (GMT) |
---|---|---|
committer | Brett Cannon <brett@python.org> | 2013-05-31 22:56:47 (GMT) |
commit | 0dbb4c7f1338d1391e7214b564ef4638bc257347 (patch) | |
tree | d9bd89758691c3b739c68e7eb50b444b15186bd1 /Lib | |
parent | f1d7b11db905db5b40e2d97fa21af06871cf89ff (diff) | |
download | cpython-0dbb4c7f1338d1391e7214b564ef4638bc257347.zip cpython-0dbb4c7f1338d1391e7214b564ef4638bc257347.tar.gz cpython-0dbb4c7f1338d1391e7214b564ef4638bc257347.tar.bz2 |
Issues #18088, 18089: Introduce
importlib.abc.Loader.init_module_attrs() and implement
importlib.abc.InspectLoader.load_module().
The importlib.abc.Loader.init_module_attrs() method sets the various
attributes on the module being loaded. It is done unconditionally to
support reloading. Typically people used
importlib.util.module_for_loader, but since that's a decorator there
was no way to override it's actions, so init_module_attrs() came into
existence to allow for overriding. This is also why module_for_loader
is now pending deprecation (having its other use replaced by
importlib.util.module_to_load).
All of this allowed for importlib.abc.InspectLoader.load_module() to
be implemented. At this point you can now implement a loader with
nothing more than get_code() (which only requires get_source();
package support requires is_package()). Thanks to init_module_attrs()
the implementation of load_module() is basically a context manager
containing 2 methods calls, a call to exec(), and a return statement.
Diffstat (limited to 'Lib')
-rw-r--r-- | Lib/importlib/_bootstrap.py | 125 | ||||
-rw-r--r-- | Lib/importlib/abc.py | 32 | ||||
-rw-r--r-- | Lib/importlib/util.py | 45 | ||||
-rw-r--r-- | Lib/test/test_importlib/source/test_file_loader.py | 33 | ||||
-rw-r--r-- | Lib/test/test_importlib/test_abc.py | 202 | ||||
-rw-r--r-- | Lib/test/test_importlib/test_util.py | 24 |
6 files changed, 357 insertions, 104 deletions
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index d4d2f91..f8a1b0b 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -538,6 +538,32 @@ def module_to_load(name, *, reset_name=True): return _ModuleManager(name, reset_name=reset_name) +def _init_package_attrs(loader, module): + """Set __package__ and __path__ based on what loader.is_package() says.""" + name = module.__name__ + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + module.__package__ = name + module.__path__ = [] + else: + module.__package__ = name.rpartition('.')[0] + + +def _init_file_attrs(loader, module): + """Set __file__ and __path__ based on loader.get_filename().""" + try: + module.__file__ = loader.get_filename(module.__name__) + except ImportError: + pass + else: + if module.__name__ == module.__package__: + module.__path__.append(_path_split(module.__file__)[0]) + + def set_package(fxn): """Set __package__ on the returned module.""" def set_package_wrapper(*args, **kwargs): @@ -562,42 +588,6 @@ def set_loader(fxn): return set_loader_wrapper -def module_for_loader(fxn): - """Decorator to handle selecting the proper module for loaders. - - The decorated function is passed the module to use instead of the module - name. The module passed in to the function is either from sys.modules if - it already exists or is a new module. If the module is new, then __name__ - is set the first argument to the method, __loader__ is set to self, and - __package__ is set accordingly (if self.is_package() is defined) will be set - before it is passed to the decorated function (if self.is_package() does - not work for the module it will be set post-load). - - If an exception is raised and the decorator created the module it is - subsequently removed from sys.modules. - - The decorator assumes that the decorated function takes the module name as - the second argument. - - """ - def module_for_loader_wrapper(self, fullname, *args, **kwargs): - with module_to_load(fullname) as module: - module.__loader__ = self - try: - is_package = self.is_package(fullname) - except (ImportError, AttributeError): - pass - else: - if is_package: - module.__package__ = fullname - else: - module.__package__ = fullname.rpartition('.')[0] - # If __package__ was not set above, __import__() will do it later. - return fxn(self, module, *args, **kwargs) - _wrap(module_for_loader_wrapper, fxn) - return module_for_loader_wrapper - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -904,25 +894,32 @@ class _LoaderBasics: tail_name = fullname.rpartition('.')[2] return filename_base == '__init__' and tail_name != '__init__' - @module_for_loader - def _load_module(self, module, *, sourceless=False): - """Helper for load_module able to handle either source or sourceless - loading.""" - name = module.__name__ - code_object = self.get_code(name) - module.__file__ = self.get_filename(name) - if not sourceless: + def init_module_attrs(self, module): + """Set various attributes on the module. + + ExecutionLoader.init_module_attrs() is used to set __loader__, + __package__, __file__, and optionally __path__. The __cached__ attribute + is set using imp.cache_from_source() and __file__. + """ + module.__loader__ = self # Loader + _init_package_attrs(self, module) # InspectLoader + _init_file_attrs(self, module) # ExecutionLoader + if hasattr(module, '__file__'): # SourceLoader try: module.__cached__ = cache_from_source(module.__file__) except NotImplementedError: - module.__cached__ = module.__file__ - else: - module.__cached__ = module.__file__ - if self.is_package(name): - module.__path__ = [_path_split(module.__file__)[0]] - # __package__ and __loader set by @module_for_loader. - _call_with_frames_removed(exec, code_object, module.__dict__) - return module + pass + + def load_module(self, fullname): + """Load the specified module into sys.modules and return it.""" + with module_to_load(fullname) as module: + self.init_module_attrs(module) + code = self.get_code(fullname) + if code is None: + raise ImportError('cannot load module {!r} when get_code() ' + 'returns None'.format(fullname)) + _call_with_frames_removed(exec, code, module.__dict__) + return module class SourceLoader(_LoaderBasics): @@ -1046,16 +1043,6 @@ class SourceLoader(_LoaderBasics): pass return code_object - def load_module(self, fullname): - """Concrete implementation of Loader.load_module. - - Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be - implemented to load source code. Use of bytecode is dictated by whether - get_code uses/writes bytecode. - - """ - return self._load_module(fullname) - class FileLoader: @@ -1133,8 +1120,9 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics): """Loader which handles sourceless file imports.""" - def load_module(self, fullname): - return self._load_module(fullname, sourceless=True) + def init_module_attrs(self, module): + super().init_module_attrs(module) + module.__cached__ = module.__file__ def get_code(self, fullname): path = self.get_filename(fullname) @@ -1259,12 +1247,13 @@ class NamespaceLoader: def module_repr(cls, module): return "<module '{}' (namespace)>".format(module.__name__) - @module_for_loader - def load_module(self, module): + def load_module(self, fullname): """Load a namespace module.""" _verbose_message('namespace module loaded with path {!r}', self._path) - module.__path__ = self._path - return module + with module_to_load(fullname) as module: + module.__path__ = self._path + module.__package__ = fullname + return module # Finders ##################################################################### diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 897f343..417fe41 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -8,11 +8,6 @@ except ImportError as exc: raise _frozen_importlib = None import abc -import imp -import marshal -import sys -import tokenize -import warnings def _register(abstract_cls, *classes): @@ -113,6 +108,10 @@ class Loader(metaclass=abc.ABCMeta): """ raise NotImplementedError + def init_module_attrs(self, module): + """Set the module's __loader__ attribute.""" + module.__loader__ = self + class ResourceLoader(Loader): @@ -177,6 +176,17 @@ class InspectLoader(Loader): argument should be where the data was retrieved (when applicable).""" return compile(data, path, 'exec', dont_inherit=True) + def init_module_attrs(self, module): + """Initialize the __loader__ and __package__ attributes of the module. + + The name of the module is gleaned from module.__name__. The __package__ + attribute is set based on self.is_package(). + """ + super().init_module_attrs(module) + _bootstrap._init_package_attrs(self, module) + + load_module = _bootstrap._LoaderBasics.load_module + _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, machinery.ExtensionFileLoader) @@ -215,6 +225,18 @@ class ExecutionLoader(InspectLoader): else: return self.source_to_code(source, path) + def init_module_attrs(self, module): + """Initialize the module's attributes. + + It is assumed that the module's name has been set on module.__name__. + It is also assumed that any path returned by self.get_filename() uses + (one of) the operating system's path separator(s) to separate filenames + from directories in order to set __path__ intelligently. + InspectLoader.init_module_attrs() sets __loader__ and __package__. + """ + super().init_module_attrs(module) + _bootstrap._init_file_attrs(self, module) + class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 74eef2a..573b08c 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,11 +1,13 @@ """Utility code for constructing importers, etc.""" from ._bootstrap import module_to_load -from ._bootstrap import module_for_loader from ._bootstrap import set_loader from ._bootstrap import set_package from ._bootstrap import _resolve_name +import functools +import warnings + def resolve_name(name, package): """Resolve a relative module name to an absolute one.""" @@ -20,3 +22,44 @@ def resolve_name(name, package): break level += 1 return _resolve_name(name[level:], package, level) + + +def module_for_loader(fxn): + """Decorator to handle selecting the proper module for loaders. + + The decorated function is passed the module to use instead of the module + name. The module passed in to the function is either from sys.modules if + it already exists or is a new module. If the module is new, then __name__ + is set the first argument to the method, __loader__ is set to self, and + __package__ is set accordingly (if self.is_package() is defined) will be set + before it is passed to the decorated function (if self.is_package() does + not work for the module it will be set post-load). + + If an exception is raised and the decorator created the module it is + subsequently removed from sys.modules. + + The decorator assumes that the decorated function takes the module name as + the second argument. + + """ + warnings.warn('To make it easier for subclasses, please use ' + 'importlib.util.module_to_load() and ' + 'importlib.abc.Loader.init_module_attrs()', + PendingDeprecationWarning, stacklevel=2) + @functools.wraps(fxn) + def module_for_loader_wrapper(self, fullname, *args, **kwargs): + with module_to_load(fullname) as module: + module.__loader__ = self + try: + is_package = self.is_package(fullname) + except (ImportError, AttributeError): + pass + else: + if is_package: + module.__package__ = fullname + else: + module.__package__ = fullname.rpartition('.')[0] + # If __package__ was not set above, __import__() will do it later. + return fxn(self, module, *args, **kwargs) + + return module_for_loader_wrapper
\ No newline at end of file diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 80999e8..6e248f3 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -15,7 +15,7 @@ import stat import sys import unittest -from test.support import make_legacy_pyc +from test.support import make_legacy_pyc, unload class SimpleTest(unittest.TestCase): @@ -26,23 +26,13 @@ class SimpleTest(unittest.TestCase): """ def test_load_module_API(self): - # If fullname is not specified that assume self.name is desired. - class TesterMixin(importlib.abc.Loader): - def load_module(self, fullname): return fullname - def module_repr(self, module): return '<module>' - - class Tester(importlib.abc.FileLoader, TesterMixin): - def get_code(self, _): pass - def get_source(self, _): pass - def is_package(self, _): pass + class Tester(importlib.abc.FileLoader): + def get_source(self, _): return 'attr = 42' + def is_package(self, _): return False - name = 'mod_name' - loader = Tester(name, 'some_path') - self.assertEqual(name, loader.load_module()) - self.assertEqual(name, loader.load_module(None)) - self.assertEqual(name, loader.load_module(name)) - with self.assertRaises(ImportError): - loader.load_module(loader.name + 'XXX') + loader = Tester('blah', 'blah.py') + self.addCleanup(unload, 'blah') + module = loader.load_module() # Should not raise an exception. def test_get_filename_API(self): # If fullname is not set then assume self.path is desired. @@ -473,13 +463,6 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): self._test_non_code_marshal(del_source=True) -def test_main(): - from test.support import run_unittest - run_unittest(SimpleTest, - SourceLoaderBadBytecodeTest, - SourcelessLoaderBadBytecodeTest - ) - if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index b443337..ea417f5 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -2,12 +2,14 @@ import importlib from importlib import abc from importlib import machinery +import contextlib import imp import inspect import io import marshal import os import sys +from test import support import unittest from unittest import mock @@ -198,6 +200,15 @@ class ExecutionLoaderDefaultsTests(unittest.TestCase): with self.assertRaises(ImportError): self.ins.get_filename('blah') +##### Loader concrete methods ################################################## +class LoaderConcreteMethodTests(unittest.TestCase): + + def test_init_module_attrs(self): + loader = LoaderSubclass() + module = imp.new_module('blah') + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + ##### InspectLoader concrete methods ########################################### class InspectLoaderSourceToCodeTests(unittest.TestCase): @@ -269,6 +280,93 @@ class InspectLoaderGetCodeTests(unittest.TestCase): loader.get_code('blah') +class InspectLoaderInitModuleTests(unittest.TestCase): + + @staticmethod + def mock_is_package(return_value): + return mock.patch.object(InspectLoaderSubclass, 'is_package', + return_value=return_value) + + def init_module_attrs(self, name): + loader = InspectLoaderSubclass() + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + return module + + def test_package(self): + # If a package, then __package__ == __name__, __path__ == [] + with self.mock_is_package(True): + name = 'blah' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, name) + self.assertEqual(module.__path__, []) + + def test_toplevel(self): + # If a module is top-level, __package__ == '' + with self.mock_is_package(False): + name = 'blah' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, '') + + def test_submodule(self): + # If a module is contained within a package then set __package__ to the + # package name. + with self.mock_is_package(False): + name = 'pkg.mod' + module = self.init_module_attrs(name) + self.assertEqual(module.__package__, 'pkg') + + def test_is_package_ImportError(self): + # If is_package() raises ImportError, __package__ should be None and + # __path__ should not be set. + with self.mock_is_package(False) as mocked_method: + mocked_method.side_effect = ImportError + name = 'mod' + module = self.init_module_attrs(name) + self.assertIsNone(module.__package__) + self.assertFalse(hasattr(module, '__path__')) + + +class InspectLoaderLoadModuleTests(unittest.TestCase): + + """Test InspectLoader.load_module().""" + + module_name = 'blah' + + def setUp(self): + support.unload(self.module_name) + self.addCleanup(support.unload, self.module_name) + + def mock_get_code(self): + return mock.patch.object(InspectLoaderSubclass, 'get_code') + + def test_get_code_ImportError(self): + # If get_code() raises ImportError, it should propagate. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.side_effect = ImportError + with self.assertRaises(ImportError): + loader = InspectLoaderSubclass() + loader.load_module(self.module_name) + + def test_get_code_None(self): + # If get_code() returns None, raise ImportError. + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = None + with self.assertRaises(ImportError): + loader = InspectLoaderSubclass() + loader.load_module(self.module_name) + + def test_module_returned(self): + # The loaded module should be returned. + code = compile('attr = 42', '<string>', 'exec') + with self.mock_get_code() as mocked_get_code: + mocked_get_code.return_value = code + loader = InspectLoaderSubclass() + module = loader.load_module(self.module_name) + self.assertEqual(module, sys.modules[self.module_name]) + + ##### ExecutionLoader concrete methods ######################################### class ExecutionLoaderGetCodeTests(unittest.TestCase): @@ -327,6 +425,69 @@ class ExecutionLoaderGetCodeTests(unittest.TestCase): self.assertEqual(module.attr, 42) +class ExecutionLoaderInitModuleTests(unittest.TestCase): + + @staticmethod + @contextlib.contextmanager + def mock_methods(is_package, filename): + is_package_manager = InspectLoaderInitModuleTests.mock_is_package(is_package) + get_filename_manager = mock.patch.object(ExecutionLoaderSubclass, + 'get_filename', return_value=filename) + with is_package_manager as mock_is_package: + with get_filename_manager as mock_get_filename: + yield {'is_package': mock_is_package, + 'get_filename': mock_get_filename} + + def test_toplevel(self): + # Verify __loader__, __file__, and __package__; no __path__. + name = 'blah' + path = os.path.join('some', 'path', '{}.py'.format(name)) + with self.mock_methods(False, path): + loader = ExecutionLoaderSubclass() + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__file__, path) + self.assertEqual(module.__package__, '') + self.assertFalse(hasattr(module, '__path__')) + + def test_package(self): + # Verify __loader__, __file__, __package__, and __path__. + name = 'pkg' + path = os.path.join('some', 'pkg', '__init__.py') + with self.mock_methods(True, path): + loader = ExecutionLoaderSubclass() + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertIs(module.__loader__, loader) + self.assertEqual(module.__file__, path) + self.assertEqual(module.__package__, 'pkg') + self.assertEqual(module.__path__, [os.path.dirname(path)]) + + def test_submodule(self): + # Verify __package__ and not __path__; test_toplevel() takes care of + # other attributes. + name = 'pkg.submodule' + path = os.path.join('some', 'pkg', 'submodule.py') + with self.mock_methods(False, path): + loader = ExecutionLoaderSubclass() + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertEqual(module.__package__, 'pkg') + self.assertEqual(module.__file__, path) + self.assertFalse(hasattr(module, '__path__')) + + def test_get_filename_ImportError(self): + # If get_filename() raises ImportError, don't set __file__. + name = 'blah' + path = 'blah.py' + with self.mock_methods(False, path) as mocked_methods: + mocked_methods['get_filename'].side_effect = ImportError + loader = ExecutionLoaderSubclass() + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertFalse(hasattr(module, '__file__')) + ##### SourceLoader concrete methods ############################################ class SourceOnlyLoaderMock(abc.SourceLoader): @@ -621,6 +782,47 @@ class SourceLoaderGetSourceTests(unittest.TestCase): self.assertEqual(mock.get_source(name), expect) +class SourceLoaderInitModuleAttrTests(unittest.TestCase): + + """Tests for importlib.abc.SourceLoader.init_module_attrs().""" + + def test_init_module_attrs(self): + # If __file__ set, __cached__ == imp.cached_from_source(__file__). + name = 'blah' + path = 'blah.py' + loader = SourceOnlyLoaderMock(path) + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertEqual(module.__loader__, loader) + self.assertEqual(module.__package__, '') + self.assertEqual(module.__file__, path) + self.assertEqual(module.__cached__, imp.cache_from_source(path)) + + @mock.patch('importlib._bootstrap.cache_from_source') + def test_cache_from_source_NotImplementedError(self, mock_cache_from_source): + # If imp.cache_from_source() raises NotImplementedError don't set + # __cached__. + mock_cache_from_source.side_effect = NotImplementedError + name = 'blah' + path = 'blah.py' + loader = SourceOnlyLoaderMock(path) + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertEqual(module.__file__, path) + self.assertFalse(hasattr(module, '__cached__')) + + def test_no_get_filename(self): + # No __file__, no __cached__. + with mock.patch.object(SourceOnlyLoaderMock, 'get_filename') as mocked: + mocked.side_effect = ImportError + name = 'blah' + loader = SourceOnlyLoaderMock('blah.py') + module = imp.new_module(name) + loader.init_module_attrs(module) + self.assertFalse(hasattr(module, '__file__')) + self.assertFalse(hasattr(module, '__cached__')) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 9897def..e6b0084 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -5,6 +5,7 @@ import sys from test import support import types import unittest +import warnings class ModuleToLoadTests(unittest.TestCase): @@ -72,14 +73,27 @@ class ModuleForLoaderTests(unittest.TestCase): """Tests for importlib.util.module_for_loader.""" + @staticmethod + def module_for_loader(func): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', PendingDeprecationWarning) + return util.module_for_loader(func) + + def test_warning(self): + # Should raise a PendingDeprecationWarning when used. + with warnings.catch_warnings(): + warnings.simplefilter('error', PendingDeprecationWarning) + with self.assertRaises(PendingDeprecationWarning): + func = util.module_for_loader(lambda x: x) + def return_module(self, name): - fxn = util.module_for_loader(lambda self, module: module) + fxn = self.module_for_loader(lambda self, module: module) return fxn(self, name) def raise_exception(self, name): def to_wrap(self, module): raise ImportError - fxn = util.module_for_loader(to_wrap) + fxn = self.module_for_loader(to_wrap) try: fxn(self, name) except ImportError: @@ -100,7 +114,7 @@ class ModuleForLoaderTests(unittest.TestCase): class FakeLoader: def is_package(self, name): return True - @util.module_for_loader + @self.module_for_loader def load_module(self, module): return module name = 'a.b.c' @@ -134,7 +148,7 @@ class ModuleForLoaderTests(unittest.TestCase): def test_decorator_attrs(self): def fxn(self, module): pass - wrapped = util.module_for_loader(fxn) + wrapped = self.module_for_loader(fxn) self.assertEqual(wrapped.__name__, fxn.__name__) self.assertEqual(wrapped.__qualname__, fxn.__qualname__) @@ -160,7 +174,7 @@ class ModuleForLoaderTests(unittest.TestCase): self._pkg = is_package def is_package(self, name): return self._pkg - @util.module_for_loader + @self.module_for_loader def load_module(self, module): return module |