diff options
Diffstat (limited to 'Lib/importlib')
| -rw-r--r-- | Lib/importlib/__init__.py | 8 | ||||
| -rw-r--r-- | Lib/importlib/_bootstrap.py | 394 | ||||
| -rw-r--r-- | Lib/importlib/abc.py | 5 | ||||
| -rw-r--r-- | Lib/importlib/util.py | 99 |
4 files changed, 276 insertions, 230 deletions
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 1bc9947..e99f50e 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -73,7 +73,7 @@ def find_loader(name, path=None): except KeyError: pass except AttributeError: - raise ValueError('{}.__loader__ is not set'.format(name)) + raise ValueError('{}.__loader__ is not set'.format(name)) from None spec = _bootstrap._find_spec(name, path) # We won't worry about malformed specs (missing attributes). @@ -138,15 +138,15 @@ def reload(module): parent = sys.modules[parent_name] except KeyError: msg = "parent {!r} not in sys.modules" - raise ImportError(msg.format(parent_name), name=parent_name) + raise ImportError(msg.format(parent_name), + name=parent_name) from None else: pkgpath = parent.__path__ else: pkgpath = None target = module spec = module.__spec__ = _bootstrap._find_spec(name, pkgpath, target) - methods = _bootstrap._SpecMethods(spec) - methods.exec(module) + _bootstrap._exec(spec, module) # The module may have replaced itself in sys.modules! return sys.modules[name] finally: diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 5b91c05..e694521 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module. # # IMPORTANT: Whenever making changes to this module, be sure to run # a top-level make in order to get the frozen version of the module -# update. Not doing so will result in the Makefile to fail for +# updated. Not doing so will result in the Makefile to fail for # all others who don't have a ./python around to freeze the module # in the early stages of compilation. # @@ -419,12 +419,13 @@ def _call_with_frames_removed(f, *args, **kwds): # Python 3.4a4 3290 (changes to __qualname__ computation) # Python 3.4a4 3300 (more changes to __qualname__ computation) # Python 3.4rc2 3310 (alter __qualname__ computation) +# Python 3.5a0 3320 (matrix multiplication operator) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3310).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3320).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' @@ -580,6 +581,7 @@ def _find_module_shim(self, fullname): return loader +# Typically used by loader classes as a method replacement. def _load_module_shim(self, fullname): """Load the specified module into sys.modules and return it. @@ -587,13 +589,12 @@ def _load_module_shim(self, fullname): """ spec = spec_from_loader(fullname, self) - methods = _SpecMethods(spec) if fullname in sys.modules: module = sys.modules[fullname] - methods.exec(module) + _exec(spec, module) return sys.modules[fullname] else: - return methods.load() + return _load(spec) def _validate_bytecode_header(data, source_stats=None, name=None, path=None): @@ -704,7 +705,7 @@ def _module_repr(module): pass else: if spec is not None: - return _SpecMethods(spec).module_repr() + return _module_repr_from_spec(spec) # We could use module.__class__.__name__ instead of 'module' in the # various repr permutations. @@ -990,234 +991,182 @@ def _spec_from_module(module, loader=None, origin=None): return spec -class _SpecMethods: - - """Convenience wrapper around spec objects to provide spec-specific - methods.""" - - # The various spec_from_* functions could be made factory methods here. - - def __init__(self, spec): - self.spec = spec - - def module_repr(self): - """Return the repr to use for the module.""" - # We mostly replicate _module_repr() using the spec attributes. - spec = self.spec - name = '?' if spec.name is None else spec.name - if spec.origin is None: - if spec.loader is None: - return '<module {!r}>'.format(name) - else: - return '<module {!r} ({!r})>'.format(name, spec.loader) - else: - if spec.has_location: - return '<module {!r} from {!r}>'.format(name, spec.origin) - else: - return '<module {!r} ({})>'.format(spec.name, spec.origin) - - def init_module_attrs(self, module, *, _override=False, _force_name=True): - """Set the module's attributes. - - All missing import-related module attributes will be set. Here - is how the spec attributes map onto the module: - - spec.name -> module.__name__ - spec.loader -> module.__loader__ - spec.parent -> module.__package__ - spec -> module.__spec__ - - Optional: - spec.origin -> module.__file__ (if spec.set_fileattr is true) - spec.cached -> module.__cached__ (if __file__ also set) - spec.submodule_search_locations -> module.__path__ (if set) - - """ - spec = self.spec - - # The passed in module may be not support attribute assignment, - # in which case we simply don't set the attributes. - - # __name__ - if (_override or _force_name or - getattr(module, '__name__', None) is None): - try: - module.__name__ = spec.name - except AttributeError: - pass - - # __loader__ - if _override or getattr(module, '__loader__', None) is None: - loader = spec.loader - if loader is None: - # A backward compatibility hack. - if spec.submodule_search_locations is not None: - loader = _NamespaceLoader.__new__(_NamespaceLoader) - loader._path = spec.submodule_search_locations +def _init_module_attrs(spec, module, *, override=False): + # The passed-in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + # __name__ + if (override or getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + # __loader__ + if override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader._path = spec.submodule_search_locations + try: + module.__loader__ = loader + except AttributeError: + pass + # __package__ + if override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass + # __path__ + if override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: try: - module.__loader__ = loader + module.__path__ = spec.submodule_search_locations except AttributeError: pass - - # __package__ - if _override or getattr(module, '__package__', None) is None: + # __file__/__cached__ + if spec.has_location: + if override or getattr(module, '__file__', None) is None: try: - module.__package__ = spec.parent + module.__file__ = spec.origin except AttributeError: pass - # __spec__ - try: - module.__spec__ = spec - except AttributeError: - pass - - # __path__ - if _override or getattr(module, '__path__', None) is None: - if spec.submodule_search_locations is not None: - try: - module.__path__ = spec.submodule_search_locations - except AttributeError: - pass - - if spec.has_location: - # __file__ - if _override or getattr(module, '__file__', None) is None: + if override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: try: - module.__file__ = spec.origin + module.__cached__ = spec.cached except AttributeError: pass + return module - # __cached__ - if _override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass - def create(self): - """Return a new module to be loaded. +def module_from_spec(spec): + """Create a module based on the provided spec.""" + # Typically loaders will not implement create_module(). + module = None + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` then it means default + # module creation should be used. + module = spec.loader.create_module(spec) + if module is None: + module = _new_module(spec.name) + _init_module_attrs(spec, module) + return module - The import-related module attributes are also set with the - appropriate values from the spec. - """ - spec = self.spec - # Typically loaders will not implement create_module(). - if hasattr(spec.loader, 'create_module'): - # If create_module() returns `None` it means the default - # module creation should be used. - module = spec.loader.create_module(spec) +def _module_repr_from_spec(spec): + """Return the repr to use for the module.""" + # We mostly replicate _module_repr() using the spec attributes. + name = '?' if spec.name is None else spec.name + if spec.origin is None: + if spec.loader is None: + return '<module {!r}>'.format(name) else: - module = None - if module is None: - # This must be done before open() is ever called as the 'io' - # module implicitly imports 'locale' and would otherwise - # trigger an infinite loop. - module = _new_module(spec.name) - self.init_module_attrs(module) - return module - - def _exec(self, module): - """Do everything necessary to execute the module. - - The namespace of `module` is used as the target of execution. - This method uses the loader's `exec_module()` method. + return '<module {!r} ({!r})>'.format(name, spec.loader) + else: + if spec.has_location: + return '<module {!r} from {!r}>'.format(name, spec.origin) + else: + return '<module {!r} ({})>'.format(spec.name, spec.origin) - """ - self.spec.loader.exec_module(module) - # Used by importlib.reload() and _load_module_shim(). - def exec(self, module): - """Execute the spec in an existing module's namespace.""" - name = self.spec.name - _imp.acquire_lock() - with _ModuleLockManager(name): - if sys.modules.get(name) is not module: - msg = 'module {!r} not in sys.modules'.format(name) - raise ImportError(msg, name=name) - if self.spec.loader is None: - if self.spec.submodule_search_locations is None: - raise ImportError('missing loader', name=self.spec.name) - # namespace package - self.init_module_attrs(module, _override=True) - return module - self.init_module_attrs(module, _override=True) - if not hasattr(self.spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - self.spec.loader.load_module(name) - else: - self._exec(module) - return sys.modules[name] - - def _load_backward_compatible(self): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - spec = self.spec - spec.loader.load_module(spec.name) - # The module must be in sys.modules at this point! - module = sys.modules[spec.name] - if getattr(module, '__loader__', None) is None: - try: - module.__loader__ = spec.loader - except AttributeError: - pass - if getattr(module, '__package__', None) is None: - try: - # Since module.__path__ may not line up with - # spec.submodule_search_paths, we can't necessarily rely - # on spec.parent here. - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = spec.name.rpartition('.')[0] - except AttributeError: - pass - if getattr(module, '__spec__', None) is None: - try: - module.__spec__ = spec - except AttributeError: - pass - return module +# Used by importlib.reload() and _load_module_shim(). +def _exec(spec, module): + """Execute the spec in an existing module's namespace.""" + name = spec.name + _imp.acquire_lock() + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = 'module {!r} not in sys.modules'.format(name) + raise ImportError(msg, name=name) + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # namespace package + _init_module_attrs(spec, module, override=True) + return module + _init_module_attrs(spec, module, override=True) + if not hasattr(spec.loader, 'exec_module'): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + spec.loader.load_module(name) + else: + spec.loader.exec_module(module) + return sys.modules[name] + + +def _load_backward_compatible(spec): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + spec.loader.load_module(spec.name) + # The module must be in sys.modules at this point! + module = sys.modules[spec.name] + if getattr(module, '__loader__', None) is None: + try: + module.__loader__ = spec.loader + except AttributeError: + pass + if getattr(module, '__package__', None) is None: + try: + # Since module.__path__ may not line up with + # spec.submodule_search_paths, we can't necessarily rely + # on spec.parent here. + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = spec.name.rpartition('.')[0] + except AttributeError: + pass + if getattr(module, '__spec__', None) is None: + try: + module.__spec__ = spec + except AttributeError: + pass + return module - def _load_unlocked(self): - # A helper for direct use by the import system. - if self.spec.loader is not None: - # not a namespace package - if not hasattr(self.spec.loader, 'exec_module'): - return self._load_backward_compatible() - - module = self.create() - with _installed_safely(module): - if self.spec.loader is None: - if self.spec.submodule_search_locations is None: - raise ImportError('missing loader', name=self.spec.name) - # A namespace package so do nothing. - else: - self._exec(module) +def _load_unlocked(spec): + # A helper for direct use by the import system. + if spec.loader is not None: + # not a namespace package + if not hasattr(spec.loader, 'exec_module'): + return _load_backward_compatible(spec) + + module = module_from_spec(spec) + with _installed_safely(module): + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # A namespace package so do nothing. + else: + spec.loader.exec_module(module) - # We don't ensure that the import-related module attributes get - # set in the sys.modules replacement case. Such modules are on - # their own. - return sys.modules[self.spec.name] + # We don't ensure that the import-related module attributes get + # set in the sys.modules replacement case. Such modules are on + # their own. + return sys.modules[spec.name] - # A method used during testing of _load_unlocked() and by - # _load_module_shim(). - def load(self): - """Return a new module object, loaded by the spec's loader. +# A method used during testing of _load_unlocked() and by +# _load_module_shim(). +def _load(spec): + """Return a new module object, loaded by the spec's loader. - The module is not added to its parent. + The module is not added to its parent. - If a module is already in sys.modules, that existing module gets - clobbered. + If a module is already in sys.modules, that existing module gets + clobbered. - """ - _imp.acquire_lock() - with _ModuleLockManager(self.spec.name): - return self._load_unlocked() + """ + _imp.acquire_lock() + with _ModuleLockManager(spec.name): + return _load_unlocked(spec) def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1799,7 +1748,7 @@ class _NamespacePath: self._path.append(item) -# We use this exclusively in init_module_attrs() for backward-compatibility. +# We use this exclusively in module_from_spec() for backward-compatibility. class _NamespaceLoader: def __init__(self, name, path, path_finder): self._path = _NamespacePath(name, path, path_finder) @@ -1857,7 +1806,7 @@ class PathFinder: If 'hooks' is false then use sys.path_hooks. """ - if not sys.path_hooks: + if sys.path_hooks is not None and not sys.path_hooks: _warnings.warn('sys.path_hooks is empty', ImportWarning) for hook in sys.path_hooks: try: @@ -1876,7 +1825,12 @@ class PathFinder: """ if path == '': - path = _os.getcwd() + try: + path = _os.getcwd() + except FileNotFoundError: + # Don't cache the failure as the cwd can easily change to + # a valid directory later on. + return None try: finder = sys.path_importer_cache[path] except KeyError: @@ -2146,7 +2100,7 @@ def _find_spec_legacy(finder, name, path): def _find_spec(name, path, target=None): """Find a module's loader.""" - if not sys.meta_path: + if sys.meta_path is not None and not sys.meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) # We check sys.modules here for the reload case. While a passed-in # target will usually indicate a reload there is no guarantee, whereas @@ -2218,12 +2172,12 @@ def _find_and_load_unlocked(name, import_): path = parent_module.__path__ except AttributeError: msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) - raise ImportError(msg, name=name) + raise ImportError(msg, name=name) from None spec = _find_spec(name, path) if spec is None: raise ImportError(_ERR_MSG.format(name), name=name) else: - module = _SpecMethods(spec)._load_unlocked() + module = _load_unlocked(spec) if parent: # Set the module as an attribute on its parent. parent_module = sys.modules[parent] @@ -2358,8 +2312,7 @@ def _builtin_from_name(name): spec = BuiltinImporter.find_spec(name) if spec is None: raise ImportError('no built-in module named ' + name) - methods = _SpecMethods(spec) - return methods._load_unlocked() + return _load_unlocked(spec) def _setup(sys_module, _imp_module): @@ -2390,8 +2343,7 @@ def _setup(sys_module, _imp_module): else: continue spec = _spec_from_module(module, loader) - methods = _SpecMethods(spec) - methods.init_module_attrs(module) + _init_module_attrs(spec, module) # Directly load built-in modules needed during bootstrap. self_module = sys.modules[__name__] diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 558abd3..6b6a602 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -126,7 +126,7 @@ class Loader(metaclass=abc.ABCMeta): create_module() is optional. """ - # By default, defer to _SpecMethods.create() for the new module. + # By default, defer to default semantics for the new module. return None # We don't define exec_module() here since that would break @@ -217,7 +217,8 @@ class InspectLoader(Loader): """ raise ImportError - def source_to_code(self, data, path='<string>'): + @staticmethod + def source_to_code(data, path='<string>'): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 6d73b1d..c42ef14 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,8 +1,9 @@ """Utility code for constructing importers, etc.""" - +from . import abc from ._bootstrap import MAGIC_NUMBER from ._bootstrap import cache_from_source from ._bootstrap import decode_source +from ._bootstrap import module_from_spec from ._bootstrap import source_from_cache from ._bootstrap import spec_from_loader from ._bootstrap import spec_from_file_location @@ -12,6 +13,7 @@ from ._bootstrap import _find_spec from contextlib import contextmanager import functools import sys +import types import warnings @@ -54,7 +56,7 @@ def _find_spec_from_path(name, path=None): try: spec = module.__spec__ except AttributeError: - raise ValueError('{}.__spec__ is not set'.format(name)) + raise ValueError('{}.__spec__ is not set'.format(name)) from None else: if spec is None: raise ValueError('{}.__spec__ is None'.format(name)) @@ -94,7 +96,7 @@ def find_spec(name, package=None): try: spec = module.__spec__ except AttributeError: - raise ValueError('{}.__spec__ is not set'.format(name)) + raise ValueError('{}.__spec__ is not set'.format(name)) from None else: if spec is None: raise ValueError('{}.__spec__ is None'.format(name)) @@ -200,3 +202,94 @@ def module_for_loader(fxn): return fxn(self, module, *args, **kwargs) return module_for_loader_wrapper + + +class _Module(types.ModuleType): + + """A subclass of the module type to allow __class__ manipulation.""" + + +class _LazyModule(types.ModuleType): + + """A subclass of the module type which triggers loading upon attribute access.""" + + def __getattribute__(self, attr): + """Trigger the load of the module and return the attribute.""" + # All module metadata must be garnered from __spec__ in order to avoid + # using mutated values. + # Stop triggering this method. + self.__class__ = _Module + # Get the original name to make sure no object substitution occurred + # in sys.modules. + original_name = self.__spec__.name + # Figure out exactly what attributes were mutated between the creation + # of the module and now. + attrs_then = self.__spec__.loader_state + attrs_now = self.__dict__ + attrs_updated = {} + for key, value in attrs_now.items(): + # Code that set the attribute may have kept a reference to the + # assigned object, making identity more important than equality. + if key not in attrs_then: + attrs_updated[key] = value + elif id(attrs_now[key]) != id(attrs_then[key]): + attrs_updated[key] = value + self.__spec__.loader.exec_module(self) + # If exec_module() was used directly there is no guarantee the module + # object was put into sys.modules. + if original_name in sys.modules: + if id(self) != id(sys.modules[original_name]): + msg = ('module object for {!r} substituted in sys.modules ' + 'during a lazy load') + raise ValueError(msg.format(original_name)) + # Update after loading since that's what would happen in an eager + # loading situation. + self.__dict__.update(attrs_updated) + return getattr(self, attr) + + def __delattr__(self, attr): + """Trigger the load and then perform the deletion.""" + # To trigger the load and raise an exception if the attribute + # doesn't exist. + self.__getattribute__(attr) + delattr(self, attr) + + +class LazyLoader(abc.Loader): + + """A loader that creates a module which defers loading until attribute access.""" + + @staticmethod + def __check_eager_loader(loader): + if not hasattr(loader, 'exec_module'): + raise TypeError('loader must define exec_module()') + elif hasattr(loader.__class__, 'create_module'): + if abc.Loader.create_module != loader.__class__.create_module: + # Only care if create_module() is overridden in a subclass of + # importlib.abc.Loader. + raise TypeError('loader cannot define create_module()') + + @classmethod + def factory(cls, loader): + """Construct a callable which returns the eager loader made lazy.""" + cls.__check_eager_loader(loader) + return lambda *args, **kwargs: cls(loader(*args, **kwargs)) + + def __init__(self, loader): + self.__check_eager_loader(loader) + self.loader = loader + + def create_module(self, spec): + """Create a module which can have its __class__ manipulated.""" + return _Module(spec.name) + + def exec_module(self, module): + """Make the module load lazily.""" + module.__spec__.loader = self.loader + module.__loader__ = self.loader + # Don't need to worry about deep-copying as trying to set an attribute + # on an object would have triggered the load, + # e.g. ``module.__spec__.loader = None`` would trigger a load from + # trying to access module.__spec__. + module.__spec__.loader_state = module.__dict__.copy() + module.__class__ = _LazyModule |