Issues #18088, 18089: Introduce

importlib.abc.Loader.init_module_attrs() and implement
importlib.abc.InspectLoader.load_module().

The importlib.abc.Loader.init_module_attrs() method sets the various
attributes on the module being loaded. It is done unconditionally to
support reloading. Typically people used
importlib.util.module_for_loader, but since that's a decorator there
was no way to override it's actions, so init_module_attrs() came into
existence to allow for overriding. This is also why module_for_loader
is now pending deprecation (having its other use replaced by
importlib.util.module_to_load).

All of this allowed for importlib.abc.InspectLoader.load_module() to
be implemented. At this point you can now implement a loader with
nothing more than get_code() (which only requires get_source();
package support requires is_package()). Thanks to init_module_attrs()
the implementation of load_module() is basically a context manager
containing 2 methods calls, a call to exec(), and a return statement.

3 files changed, 128 insertions, 74 deletions

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index d4d2f91..f8a1b0b 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -538,6 +538,32 @@ def module_to_load(name, *, reset_name=True):
     return _ModuleManager(name, reset_name=reset_name)
 
 
+def _init_package_attrs(loader, module):
+    """Set __package__ and __path__ based on what loader.is_package() says."""
+    name = module.__name__
+    try:
+        is_package = loader.is_package(name)
+    except ImportError:
+        pass
+    else:
+        if is_package:
+            module.__package__ = name
+            module.__path__ = []
+        else:
+            module.__package__ = name.rpartition('.')[0]
+
+
+def _init_file_attrs(loader, module):
+    """Set __file__ and __path__ based on loader.get_filename()."""
+    try:
+        module.__file__ = loader.get_filename(module.__name__)
+    except ImportError:
+        pass
+    else:
+        if module.__name__ == module.__package__:
+            module.__path__.append(_path_split(module.__file__)[0])
+
+
 def set_package(fxn):
     """Set __package__ on the returned module."""
     def set_package_wrapper(*args, **kwargs):
@@ -562,42 +588,6 @@ def set_loader(fxn):
     return set_loader_wrapper
 
 
-def module_for_loader(fxn):
-    """Decorator to handle selecting the proper module for loaders.
-
-    The decorated function is passed the module to use instead of the module
-    name. The module passed in to the function is either from sys.modules if
-    it already exists or is a new module. If the module is new, then __name__
-    is set the first argument to the method, __loader__ is set to self, and
-    __package__ is set accordingly (if self.is_package() is defined) will be set
-    before it is passed to the decorated function (if self.is_package() does
-    not work for the module it will be set post-load).
-
-    If an exception is raised and the decorator created the module it is
-    subsequently removed from sys.modules.
-
-    The decorator assumes that the decorated function takes the module name as
-    the second argument.
-
-    """
-    def module_for_loader_wrapper(self, fullname, *args, **kwargs):
-        with module_to_load(fullname) as module:
-            module.__loader__ = self
-            try:
-                is_package = self.is_package(fullname)
-            except (ImportError, AttributeError):
-                pass
-            else:
-                if is_package:
-                    module.__package__ = fullname
-                else:
-                    module.__package__ = fullname.rpartition('.')[0]
-            # If __package__ was not set above, __import__() will do it later.
-            return fxn(self, module, *args, **kwargs)
-    _wrap(module_for_loader_wrapper, fxn)
-    return module_for_loader_wrapper
-
-
 def _check_name(method):
     """Decorator to verify that the module being requested matches the one the
     loader can handle.
@@ -904,25 +894,32 @@ class _LoaderBasics:
         tail_name = fullname.rpartition('.')[2]
         return filename_base == '__init__' and tail_name != '__init__'
 
-    @module_for_loader
-    def _load_module(self, module, *, sourceless=False):
-        """Helper for load_module able to handle either source or sourceless
-        loading."""
-        name = module.__name__
-        code_object = self.get_code(name)
-        module.__file__ = self.get_filename(name)
-        if not sourceless:
+    def init_module_attrs(self, module):
+        """Set various attributes on the module.
+
+        ExecutionLoader.init_module_attrs() is used to set __loader__,
+        __package__, __file__, and optionally __path__. The __cached__ attribute
+        is set using imp.cache_from_source() and __file__.
+        """
+        module.__loader__ = self  # Loader
+        _init_package_attrs(self, module)  # InspectLoader
+        _init_file_attrs(self, module)  # ExecutionLoader
+        if hasattr(module, '__file__'):  # SourceLoader
             try:
                 module.__cached__ = cache_from_source(module.__file__)
             except NotImplementedError:
-                module.__cached__ = module.__file__
-        else:
-            module.__cached__ = module.__file__
-        if self.is_package(name):
-            module.__path__ = [_path_split(module.__file__)[0]]
-        # __package__ and __loader set by @module_for_loader.
-        _call_with_frames_removed(exec, code_object, module.__dict__)
-        return module
+                pass
+
+    def load_module(self, fullname):
+        """Load the specified module into sys.modules and return it."""
+        with module_to_load(fullname) as module:
+            self.init_module_attrs(module)
+            code = self.get_code(fullname)
+            if code is None:
+                raise ImportError('cannot load module {!r} when get_code() '
+                                  'returns None'.format(fullname))
+            _call_with_frames_removed(exec, code, module.__dict__)
+            return module
 
 
 class SourceLoader(_LoaderBasics):
@@ -1046,16 +1043,6 @@ class SourceLoader(_LoaderBasics):
                 pass
         return code_object
 
-    def load_module(self, fullname):
-        """Concrete implementation of Loader.load_module.
-
-        Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be
-        implemented to load source code. Use of bytecode is dictated by whether
-        get_code uses/writes bytecode.
-
-        """
-        return self._load_module(fullname)
-
 
 class FileLoader:
 
@@ -1133,8 +1120,9 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
 
     """Loader which handles sourceless file imports."""
 
-    def load_module(self, fullname):
-        return self._load_module(fullname, sourceless=True)
+    def init_module_attrs(self, module):
+        super().init_module_attrs(module)
+        module.__cached__ = module.__file__
 
     def get_code(self, fullname):
         path = self.get_filename(fullname)
@@ -1259,12 +1247,13 @@ class NamespaceLoader:
     def module_repr(cls, module):
         return "<module '{}' (namespace)>".format(module.__name__)
 
-    @module_for_loader
-    def load_module(self, module):
+    def load_module(self, fullname):
         """Load a namespace module."""
         _verbose_message('namespace module loaded with path {!r}', self._path)
-        module.__path__ = self._path
-        return module
+        with module_to_load(fullname) as module:
+            module.__path__ = self._path
+            module.__package__ = fullname
+            return module
 
 
 # Finders #####################################################################
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index 897f343..417fe41 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -8,11 +8,6 @@ except ImportError as exc:
         raise
     _frozen_importlib = None
 import abc
-import imp
-import marshal
-import sys
-import tokenize
-import warnings
 
 
 def _register(abstract_cls, *classes):
@@ -113,6 +108,10 @@ class Loader(metaclass=abc.ABCMeta):
         """
         raise NotImplementedError
 
+    def init_module_attrs(self, module):
+        """Set the module's __loader__ attribute."""
+        module.__loader__ = self
+
 
 class ResourceLoader(Loader):
 
@@ -177,6 +176,17 @@ class InspectLoader(Loader):
         argument should be where the data was retrieved (when applicable)."""
         return compile(data, path, 'exec', dont_inherit=True)
 
+    def init_module_attrs(self, module):
+        """Initialize the __loader__ and __package__ attributes of the module.
+
+        The name of the module is gleaned from module.__name__. The __package__
+        attribute is set based on self.is_package().
+        """
+        super().init_module_attrs(module)
+        _bootstrap._init_package_attrs(self, module)
+
+    load_module = _bootstrap._LoaderBasics.load_module
+
 _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter,
             machinery.ExtensionFileLoader)
 
@@ -215,6 +225,18 @@ class ExecutionLoader(InspectLoader):
         else:
             return self.source_to_code(source, path)
 
+    def init_module_attrs(self, module):
+        """Initialize the module's attributes.
+
+        It is assumed that the module's name has been set on module.__name__.
+        It is also assumed that any path returned by self.get_filename() uses
+        (one of) the operating system's path separator(s) to separate filenames
+        from directories in order to set __path__ intelligently.
+        InspectLoader.init_module_attrs() sets __loader__ and __package__.
+        """
+        super().init_module_attrs(module)
+        _bootstrap._init_file_attrs(self, module)
+
 
 class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
 
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index 74eef2a..573b08c 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -1,11 +1,13 @@
 """Utility code for constructing importers, etc."""
 
 from ._bootstrap import module_to_load
-from ._bootstrap import module_for_loader
 from ._bootstrap import set_loader
 from ._bootstrap import set_package
 from ._bootstrap import _resolve_name
 
+import functools
+import warnings
+
 
 def resolve_name(name, package):
     """Resolve a relative module name to an absolute one."""
@@ -20,3 +22,44 @@ def resolve_name(name, package):
             break
         level += 1
     return _resolve_name(name[level:], package, level)
+
+
+def module_for_loader(fxn):
+    """Decorator to handle selecting the proper module for loaders.
+
+    The decorated function is passed the module to use instead of the module
+    name. The module passed in to the function is either from sys.modules if
+    it already exists or is a new module. If the module is new, then __name__
+    is set the first argument to the method, __loader__ is set to self, and
+    __package__ is set accordingly (if self.is_package() is defined) will be set
+    before it is passed to the decorated function (if self.is_package() does
+    not work for the module it will be set post-load).
+
+    If an exception is raised and the decorator created the module it is
+    subsequently removed from sys.modules.
+
+    The decorator assumes that the decorated function takes the module name as
+    the second argument.
+
+    """
+    warnings.warn('To make it easier for subclasses, please use '
+                  'importlib.util.module_to_load() and '
+                  'importlib.abc.Loader.init_module_attrs()',
+                  PendingDeprecationWarning, stacklevel=2)
+    @functools.wraps(fxn)
+    def module_for_loader_wrapper(self, fullname, *args, **kwargs):
+        with module_to_load(fullname) as module:
+            module.__loader__ = self
+            try:
+                is_package = self.is_package(fullname)
+            except (ImportError, AttributeError):
+                pass
+            else:
+                if is_package:
+                    module.__package__ = fullname
+                else:
+                    module.__package__ = fullname.rpartition('.')[0]
+            # If __package__ was not set above, __import__() will do it later.
+            return fxn(self, module, *args, **kwargs)
+
+    return module_for_loader_wrapper
\ No newline at end of file