Issue #26186: Remove the restriction that built-in and extension

modules  can't be lazily loaded.

Thanks to Python 3.6 allowing for types.ModuleType to have its
__class__ mutated, the restriction can be lifted by calling
create_module() on the wrapped loader.

4 files changed, 50 insertions, 27 deletions

diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst
index 656d51a..c1c3b12 100644
--- a/Doc/library/importlib.rst
+++ b/Doc/library/importlib.rst
@@ -379,10 +379,14 @@ ABC hierarchy::
 
        An abstract method that executes the module in its own namespace
        when a module is imported or reloaded.  The module should already
-       be initialized when exec_module() is called.
+       be initialized when ``exec_module()`` is called. When this method exists,
+       :meth:`~importlib.abc.Loader.create_module` must be defined.
 
        .. versionadded:: 3.4
 
+       .. versionchanged:: 3.6
+          :meth:`~importlib.abc.Loader.create_module` must also be defined.
+
     .. method:: load_module(fullname)
 
         A legacy method for loading a module. If the module cannot be
@@ -1200,12 +1204,13 @@ an :term:`importer`.
 
 .. function:: module_from_spec(spec)
 
-   Create a new module based on **spec** and ``spec.loader.create_module()``.
+   Create a new module based on **spec** and
+   :meth:`spec.loader.create_module <importlib.abc.Loader.create_module>`.
 
-   If ``spec.loader.create_module()`` does not return ``None``, then any
-   pre-existing attributes will not be reset. Also, no :exc:`AttributeError`
-   will be raised if triggered while accessing **spec** or setting an attribute
-   on the module.
+   If :meth:`spec.loader.create_module <importlib.abc.Loader.create_module>`
+   does not return ``None``, then any pre-existing attributes will not be reset.
+   Also, no :exc:`AttributeError` will be raised if triggered while accessing
+   **spec** or setting an attribute on the module.
 
    This function is preferred over using :class:`types.ModuleType` to create a
    new module as **spec** is used to set as many import-controlled attributes on
@@ -1267,7 +1272,8 @@ an :term:`importer`.
 
 .. decorator:: set_package
 
-   A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the :attr:`__package__` attribute on the returned module. If :attr:`__package__`
+   A :term:`decorator` for :meth:`importlib.abc.Loader.load_module` to set the
+   :attr:`__package__` attribute on the returned module. If :attr:`__package__`
    is set and has a value other than ``None`` it will not be changed.
 
    .. deprecated:: 3.4
@@ -1300,13 +1306,12 @@ an :term:`importer`.
    This class **only** works with loaders that define
    :meth:`~importlib.abc.Loader.exec_module` as control over what module type
    is used for the module is required. For those same reasons, the loader's
-   :meth:`~importlib.abc.Loader.create_module` method will be ignored (i.e., the
-   loader's method should only return ``None``; this excludes
-   :class:`BuiltinImporter` and :class:`ExtensionFileLoader`). Finally,
-   modules which substitute the object placed into :attr:`sys.modules` will
-   not work as there is no way to properly replace the module references
-   throughout the interpreter safely; :exc:`ValueError` is raised if such a
-   substitution is detected.
+   :meth:`~importlib.abc.Loader.create_module` method must return ``None`` or a
+   type for which its ``__class__`` attribute can be mutated along with not
+   using :term:`slots <__slots__>`. Finally, modules which substitute the object
+   placed into :attr:`sys.modules` will not work as there is no way to properly
+   replace the module references throughout the interpreter safely;
+   :exc:`ValueError` is raised if such a substitution is detected.
 
    .. note::
       For projects where startup time is critical, this class allows for
@@ -1317,6 +1322,11 @@ an :term:`importer`.
 
    .. versionadded:: 3.5
 
+   .. versionchanged:: 3.6
+      Began calling :meth:`~importlib.abc.Loader.create_module`, removing the
+      compatibility warning for :class:`importlib.machinery.BuiltinImporter` and
+      :class:`importlib.machinery.ExtensionFileLoader`.
+
    .. classmethod:: factory(loader)
 
       A static method which returns a callable that creates a lazy loader. This
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
index 177b17f..ea4dd8c 100644
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -291,6 +291,16 @@ The idlelib package is being modernized and refactored to make IDLE look and wor
 In compensation, the eventual result with be that some idlelib classes will be easier to use, with better APIs and docstrings explaining them.  Additional useful information will be added to idlelib when available.
 
 
+importlib
+---------
+
+:class:`importlib.util.LazyLoader` now calls
+:meth:`~importlib.abc.Loader.create_module` on the wrapped loader, removing the
+restriction that :class:`importlib.machinery.BuiltinImporter` and
+:class:`importlib.machinery.ExtensionFileLoader` couldn't be used with
+:class:`importlib.util.LazyLoader`.
+
+
 os
 --
 
@@ -620,6 +630,9 @@ that may require changes to your code.
 Changes in the Python API
 -------------------------
 
+* When :meth:`importlib.abc.Loader.exec_module` is defined,
+  :meth:`importlib.abc.Loader.create_module` must also be defined.
+
 * The format of the ``co_lnotab`` attribute of code objects changed to support
   negative line number delta. By default, Python does not emit bytecode with
   negative line number delta. Functions using ``frame.f_lineno``,
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index a9d0f1e..6bdf0d4 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -204,11 +204,6 @@ def module_for_loader(fxn):
     return module_for_loader_wrapper
 
 
-class _Module(types.ModuleType):
-
-    """A subclass of the module type to allow __class__ manipulation."""
-
-
 class _LazyModule(types.ModuleType):
 
     """A subclass of the module type which triggers loading upon attribute access."""
@@ -218,13 +213,14 @@ class _LazyModule(types.ModuleType):
         # All module metadata must be garnered from __spec__ in order to avoid
         # using mutated values.
         # Stop triggering this method.
-        self.__class__ = _Module
+        self.__class__ = types.ModuleType
         # Get the original name to make sure no object substitution occurred
         # in sys.modules.
         original_name = self.__spec__.name
         # Figure out exactly what attributes were mutated between the creation
         # of the module and now.
-        attrs_then = self.__spec__.loader_state
+        attrs_then = self.__spec__.loader_state['__dict__']
+        original_type = self.__spec__.loader_state['__class__']
         attrs_now = self.__dict__
         attrs_updated = {}
         for key, value in attrs_now.items():
@@ -239,9 +235,9 @@ class _LazyModule(types.ModuleType):
         # object was put into sys.modules.
         if original_name in sys.modules:
             if id(self) != id(sys.modules[original_name]):
-                msg = ('module object for {!r} substituted in sys.modules '
-                       'during a lazy load')
-                raise ValueError(msg.format(original_name))
+                raise ValueError(f"module object for {original_name!r} "
+                                  "substituted in sys.modules during a lazy "
+                                  "load")
         # Update after loading since that's what would happen in an eager
         # loading situation.
         self.__dict__.update(attrs_updated)
@@ -275,8 +271,7 @@ class LazyLoader(abc.Loader):
         self.loader = loader
 
     def create_module(self, spec):
-        """Create a module which can have its __class__ manipulated."""
-        return _Module(spec.name)
+        return self.loader.create_module(spec)
 
     def exec_module(self, module):
         """Make the module load lazily."""
@@ -286,5 +281,8 @@ class LazyLoader(abc.Loader):
         # on an object would have triggered the load,
         # e.g. ``module.__spec__.loader = None`` would trigger a load from
         # trying to access module.__spec__.
-        module.__spec__.loader_state = module.__dict__.copy()
+        loader_state = {}
+        loader_state['__dict__'] = module.__dict__.copy()
+        loader_state['__class__'] = module.__class__
+        module.__spec__.loader_state = loader_state
         module.__class__ = _LazyModule
diff --git a/Lib/test/test_importlib/test_lazy.py b/Lib/test/test_importlib/test_lazy.py
index cc383c2..ffd8dc6 100644
--- a/Lib/test/test_importlib/test_lazy.py
+++ b/Lib/test/test_importlib/test_lazy.py
@@ -66,6 +66,8 @@ class LazyLoaderTests(unittest.TestCase):
         spec = util.spec_from_loader(TestingImporter.module_name,
                                      util.LazyLoader(loader))
         module = spec.loader.create_module(spec)
+        if module is None:
+            module = types.ModuleType(TestingImporter.module_name)
         module.__spec__ = spec
         module.__loader__ = spec.loader
         spec.loader.exec_module(module)