PEP 489: Multi-phase extension module initialization

Known limitations of the current implementation:

- documentation changes are incomplete
- there's a reference leak I haven't tracked down yet

The leak is most visible by running:

  ./python -m test -R3:3 test_importlib

However, you can also see it by running:

  ./python -X showrefcount

Importing the array or _testmultiphase modules, and
then deleting them from both sys.modules and the local
namespace shows significant increases in the total
number of active references each cycle. By contrast,
with _testcapi (which continues to use single-phase
initialisation) the global refcounts stabilise after
a couple of cycles.

2 files changed, 28 insertions, 28 deletions

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 860703c..931754e 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -735,16 +735,17 @@ class BuiltinImporter:
         return spec.loader if spec is not None else None
 
     @classmethod
-    @_requires_builtin
-    def load_module(cls, fullname):
-        """Load a built-in module."""
-        # Once an exec_module() implementation is added we can also
-        # add a deprecation warning here.
-        with _ManageReload(fullname):
-            module = _call_with_frames_removed(_imp.init_builtin, fullname)
-        module.__loader__ = cls
-        module.__package__ = ''
-        return module
+    def create_module(self, spec):
+        """Create a built-in module"""
+        if spec.name not in sys.builtin_module_names:
+            raise ImportError('{!r} is not a built-in module'.format(spec.name),
+                              name=spec.name)
+        return _call_with_frames_removed(_imp.create_builtin, spec)
+
+    @classmethod
+    def exec_module(self, module):
+        """Exec a built-in module"""
+        _call_with_frames_removed(_imp.exec_dynamic, module)
 
     @classmethod
     @_requires_builtin
@@ -764,6 +765,8 @@ class BuiltinImporter:
         """Return False as built-in modules are never packages."""
         return False
 
+    load_module = classmethod(_load_module_shim)
+
 
 class FrozenImporter:
 
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index f176961..510fa92 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -378,7 +378,8 @@ def _check_name(method):
         if name is None:
             name = self.name
         elif self.name != name:
-            raise ImportError('loader cannot handle %s' % name, name=name)
+            raise ImportError('loader for %s cannot handle %s' %
+                                (self.name, name), name=name)
         return method(self, name, *args, **kwargs)
     try:
         _wrap = _bootstrap._wrap
@@ -875,7 +876,7 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
 EXTENSION_SUFFIXES = []
 
 
-class ExtensionFileLoader:
+class ExtensionFileLoader(FileLoader, _LoaderBasics):
 
     """Loader for extension modules.
 
@@ -894,24 +895,20 @@ class ExtensionFileLoader:
     def __hash__(self):
         return hash(self.name) ^ hash(self.path)
 
-    @_check_name
-    def load_module(self, fullname):
-        """Load an extension module."""
-        # Once an exec_module() implementation is added we can also
-        # add a deprecation warning here.
-        with _bootstrap._ManageReload(fullname):
-            module = _bootstrap._call_with_frames_removed(_imp.load_dynamic,
-                                                          fullname, self.path)
-        _verbose_message('extension module loaded from {!r}', self.path)
-        is_package = self.is_package(fullname)
-        if is_package and not hasattr(module, '__path__'):
-            module.__path__ = [_path_split(self.path)[0]]
-        module.__loader__ = self
-        module.__package__ = module.__name__
-        if not is_package:
-            module.__package__ = module.__package__.rpartition('.')[0]
+    def create_module(self, spec):
+        """Create an unitialized extension module"""
+        module = _bootstrap._call_with_frames_removed(
+            _imp.create_dynamic, spec)
+        _verbose_message('extension module {!r} loaded from {!r}',
+                         spec.name, self.path)
         return module
 
+    def exec_module(self, module):
+        """Initialize an extension module"""
+        _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
+        _verbose_message('extension module {!r} executed from {!r}',
+                         self.name, self.path)
+
     def is_package(self, fullname):
         """Return True if the extension module is a package."""
         file_name = _path_split(self.path)[1]