Issue #15576: Allow extension modules to be a package's __init__

module again. Also took the opportunity to stop accidentally exporting
_imp.extension_suffixes() as public.

10 files changed, 54 insertions, 45 deletions

diff --git a/Lib/imp.py b/Lib/imp.py
index 419f631..80b04c8 100644
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -9,7 +9,7 @@ functionality over this module.
 from _imp import (lock_held, acquire_lock, release_lock,
                   load_dynamic, get_frozen_object, is_frozen_package,
                   init_builtin, init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename, extension_suffixes)
+                  _fix_co_filename)
 
 # Directly exposed by this module
 from importlib._bootstrap import new_module
@@ -51,7 +51,7 @@ def get_suffixes():
     warnings.warn('imp.get_suffixes() is deprecated; use the constants '
                   'defined on importlib.machinery instead',
                   DeprecationWarning, 2)
-    extensions = [(s, 'rb', C_EXTENSION) for s in extension_suffixes()]
+    extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
     source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
     bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
 
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 7b12ab0..3562dfb 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -1067,6 +1067,10 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics):
         return None
 
 
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
 class ExtensionFileLoader:
 
     """Loader for extension modules.
@@ -1089,6 +1093,8 @@ class ExtensionFileLoader:
             module = _call_with_frames_removed(_imp.load_dynamic,
                                                fullname, self.path)
             _verbose_message('extension module loaded from {!r}', self.path)
+            if self.is_package(fullname):
+                module.__path__ = [_path_split(self.path)[0]]
             return module
         except:
             if not is_reload and fullname in sys.modules:
@@ -1097,7 +1103,12 @@ class ExtensionFileLoader:
 
     def is_package(self, fullname):
         """Return False as an extension module can never be a package."""
-        return False
+        file_name = _path_split(self.path)[1]
+        for suffix in EXTENSION_SUFFIXES:
+            if file_name == '__init__' + suffix:
+                return True
+        else:
+            return False
 
     def get_code(self, fullname):
         """Return None as an extension module cannot create a code object."""
@@ -1283,14 +1294,10 @@ class FileFinder:
         """Initialize with the path to search on and a variable number of
         3-tuples containing the loader, file suffixes the loader recognizes,
         and a boolean of whether the loader handles packages."""
-        packages = []
-        modules = []
-        for loader, suffixes, supports_packages in details:
-            modules.extend((suffix, loader) for suffix in suffixes)
-            if supports_packages:
-                packages.extend((suffix, loader) for suffix in suffixes)
-        self.packages = packages
-        self.modules = modules
+        loaders = []
+        for loader, suffixes in details:
+            loaders.extend((suffix, loader) for suffix in suffixes)
+        self._loaders = loaders
         # Base (directory) path
         self.path = path or '.'
         self._path_mtime = -1
@@ -1336,7 +1343,7 @@ class FileFinder:
         if cache_module in cache:
             base_path = _path_join(self.path, tail_module)
             if _path_isdir(base_path):
-                for suffix, loader in self.packages:
+                for suffix, loader in self._loaders:
                     init_filename = '__init__' + suffix
                     full_path = _path_join(base_path, init_filename)
                     if _path_isfile(full_path):
@@ -1346,7 +1353,7 @@ class FileFinder:
                     #  find a module in the next section.
                     is_namespace = True
         # Check for a file w/ a proper suffix exists.
-        for suffix, loader in self.modules:
+        for suffix, loader in self._loaders:
             if cache_module + suffix in cache:
                 full_path = _path_join(self.path, tail_module + suffix)
                 if _path_isfile(full_path):
@@ -1589,9 +1596,9 @@ def _get_supported_file_loaders():
 
     Each item is a tuple (loader, suffixes, allow_packages).
     """
-    extensions = ExtensionFileLoader, _imp.extension_suffixes(), False
-    source = SourceFileLoader, SOURCE_SUFFIXES, True
-    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True
+    extensions = ExtensionFileLoader, _imp.extension_suffixes()
+    source = SourceFileLoader, SOURCE_SUFFIXES
+    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
     return [extensions, source, bytecode]
 
 
@@ -1689,9 +1696,10 @@ def _setup(sys_module, _imp_module):
     setattr(self_module, 'path_separators', set(path_separators))
     # Constants
     setattr(self_module, '_relax_case', _make_relax_case())
+    EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
     if builtin_os == 'nt':
         SOURCE_SUFFIXES.append('.pyw')
-        if '_d.pyd' in _imp.extension_suffixes():
+        if '_d.pyd' in EXTENSION_SUFFIXES:
             WindowsRegistryFinder.DEBUG_BUILD = True
 
 
diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py
index fed6f2c..ff826e4 100644
--- a/Lib/importlib/machinery.py
+++ b/Lib/importlib/machinery.py
@@ -3,7 +3,8 @@
 import _imp
 
 from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
-                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES)
+                         OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
+                         EXTENSION_SUFFIXES)
 from ._bootstrap import BuiltinImporter
 from ._bootstrap import FrozenImporter
 from ._bootstrap import WindowsRegistryFinder
@@ -13,7 +14,6 @@ from ._bootstrap import SourceFileLoader
 from ._bootstrap import SourcelessFileLoader
 from ._bootstrap import ExtensionFileLoader
 
-EXTENSION_SUFFIXES = _imp.extension_suffixes()
 
 def all_suffixes():
     """Returns a list of all recognized module suffixes for this process"""
diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py
index bdc21e7..76c53e4 100644
--- a/Lib/test/test_importlib/extension/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py
@@ -16,8 +16,7 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
         assert good_name != bad_name
         finder = _bootstrap.FileFinder(ext_util.PATH,
                                         (_bootstrap.ExtensionFileLoader,
-                                            imp.extension_suffixes(),
-                                            False))
+                                         _bootstrap.EXTENSION_SUFFIXES))
         return finder.find_module(bad_name)
 
     def test_case_sensitive(self):
diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py
index 1c60292..a63cfdb 100644
--- a/Lib/test/test_importlib/extension/test_finder.py
+++ b/Lib/test/test_importlib/extension/test_finder.py
@@ -1,8 +1,7 @@
-from importlib import _bootstrap
+from importlib import machinery
 from .. import abc
 from . import util
 
-import imp
 import unittest
 
 class FinderTests(abc.FinderTests):
@@ -10,17 +9,16 @@ class FinderTests(abc.FinderTests):
     """Test the finder for extension modules."""
 
     def find_module(self, fullname):
-        importer = _bootstrap.FileFinder(util.PATH,
-                                          (_bootstrap.ExtensionFileLoader,
-                                              imp.extension_suffixes(),
-                                              False))
+        importer = machinery.FileFinder(util.PATH,
+                                        (machinery.ExtensionFileLoader,
+                                         machinery.EXTENSION_SUFFIXES))
         return importer.find_module(fullname)
 
     def test_module(self):
         self.assertTrue(self.find_module(util.NAME))
 
     def test_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_module_in_package(self):
@@ -28,7 +26,7 @@ class FinderTests(abc.FinderTests):
         pass
 
     def test_package_in_package(self):
-        # Extension modules cannot be an __init__ for a package.
+        # No extension module as an __init__ available for testing.
         pass
 
     def test_package_over_module(self):
@@ -38,8 +36,6 @@ class FinderTests(abc.FinderTests):
     def test_failure(self):
         self.assertIsNone(self.find_module('asdfjkl;'))
 
-    # XXX Raise an exception if someone tries to use the 'path' argument?
-
 
 def test_main():
     from test.support import run_unittest
diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py
index 917843f..ca5af20 100644
--- a/Lib/test/test_importlib/extension/test_loader.py
+++ b/Lib/test/test_importlib/extension/test_loader.py
@@ -3,6 +3,7 @@ from . import util as ext_util
 from .. import abc
 from .. import util
 
+import os.path
 import sys
 import unittest
 
@@ -38,11 +39,11 @@ class LoaderTests(abc.LoaderTests):
                                   machinery.ExtensionFileLoader)
 
     def test_package(self):
-        # Extensions are not found in packages.
+        # No extension module as __init__ available for testing.
         pass
 
     def test_lacking_parent(self):
-        # Extensions are not found in packages.
+        # No extension module in a package available for testing.
         pass
 
     def test_module_reuse(self):
@@ -61,6 +62,13 @@ class LoaderTests(abc.LoaderTests):
             self.load_module(name)
         self.assertEqual(cm.exception.name, name)
 
+    def test_is_package(self):
+        self.assertFalse(self.loader.is_package(ext_util.NAME))
+        for suffix in machinery.EXTENSION_SUFFIXES:
+            path = os.path.join('some', 'path', 'pkg', '__init__' + suffix)
+            loader = machinery.ExtensionFileLoader('pkg', path)
+            self.assertTrue(loader.is_package('pkg'))
+
 
 def test_main():
     from test.support import run_unittest
diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py
index 129e6e2..1d969a1 100644
--- a/Lib/test/test_importlib/extension/test_path_hook.py
+++ b/Lib/test/test_importlib/extension/test_path_hook.py
@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import machinery
 from . import util
 
 import collections
@@ -14,8 +14,8 @@ class PathHookTests(unittest.TestCase):
     # XXX Should it only work for directories containing an extension module?
 
     def hook(self, entry):
-        return _bootstrap.FileFinder.path_hook((_bootstrap.ExtensionFileLoader,
-            imp.extension_suffixes(), False))(entry)
+        return machinery.FileFinder.path_hook((machinery.ExtensionFileLoader,
+            machinery.EXTENSION_SUFFIXES))(entry)
 
     def test_success(self):
         # Path hook should handle a directory where a known extension module
diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
index 21a4378..241173f 100644
--- a/Lib/test/test_importlib/source/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -23,11 +23,9 @@ class CaseSensitivityTest(unittest.TestCase):
     def find(self, path):
         finder = machinery.FileFinder(path,
                                       (machinery.SourceFileLoader,
-                                            machinery.SOURCE_SUFFIXES,
-                                            True),
+                                            machinery.SOURCE_SUFFIXES),
                                         (machinery.SourcelessFileLoader,
-                                            machinery.BYTECODE_SUFFIXES,
-                                            True))
+                                            machinery.BYTECODE_SUFFIXES))
         return finder.find_module(self.name)
 
     def sensitivity_test(self):
diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py
index fa5d356..40905eb 100644
--- a/Lib/test/test_importlib/source/test_finder.py
+++ b/Lib/test/test_importlib/source/test_finder.py
@@ -37,9 +37,9 @@ class FinderTests(abc.FinderTests):
 
     def import_(self, root, module):
         loader_details = [(machinery.SourceFileLoader,
-                            machinery.SOURCE_SUFFIXES, True),
+                            machinery.SOURCE_SUFFIXES),
                           (machinery.SourcelessFileLoader,
-                            machinery.BYTECODE_SUFFIXES, True)]
+                            machinery.BYTECODE_SUFFIXES)]
         finder = machinery.FileFinder(root, *loader_details)
         return finder.find_module(module)
 
@@ -120,7 +120,7 @@ class FinderTests(abc.FinderTests):
     def test_empty_string_for_dir(self):
         # The empty string from sys.path means to search in the cwd.
         finder = machinery.FileFinder('', (machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
         with open('mod.py', 'w') as file:
             file.write("# test file for importlib")
         try:
@@ -132,7 +132,7 @@ class FinderTests(abc.FinderTests):
     def test_invalidate_caches(self):
         # invalidate_caches() should reset the mtime.
         finder = machinery.FileFinder('', (machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
         finder._path_mtime = 42
         finder.invalidate_caches()
         self.assertEqual(finder._path_mtime, -1)
diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py
index 54c0699..6a78792 100644
--- a/Lib/test/test_importlib/source/test_path_hook.py
+++ b/Lib/test/test_importlib/source/test_path_hook.py
@@ -11,7 +11,7 @@ class PathHookTest(unittest.TestCase):
 
     def path_hook(self):
         return machinery.FileFinder.path_hook((machinery.SourceFileLoader,
-            machinery.SOURCE_SUFFIXES, True))
+            machinery.SOURCE_SUFFIXES))
 
     def test_success(self):
         with source_util.create_modules('dummy') as mapping: