Issue 19944: Fix importlib.find_spec() so it imports parents as needed.

The function is also moved to importlib.util.

3 files changed, 182 insertions, 150 deletions

diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py
index c3c19f4..744001b 100644
--- a/Lib/test/test_importlib/test_api.py
+++ b/Lib/test/test_importlib/test_api.py
@@ -4,7 +4,6 @@ frozen_init, source_init = util.import_importlib('importlib')
 frozen_util, source_util = util.import_importlib('importlib.util')
 frozen_machinery, source_machinery = util.import_importlib('importlib.machinery')
 
-from contextlib import contextmanager
 import os.path
 import sys
 from test import support
@@ -13,37 +12,6 @@ import unittest
 import warnings
 
 
-@contextmanager
-def temp_module(name, content='', *, pkg=False):
-    conflicts = [n for n in sys.modules if n.partition('.')[0] == name]
-    with support.temp_cwd(None) as cwd:
-        with util.uncache(name, *conflicts):
-            with support.DirsOnSysPath(cwd):
-                frozen_init.invalidate_caches()
-
-                location = os.path.join(cwd, name)
-                if pkg:
-                    modpath = os.path.join(location, '__init__.py')
-                    os.mkdir(name)
-                else:
-                    modpath = location + '.py'
-                    if content is None:
-                        # Make sure the module file gets created.
-                        content = ''
-                if content is not None:
-                    # not a namespace package
-                    with open(modpath, 'w') as modfile:
-                        modfile.write(content)
-                yield location
-
-
-def submodule(parent, name, pkg_dir, content=''):
-    path = os.path.join(pkg_dir, name + '.py')
-    with open(path, 'w') as subfile:
-        subfile.write(content)
-    return '{}.{}'.format(parent, name), path
-
-
 class ImportModuleTests:
 
     """Test importlib.import_module."""
@@ -210,121 +178,6 @@ class Source_FindLoaderTests(FindLoaderTests, unittest.TestCase):
     init = source_init
 
 
-class FindSpecTests:
-
-    class FakeMetaFinder:
-        @staticmethod
-        def find_spec(name, path=None, target=None): return name, path, target
-
-    def test_sys_modules(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            module = types.ModuleType(name)
-            loader = 'a loader!'
-            spec = self.machinery.ModuleSpec(name, loader)
-            module.__loader__ = loader
-            module.__spec__ = spec
-            sys.modules[name] = module
-            found = self.init.find_spec(name)
-            self.assertEqual(found, spec)
-
-    def test_sys_modules_without___loader__(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            module = types.ModuleType(name)
-            del module.__loader__
-            loader = 'a loader!'
-            spec = self.machinery.ModuleSpec(name, loader)
-            module.__spec__ = spec
-            sys.modules[name] = module
-            found = self.init.find_spec(name)
-            self.assertEqual(found, spec)
-
-    def test_sys_modules_spec_is_None(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            module = types.ModuleType(name)
-            module.__spec__ = None
-            sys.modules[name] = module
-            with self.assertRaises(ValueError):
-                self.init.find_spec(name)
-
-    def test_sys_modules_loader_is_None(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            module = types.ModuleType(name)
-            spec = self.machinery.ModuleSpec(name, None)
-            module.__spec__ = spec
-            sys.modules[name] = module
-            found = self.init.find_spec(name)
-            self.assertEqual(found, spec)
-
-    def test_sys_modules_spec_is_not_set(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            module = types.ModuleType(name)
-            try:
-                del module.__spec__
-            except AttributeError:
-                pass
-            sys.modules[name] = module
-            with self.assertRaises(ValueError):
-                self.init.find_spec(name)
-
-    def test_success(self):
-        name = 'some_mod'
-        with util.uncache(name):
-            with util.import_state(meta_path=[self.FakeMetaFinder]):
-                self.assertEqual((name, None, None),
-                                 self.init.find_spec(name))
-
-    def test_success_path(self):
-        # Searching on a path should work.
-        name = 'some_mod'
-        path = 'path to some place'
-        with util.uncache(name):
-            with util.import_state(meta_path=[self.FakeMetaFinder]):
-                self.assertEqual((name, path, None),
-                                 self.init.find_spec(name, path))
-
-    def test_nothing(self):
-        # None is returned upon failure to find a loader.
-        self.assertIsNone(self.init.find_spec('nevergoingtofindthismodule'))
-
-    def test_find_submodule(self):
-        name = 'spam'
-        subname = 'ham'
-        with temp_module(name, pkg=True) as pkg_dir:
-            fullname, _ = submodule(name, subname, pkg_dir)
-            spec = self.init.find_spec(fullname, [pkg_dir])
-            self.assertIsNot(spec, None)
-            self.assertNotIn(name, sorted(sys.modules))
-            # Ensure successive calls behave the same.
-            spec_again = self.init.find_spec(fullname, [pkg_dir])
-            self.assertEqual(spec_again, spec)
-
-    def test_find_submodule_missing_path(self):
-        name = 'spam'
-        subname = 'ham'
-        with temp_module(name, pkg=True) as pkg_dir:
-            fullname, _ = submodule(name, subname, pkg_dir)
-            spec = self.init.find_spec(fullname)
-            self.assertIs(spec, None)
-            self.assertNotIn(name, sorted(sys.modules))
-            # Ensure successive calls behave the same.
-            spec = self.init.find_spec(fullname)
-            self.assertIs(spec, None)
-
-
-class Frozen_FindSpecTests(FindSpecTests, unittest.TestCase):
-    init = frozen_init
-    machinery = frozen_machinery
-
-class Source_FindSpecTests(FindSpecTests, unittest.TestCase):
-    init = source_init
-    machinery = source_machinery
-
-
 class ReloadTests:
 
     """Test module reloading for builtin and extension modules."""
@@ -484,8 +337,8 @@ class ReloadTests:
         # See #19851.
         name = 'spam'
         subname = 'ham'
-        with temp_module(name, pkg=True) as pkg_dir:
-            fullname, _ = submodule(name, subname, pkg_dir)
+        with util.temp_module(name, pkg=True) as pkg_dir:
+            fullname, _ = util.submodule(name, subname, pkg_dir)
             ham = self.init.import_module(fullname)
             reloaded = self.init.reload(ham)
             self.assertIs(reloaded, ham)
diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py
index b6fe864..b2823c6 100644
--- a/Lib/test/test_importlib/test_util.py
+++ b/Lib/test/test_importlib/test_util.py
@@ -1,5 +1,7 @@
 from importlib import util
 from . import util as test_util
+frozen_init, source_init = test_util.import_importlib('importlib')
+frozen_machinery, source_machinery = test_util.import_importlib('importlib.machinery')
 frozen_util, source_util = test_util.import_importlib('importlib.util')
 
 import os
@@ -310,6 +312,151 @@ Frozen_ResolveNameTests, Source_ResolveNameTests = test_util.test_both(
         util=[frozen_util, source_util])
 
 
+class FindSpecTests:
+
+    class FakeMetaFinder:
+        @staticmethod
+        def find_spec(name, path=None, target=None): return name, path, target
+
+    def test_sys_modules(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            loader = 'a loader!'
+            spec = self.machinery.ModuleSpec(name, loader)
+            module.__loader__ = loader
+            module.__spec__ = spec
+            sys.modules[name] = module
+            found = self.util.find_spec(name)
+            self.assertEqual(found, spec)
+
+    def test_sys_modules_without___loader__(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            del module.__loader__
+            loader = 'a loader!'
+            spec = self.machinery.ModuleSpec(name, loader)
+            module.__spec__ = spec
+            sys.modules[name] = module
+            found = self.util.find_spec(name)
+            self.assertEqual(found, spec)
+
+    def test_sys_modules_spec_is_None(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            module.__spec__ = None
+            sys.modules[name] = module
+            with self.assertRaises(ValueError):
+                self.util.find_spec(name)
+
+    def test_sys_modules_loader_is_None(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            spec = self.machinery.ModuleSpec(name, None)
+            module.__spec__ = spec
+            sys.modules[name] = module
+            found = self.util.find_spec(name)
+            self.assertEqual(found, spec)
+
+    def test_sys_modules_spec_is_not_set(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            module = types.ModuleType(name)
+            try:
+                del module.__spec__
+            except AttributeError:
+                pass
+            sys.modules[name] = module
+            with self.assertRaises(ValueError):
+                self.util.find_spec(name)
+
+    def test_success(self):
+        name = 'some_mod'
+        with test_util.uncache(name):
+            with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+                self.assertEqual((name, None, None),
+                                 self.util.find_spec(name))
+
+#    def test_success_path(self):
+#        # Searching on a path should work.
+#        name = 'some_mod'
+#        path = 'path to some place'
+#        with test_util.uncache(name):
+#            with test_util.import_state(meta_path=[self.FakeMetaFinder]):
+#                self.assertEqual((name, path, None),
+#                                 self.util.find_spec(name, path))
+
+    def test_nothing(self):
+        # None is returned upon failure to find a loader.
+        self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule'))
+
+    def test_find_submodule(self):
+        name = 'spam'
+        subname = 'ham'
+        with test_util.temp_module(name, pkg=True) as pkg_dir:
+            fullname, _ = test_util.submodule(name, subname, pkg_dir)
+            spec = self.util.find_spec(fullname)
+            self.assertIsNot(spec, None)
+            self.assertIn(name, sorted(sys.modules))
+            self.assertNotIn(fullname, sorted(sys.modules))
+            # Ensure successive calls behave the same.
+            spec_again = self.util.find_spec(fullname)
+            self.assertEqual(spec_again, spec)
+
+    def test_find_submodule_parent_already_imported(self):
+        name = 'spam'
+        subname = 'ham'
+        with test_util.temp_module(name, pkg=True) as pkg_dir:
+            self.init.import_module(name)
+            fullname, _ = test_util.submodule(name, subname, pkg_dir)
+            spec = self.util.find_spec(fullname)
+            self.assertIsNot(spec, None)
+            self.assertIn(name, sorted(sys.modules))
+            self.assertNotIn(fullname, sorted(sys.modules))
+            # Ensure successive calls behave the same.
+            spec_again = self.util.find_spec(fullname)
+            self.assertEqual(spec_again, spec)
+
+    def test_find_relative_module(self):
+        name = 'spam'
+        subname = 'ham'
+        with test_util.temp_module(name, pkg=True) as pkg_dir:
+            fullname, _ = test_util.submodule(name, subname, pkg_dir)
+            relname = '.' + subname
+            spec = self.util.find_spec(relname, name)
+            self.assertIsNot(spec, None)
+            self.assertIn(name, sorted(sys.modules))
+            self.assertNotIn(fullname, sorted(sys.modules))
+            # Ensure successive calls behave the same.
+            spec_again = self.util.find_spec(fullname)
+            self.assertEqual(spec_again, spec)
+
+    def test_find_relative_module_missing_package(self):
+        name = 'spam'
+        subname = 'ham'
+        with test_util.temp_module(name, pkg=True) as pkg_dir:
+            fullname, _ = test_util.submodule(name, subname, pkg_dir)
+            relname = '.' + subname
+            with self.assertRaises(ValueError):
+                self.util.find_spec(relname)
+            self.assertNotIn(name, sorted(sys.modules))
+            self.assertNotIn(fullname, sorted(sys.modules))
+
+
+class Frozen_FindSpecTests(FindSpecTests, unittest.TestCase):
+    init = frozen_init
+    machinery = frozen_machinery
+    util = frozen_util
+
+class Source_FindSpecTests(FindSpecTests, unittest.TestCase):
+    init = source_init
+    machinery = source_machinery
+    util = source_util
+
+
 class MagicNumberTests:
 
     def test_length(self):
diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py
index a0dee6e..885cec3 100644
--- a/Lib/test/test_importlib/util.py
+++ b/Lib/test/test_importlib/util.py
@@ -1,5 +1,5 @@
 from contextlib import contextmanager
-from importlib import util
+from importlib import util, invalidate_caches
 import os.path
 from test import support
 import unittest
@@ -46,6 +46,13 @@ def case_insensitive_tests(test):
                             "requires a case-insensitive filesystem")(test)
 
 
+def submodule(parent, name, pkg_dir, content=''):
+    path = os.path.join(pkg_dir, name + '.py')
+    with open(path, 'w') as subfile:
+        subfile.write(content)
+    return '{}.{}'.format(parent, name), path
+
+
 @contextmanager
 def uncache(*names):
     """Uncache a module from sys.modules.
@@ -71,6 +78,31 @@ def uncache(*names):
             except KeyError:
                 pass
 
+
+@contextmanager
+def temp_module(name, content='', *, pkg=False):
+    conflicts = [n for n in sys.modules if n.partition('.')[0] == name]
+    with support.temp_cwd(None) as cwd:
+        with uncache(name, *conflicts):
+            with support.DirsOnSysPath(cwd):
+                invalidate_caches()
+
+                location = os.path.join(cwd, name)
+                if pkg:
+                    modpath = os.path.join(location, '__init__.py')
+                    os.mkdir(name)
+                else:
+                    modpath = location + '.py'
+                    if content is None:
+                        # Make sure the module file gets created.
+                        content = ''
+                if content is not None:
+                    # not a namespace package
+                    with open(modpath, 'w') as modfile:
+                        modfile.write(content)
+                yield location
+
+
 @contextmanager
 def import_state(**kwargs):
     """Context manager to manage the various importers and stored state in the