Issue #2377: Make importlib the implementation of __import__().

importlib._bootstrap is now frozen into Python/importlib.h and stored
as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen
code along with sys and imp and then uses _frozen_importlib._install()
to set builtins.__import__() w/ _frozen_importlib.__import__().

7 files changed, 57 insertions, 27 deletions

diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py
index 5243ebb..dbd229b 100644
--- a/Lib/test/test_frozen.py
+++ b/Lib/test/test_frozen.py
@@ -5,6 +5,12 @@ import unittest
 import sys
 
 class FrozenTests(unittest.TestCase):
+
+    module_attrs = frozenset(['__builtins__', '__cached__', '__doc__',
+                              '__file__', '__loader__', '__name__',
+                              '__package__'])
+    package_attrs = frozenset(list(module_attrs) + ['__path__'])
+
     def test_frozen(self):
         with captured_stdout() as stdout:
             try:
@@ -12,7 +18,9 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __hello__ failed:" + str(x))
             self.assertEqual(__hello__.initialized, True)
-            self.assertEqual(len(dir(__hello__)), 7, dir(__hello__))
+            expect = set(self.module_attrs)
+            expect.add('initialized')
+            self.assertEqual(set(dir(__hello__)), expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         with captured_stdout() as stdout:
@@ -21,10 +29,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__ failed:" + str(x))
             self.assertEqual(__phello__.initialized, True)
+            expect = set(self.package_attrs)
+            expect.add('initialized')
             if not "__phello__.spam" in sys.modules:
-                self.assertEqual(len(dir(__phello__)), 8, dir(__phello__))
+                self.assertEqual(set(dir(__phello__)), expect)
             else:
-                self.assertEqual(len(dir(__phello__)), 9, dir(__phello__))
+                expect.add('spam')
+                self.assertEqual(set(dir(__phello__)), expect)
             self.assertEqual(__phello__.__path__, [__phello__.__name__])
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
@@ -34,8 +45,13 @@ class FrozenTests(unittest.TestCase):
             except ImportError as x:
                 self.fail("import __phello__.spam failed:" + str(x))
             self.assertEqual(__phello__.spam.initialized, True)
-            self.assertEqual(len(dir(__phello__.spam)), 7)
-            self.assertEqual(len(dir(__phello__)), 9)
+            spam_expect = set(self.module_attrs)
+            spam_expect.add('initialized')
+            self.assertEqual(set(dir(__phello__.spam)), spam_expect)
+            phello_expect = set(self.package_attrs)
+            phello_expect.add('initialized')
+            phello_expect.add('spam')
+            self.assertEqual(set(dir(__phello__)), phello_expect)
             self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
         try:
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index bd2da72..87409fc 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -73,6 +73,7 @@ class ImportTests(unittest.TestCase):
 
             if TESTFN in sys.modules:
                 del sys.modules[TESTFN]
+            importlib.invalidate_caches()
             try:
                 try:
                     mod = __import__(TESTFN)
@@ -402,6 +403,7 @@ func_filename = func.__code__.co_filename
         py_compile.compile(self.file_name, dfile=target)
         os.remove(self.file_name)
         pyc_file = make_legacy_pyc(self.file_name)
+        importlib.invalidate_caches()
         mod = self.import_module()
         self.assertEqual(mod.module_filename, pyc_file)
         self.assertEqual(mod.code_filename, target)
@@ -509,7 +511,7 @@ class RelativeImportTests(unittest.TestCase):
 
         # Check relative import fails with package set to a non-string
         ns = dict(__package__=object())
-        self.assertRaises(ValueError, check_relative)
+        self.assertRaises(TypeError, check_relative)
 
     def test_absolute_import_without_future(self):
         # If explicit relative import syntax is used, then do not try
@@ -644,6 +646,7 @@ class PycacheTests(unittest.TestCase):
             pass
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
@@ -666,9 +669,11 @@ class PycacheTests(unittest.TestCase):
             pass
         with open(os.path.join('pep3147', 'foo.py'), 'w'):
             pass
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         unload('pep3147.foo')
         unload('pep3147')
+        importlib.invalidate_caches()
         m = __import__('pep3147.foo')
         init_pyc = imp.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py
index a4ddb15..4d0eee8 100644
--- a/Lib/test/test_pkg.py
+++ b/Lib/test/test_pkg.py
@@ -196,14 +196,15 @@ class TestPkg(unittest.TestCase):
 
         import t5
         self.assertEqual(fixdir(dir(t5)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__', 'foo', 'string', 't5'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'foo',
+                          'string', 't5'])
         self.assertEqual(fixdir(dir(t5.foo)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'string'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'string'])
         self.assertEqual(fixdir(dir(t5.string)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', 'spam'])
 
     def test_6(self):
         hier = [
@@ -219,14 +220,14 @@ class TestPkg(unittest.TestCase):
         import t6
         self.assertEqual(fixdir(dir(t6)),
                          ['__all__', '__cached__', '__doc__', '__file__',
-                          '__name__', '__package__', '__path__'])
+                          '__loader__', '__name__', '__package__', '__path__'])
         s = """
             import t6
             from t6 import *
             self.assertEqual(fixdir(dir(t6)),
                              ['__all__', '__cached__', '__doc__', '__file__',
-                              '__name__', '__package__', '__path__',
-                              'eggs', 'ham', 'spam'])
+                              '__loader__', '__name__', '__package__',
+                              '__path__', 'eggs', 'ham', 'spam'])
             self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6'])
             """
         self.run_code(s)
@@ -252,19 +253,19 @@ class TestPkg(unittest.TestCase):
         t7, sub, subsub = None, None, None
         import t7 as tas
         self.assertEqual(fixdir(dir(tas)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         from t7 import sub as subpar
         self.assertEqual(fixdir(dir(subpar)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                          '__package__', '__path__'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         from t7.sub import subsub as subsubsub
         self.assertEqual(fixdir(dir(subsubsub)),
-                         ['__cached__', '__doc__', '__file__', '__name__',
-                         '__package__', '__path__', 'spam'])
+                         ['__cached__', '__doc__', '__file__', '__loader__',
+                          '__name__', '__package__', '__path__', 'spam'])
         self.assertFalse(t7)
         self.assertFalse(sub)
         self.assertFalse(subsub)
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
index e805ed8..a9f75b9 100644
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -383,11 +383,10 @@ class PydocImportTest(unittest.TestCase):
         modname = 'testmod_xyzzy'
         testpairs = (
             ('i_am_not_here', 'i_am_not_here'),
-            ('test.i_am_not_here_either', 'i_am_not_here_either'),
-            ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'),
-            ('i_am_not_here.{}'.format(modname),
-             'i_am_not_here.{}'.format(modname)),
-            ('test.{}'.format(modname), modname),
+            ('test.i_am_not_here_either', 'test.i_am_not_here_either'),
+            ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'),
+            ('i_am_not_here.{}'.format(modname), 'i_am_not_here'),
+            ('test.{}'.format(modname), 'test.{}'.format(modname)),
             )
 
         sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py"
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index 2cede19..de44df0 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,6 +5,7 @@ import os.path
 import sys
 import re
 import tempfile
+import importlib
 import py_compile
 from test.support import (
     forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -172,11 +173,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertEqual(d1["x"], 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertEqual(d2["x"], 1)
@@ -196,11 +199,13 @@ class RunModuleTest(unittest.TestCase):
             self.assertIn("x", d1)
             self.assertTrue(d1["x"] == 1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case loader caches paths
             if verbose: print("Running from compiled:", pkg_name)
+            importlib.invalidate_caches()
             d2 = run_module(pkg_name) # Read from bytecode
             self.assertIn("x", d2)
             self.assertTrue(d2["x"] == 1)
@@ -250,11 +255,13 @@ from ..uncle.cousin import nephew
             self.assertIn("sibling", d1)
             self.assertIn("nephew", d1)
             del d1 # Ensure __loader__ entry doesn't keep file open
+            importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
             make_legacy_pyc(mod_fname)
             unload(mod_name)  # In case the loader caches paths
             if verbose: print("Running from compiled:", mod_name)
+            importlib.invalidate_caches()
             d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
             self.assertIn("__package__", d2)
             self.assertTrue(d2["__package__"] == pkg_name)
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
index 394e210..14fcdbf 100644
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import importlib
 import sys
 import os
 import unittest
@@ -59,6 +60,7 @@ class TestSupport(unittest.TestCase):
         with open(mod_filename, 'w') as f:
             print('foo = 1', file=f)
         sys.path.insert(0, os.curdir)
+        importlib.invalidate_caches()
         try:
             mod = __import__(TESTFN)
             self.assertIn(TESTFN, sys.modules)
diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
index d9bef38..fa0d48c 100644
--- a/Lib/test/test_trace.py
+++ b/Lib/test/test_trace.py
@@ -322,7 +322,7 @@ class TestCoverage(unittest.TestCase):
             self._coverage(tracer)
         if os.path.exists(TESTFN):
             files = os.listdir(TESTFN)
-            self.assertEqual(files, [])
+            self.assertEqual(files, ['_importlib.cover'])  # Ignore __import__
 
     def test_issue9936(self):
         tracer = trace.Trace(trace=0, count=1)