bpo-45020: Add -X frozen_modules=[on|off] to explicitly control use of frozen modules. (gh-28320)

Currently we freeze several modules into the runtime. For each of these modules it is essential to bootstrapping the runtime that they be frozen. Any other stdlib module that we later freeze into the runtime is not essential. We can just as well import from the .py file.  This PR lets users explicitly choose which should be used, with the new "-X frozen_modules=[on|off]" CLI flag. The default is "off" for now.

https://bugs.python.org/issue45020

2 files changed, 123 insertions, 96 deletions

diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
index eb7a4d2..fbc3fc0 100644
--- a/Lib/test/test_importlib/frozen/test_finder.py
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -6,6 +6,8 @@ machinery = util.import_importlib('importlib.machinery')
 import unittest
 import warnings
 
+from test.support import import_helper
+
 
 class FindSpecTests(abc.FinderTests):
 
@@ -13,7 +15,8 @@ class FindSpecTests(abc.FinderTests):
 
     def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
-        return finder.find_spec(name, path)
+        with import_helper.frozen_modules():
+            return finder.find_spec(name, path)
 
     def test_module(self):
         name = '__hello__'
@@ -52,7 +55,8 @@ class FinderTests(abc.FinderTests):
         finder = self.machinery.FrozenImporter
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
-            return finder.find_module(name, path)
+            with import_helper.frozen_modules():
+                return finder.find_module(name, path)
 
     def test_module(self):
         name = '__hello__'
diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
index f0cf179..1b0a56f 100644
--- a/Lib/test/test_importlib/frozen/test_loader.py
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -3,27 +3,54 @@ from .. import util
 
 machinery = util.import_importlib('importlib.machinery')
 
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
+import contextlib
 import types
 import unittest
 import warnings
 
 
+@contextlib.contextmanager
+def deprecated():
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        yield
+
+
+@contextlib.contextmanager
+def fresh(name, *, oldapi=False):
+    with util.uncache(name):
+        with import_helper.frozen_modules():
+            with captured_stdout() as stdout:
+                if oldapi:
+                    with deprecated():
+                        yield stdout
+                else:
+                    yield stdout
+
+
 class ExecModuleTests(abc.LoaderTests):
 
     def exec_module(self, name):
-        with util.uncache(name), captured_stdout() as stdout:
-            spec = self.machinery.ModuleSpec(
-                    name, self.machinery.FrozenImporter, origin='frozen',
-                    is_package=self.machinery.FrozenImporter.is_package(name))
-            module = types.ModuleType(name)
-            module.__spec__ = spec
-            assert not hasattr(module, 'initialized')
+        with import_helper.frozen_modules():
+            is_package = self.machinery.FrozenImporter.is_package(name)
+        spec = self.machinery.ModuleSpec(
+            name,
+            self.machinery.FrozenImporter,
+            origin='frozen',
+            is_package=is_package,
+        )
+        module = types.ModuleType(name)
+        module.__spec__ = spec
+        assert not hasattr(module, 'initialized')
+
+        with fresh(name) as stdout:
             self.machinery.FrozenImporter.exec_module(module)
-            self.assertTrue(module.initialized)
-            self.assertTrue(hasattr(module, '__spec__'))
-            self.assertEqual(module.__spec__.origin, 'frozen')
-            return module, stdout.getvalue()
+
+        self.assertTrue(module.initialized)
+        self.assertTrue(hasattr(module, '__spec__'))
+        self.assertEqual(module.__spec__.origin, 'frozen')
+        return module, stdout.getvalue()
 
     def test_module(self):
         name = '__hello__'
@@ -50,20 +77,19 @@ class ExecModuleTests(abc.LoaderTests):
         name = '__phello__.spam'
         with util.uncache('__phello__'):
             module, output = self.exec_module(name)
-            check = {'__name__': name}
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                        'for {name}.{attr}, {given} != {expected!r}'.format(
-                                 name=name, attr=attr, given=attr_value,
-                                 expected=value))
-            self.assertEqual(output, 'Hello world!\n')
+        check = {'__name__': name}
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                    'for {name}.{attr}, {given} != {expected!r}'.format(
+                             name=name, attr=attr, given=attr_value,
+                             expected=value))
+        self.assertEqual(output, 'Hello world!\n')
 
     def test_module_repr(self):
         name = '__hello__'
         module, output = self.exec_module(name)
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore', DeprecationWarning)
+        with deprecated():
             repr_str = self.machinery.FrozenImporter.module_repr(module)
         self.assertEqual(repr_str,
                          "<module '__hello__' (frozen)>")
@@ -78,7 +104,8 @@ class ExecModuleTests(abc.LoaderTests):
     test_state_after_failure = None
 
     def test_unloadable(self):
-        assert self.machinery.FrozenImporter.find_spec('_not_real') is None
+        with import_helper.frozen_modules():
+            assert self.machinery.FrozenImporter.find_spec('_not_real') is None
         with self.assertRaises(ImportError) as cm:
             self.exec_module('_not_real')
         self.assertEqual(cm.exception.name, '_not_real')
@@ -91,84 +118,76 @@ class ExecModuleTests(abc.LoaderTests):
 
 class LoaderTests(abc.LoaderTests):
 
+    def load_module(self, name):
+        with fresh(name, oldapi=True) as stdout:
+            module = self.machinery.FrozenImporter.load_module(name)
+        return module, stdout
+
     def test_module(self):
-        with util.uncache('__hello__'), captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__hello__')
-            check = {'__name__': '__hello__',
-                    '__package__': '',
-                    '__loader__': self.machinery.FrozenImporter,
-                    }
-            for attr, value in check.items():
-                self.assertEqual(getattr(module, attr), value)
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        module, stdout = self.load_module('__hello__')
+        check = {'__name__': '__hello__',
+                '__package__': '',
+                '__loader__': self.machinery.FrozenImporter,
+                }
+        for attr, value in check.items():
+            self.assertEqual(getattr(module, attr), value)
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_package(self):
-        with util.uncache('__phello__'),  captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__phello__')
-            check = {'__name__': '__phello__',
-                     '__package__': '__phello__',
-                     '__path__': [],
-                     '__loader__': self.machinery.FrozenImporter,
-                     }
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                                 "for __phello__.%s, %r != %r" %
-                                 (attr, attr_value, value))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        module, stdout = self.load_module('__phello__')
+        check = {'__name__': '__phello__',
+                 '__package__': '__phello__',
+                 '__path__': [],
+                 '__loader__': self.machinery.FrozenImporter,
+                 }
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                             "for __phello__.%s, %r != %r" %
+                             (attr, attr_value, value))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_lacking_parent(self):
-        with util.uncache('__phello__', '__phello__.spam'), \
-             captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__phello__.spam')
-            check = {'__name__': '__phello__.spam',
-                    '__package__': '__phello__',
-                    '__loader__': self.machinery.FrozenImporter,
-                    }
-            for attr, value in check.items():
-                attr_value = getattr(module, attr)
-                self.assertEqual(attr_value, value,
-                                 "for __phello__.spam.%s, %r != %r" %
-                                 (attr, attr_value, value))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
-            self.assertFalse(hasattr(module, '__file__'))
+        with util.uncache('__phello__'):
+            module, stdout = self.load_module('__phello__.spam')
+        check = {'__name__': '__phello__.spam',
+                '__package__': '__phello__',
+                '__loader__': self.machinery.FrozenImporter,
+                }
+        for attr, value in check.items():
+            attr_value = getattr(module, attr)
+            self.assertEqual(attr_value, value,
+                             "for __phello__.spam.%s, %r != %r" %
+                             (attr, attr_value, value))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+        self.assertFalse(hasattr(module, '__file__'))
 
     def test_module_reuse(self):
-        with util.uncache('__hello__'), captured_stdout() as stdout:
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module1 = self.machinery.FrozenImporter.load_module('__hello__')
-                module2 = self.machinery.FrozenImporter.load_module('__hello__')
-            self.assertIs(module1, module2)
-            self.assertEqual(stdout.getvalue(),
-                             'Hello world!\nHello world!\n')
+        with fresh('__hello__', oldapi=True) as stdout:
+            module1 = self.machinery.FrozenImporter.load_module('__hello__')
+            module2 = self.machinery.FrozenImporter.load_module('__hello__')
+        self.assertIs(module1, module2)
+        self.assertEqual(stdout.getvalue(),
+                         'Hello world!\nHello world!\n')
 
     def test_module_repr(self):
-        with util.uncache('__hello__'), captured_stdout():
-            with warnings.catch_warnings():
-                warnings.simplefilter('ignore', DeprecationWarning)
-                module = self.machinery.FrozenImporter.load_module('__hello__')
-                repr_str = self.machinery.FrozenImporter.module_repr(module)
-            self.assertEqual(repr_str,
-                             "<module '__hello__' (frozen)>")
+        with fresh('__hello__', oldapi=True) as stdout:
+            module = self.machinery.FrozenImporter.load_module('__hello__')
+            repr_str = self.machinery.FrozenImporter.module_repr(module)
+        self.assertEqual(repr_str,
+                         "<module '__hello__' (frozen)>")
 
     # No way to trigger an error in a frozen module.
     test_state_after_failure = None
 
     def test_unloadable(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore", DeprecationWarning)
-            assert self.machinery.FrozenImporter.find_module('_not_real') is None
+        with import_helper.frozen_modules():
+            with deprecated():
+                assert self.machinery.FrozenImporter.find_module('_not_real') is None
             with self.assertRaises(ImportError) as cm:
-                self.machinery.FrozenImporter.load_module('_not_real')
+                self.load_module('_not_real')
             self.assertEqual(cm.exception.name, '_not_real')
 
 
@@ -185,15 +204,17 @@ class InspectLoaderTests:
         # Make sure that the code object is good.
         name = '__hello__'
         with captured_stdout() as stdout:
-            code = self.machinery.FrozenImporter.get_code(name)
-            mod = types.ModuleType(name)
-            exec(code, mod.__dict__)
-            self.assertTrue(hasattr(mod, 'initialized'))
-            self.assertEqual(stdout.getvalue(), 'Hello world!\n')
+            with import_helper.frozen_modules():
+                code = self.machinery.FrozenImporter.get_code(name)
+                mod = types.ModuleType(name)
+                exec(code, mod.__dict__)
+        self.assertTrue(hasattr(mod, 'initialized'))
+        self.assertEqual(stdout.getvalue(), 'Hello world!\n')
 
     def test_get_source(self):
         # Should always return None.
-        result = self.machinery.FrozenImporter.get_source('__hello__')
+        with import_helper.frozen_modules():
+            result = self.machinery.FrozenImporter.get_source('__hello__')
         self.assertIsNone(result)
 
     def test_is_package(self):
@@ -201,7 +222,8 @@ class InspectLoaderTests:
         test_for = (('__hello__', False), ('__phello__', True),
                     ('__phello__.spam', False))
         for name, is_package in test_for:
-            result = self.machinery.FrozenImporter.is_package(name)
+            with import_helper.frozen_modules():
+                result = self.machinery.FrozenImporter.is_package(name)
             self.assertEqual(bool(result), is_package)
 
     def test_failure(self):
@@ -209,7 +231,8 @@ class InspectLoaderTests:
         for meth_name in ('get_code', 'get_source', 'is_package'):
             method = getattr(self.machinery.FrozenImporter, meth_name)
             with self.assertRaises(ImportError) as cm:
-                method('importlib')
+                with import_helper.frozen_modules():
+                    method('importlib')
             self.assertEqual(cm.exception.name, 'importlib')
 
 (Frozen_ILTests,