bpo-45020: Identify which frozen modules are actually aliases. (gh-28655)

In the list of generated frozen modules at the top of Tools/scripts/freeze_modules.py, you will find that some of the modules have a different name than the module (or .py file) that is actually frozen. Let's call each case an "alias". Aliases do not come into play until we get to the (generated) list of modules in Python/frozen.c. (The tool for freezing modules, Programs/_freeze_module, is only concerned with the source file, not the module it will be used for.)

Knowledge of which frozen modules are aliases (and the identity of the original module) normally isn't important. However, this information is valuable when we go to set __file__ on frozen stdlib modules. This change updates Tools/scripts/freeze_modules.py to map aliases to the original module name (or None if not a stdlib module) in Python/frozen.c. We also add a helper function in Python/import.c to look up a frozen module's alias and add the result of that function to the frozen info returned from find_frozen().

https://bugs.python.org/issue45020

3 files changed, 77 insertions, 22 deletions

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 5807577..49f0881 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -825,15 +825,38 @@ class FrozenImporter:
         return '<module {!r} ({})>'.format(m.__name__, FrozenImporter._ORIGIN)
 
     @classmethod
+    def _setup_module(cls, module):
+        assert not hasattr(module, '__file__'), module.__file__
+        ispkg = hasattr(module, '__path__')
+        assert not ispkg or not module.__path__, module.__path__
+        spec = module.__spec__
+        assert not ispkg or not spec.submodule_search_locations
+
+        if spec.loader_state is None:
+            spec.loader_state = type(sys.implementation)(
+                data=None,
+                origname=None,
+            )
+        elif not hasattr(spec.loader_state, 'data'):
+            spec.loader_state.data = None
+        if not getattr(spec.loader_state, 'origname', None):
+            origname = vars(module).pop('__origname__', None)
+            assert origname, 'see PyImport_ImportFrozenModuleObject()'
+            spec.loader_state.origname = origname
+
+    @classmethod
     def find_spec(cls, fullname, path=None, target=None):
         info = _call_with_frames_removed(_imp.find_frozen, fullname)
         if info is None:
             return None
-        data, ispkg = info
+        data, ispkg, origname = info
         spec = spec_from_loader(fullname, cls,
                                 origin=cls._ORIGIN,
                                 is_package=ispkg)
-        spec.loader_state = data
+        spec.loader_state = type(sys.implementation)(
+            data=data,
+            origname=origname,
+        )
         return spec
 
     @classmethod
@@ -857,7 +880,7 @@ class FrozenImporter:
         spec = module.__spec__
         name = spec.name
         try:
-            data = spec.loader_state
+            data = spec.loader_state.data
         except AttributeError:
             if not _imp.is_frozen(name):
                 raise ImportError('{!r} is not a frozen module'.format(name),
@@ -868,7 +891,7 @@ class FrozenImporter:
             # Note that if this method is called again (e.g. by
             # importlib.reload()) then _imp.get_frozen_object() will notice
             # no data was provided and will look it up.
-            spec.loader_state = None
+            spec.loader_state.data = None
         code = _call_with_frames_removed(_imp.get_frozen_object, name, data)
         exec(code, module.__dict__)
 
@@ -1220,6 +1243,8 @@ def _setup(sys_module, _imp_module):
                 continue
             spec = _spec_from_module(module, loader)
             _init_module_attrs(spec, module)
+            if loader is FrozenImporter:
+                loader._setup_module(module)
 
     # Directly load built-in modules needed during bootstrap.
     self_module = sys.modules[__name__]
diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py
index 0b15aeb..cd5586d 100644
--- a/Lib/test/test_importlib/frozen/test_finder.py
+++ b/Lib/test/test_importlib/frozen/test_finder.py
@@ -9,7 +9,15 @@ import os.path
 import unittest
 import warnings
 
-from test.support import import_helper, REPO_ROOT
+from test.support import import_helper, REPO_ROOT, STDLIB_DIR
+
+
+def resolve_stdlib_file(name, ispkg=False):
+    assert name
+    if ispkg:
+        return os.path.join(STDLIB_DIR, *name.split('.'), '__init__.py')
+    else:
+        return os.path.join(STDLIB_DIR, *name.split('.')) + '.py'
 
 
 class FindSpecTests(abc.FinderTests):
@@ -32,16 +40,30 @@ class FindSpecTests(abc.FinderTests):
             self.assertIsNone(spec.submodule_search_locations)
         self.assertIsNotNone(spec.loader_state)
 
-    def check_data(self, spec):
+    def check_loader_state(self, spec, origname=None, filename=None):
+        if not filename:
+            if not origname:
+                origname = spec.name
+
+        actual = dict(vars(spec.loader_state))
+
+        # Check the code object used to import the frozen module.
+        # We can't compare the marshaled data directly because
+        # marshal.dumps() would mark "expected" (below) as a ref,
+        # which slightly changes the output.
+        # (See https://bugs.python.org/issue34093.)
+        data = actual.pop('data')
         with import_helper.frozen_modules():
             expected = _imp.get_frozen_object(spec.name)
-        data = spec.loader_state
-        # We can't compare the marshaled data directly because
-        # marshal.dumps() would mark "expected" as a ref, which slightly
-        # changes the output.  (See https://bugs.python.org/issue34093.)
         code = marshal.loads(data)
         self.assertEqual(code, expected)
 
+        # Check the rest of spec.loader_state.
+        expected = dict(
+            origname=origname,
+        )
+        self.assertDictEqual(actual, expected)
+
     def check_search_locations(self, spec):
         # Frozen packages do not have any path entries.
         # (See https://bugs.python.org/issue21736.)
@@ -58,7 +80,7 @@ class FindSpecTests(abc.FinderTests):
             with self.subTest(f'{name} -> {name}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec)
         modules = {
             '__hello_alias__': '__hello__',
             '_frozen_importlib': 'importlib._bootstrap',
@@ -67,26 +89,28 @@ class FindSpecTests(abc.FinderTests):
             with self.subTest(f'{name} -> {origname}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname)
         modules = [
             '__phello__.__init__',
             '__phello__.ham.__init__',
         ]
         for name in modules:
-            origname = name.rpartition('.')[0]
+            origname = '<' + name.rpartition('.')[0]
+            filename = resolve_stdlib_file(name)
             with self.subTest(f'{name} -> {origname}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
         modules = {
             '__hello_only__': ('Tools', 'freeze', 'flag.py'),
         }
         for name, path in modules.items():
+            origname = None
             filename = os.path.join(REPO_ROOT, *path)
             with self.subTest(f'{name} -> {filename}'):
                 spec = self.find(name)
                 self.check_basic(spec, name)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
 
     def test_package(self):
         packages = [
@@ -94,19 +118,21 @@ class FindSpecTests(abc.FinderTests):
             '__phello__.ham',
         ]
         for name in packages:
+            filename = resolve_stdlib_file(name, ispkg=True)
             with self.subTest(f'{name} -> {name}'):
                 spec = self.find(name)
                 self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, name, filename)
                 self.check_search_locations(spec)
         packages = {
             '__phello_alias__': '__hello__',
         }
         for name, origname in packages.items():
+            filename = resolve_stdlib_file(origname, ispkg=False)
             with self.subTest(f'{name} -> {origname}'):
                 spec = self.find(name)
                 self.check_basic(spec, name, ispkg=True)
-                self.check_data(spec)
+                self.check_loader_state(spec, origname, filename)
                 self.check_search_locations(spec)
 
     # These are covered by test_module() and test_package().
diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py
index 992dcef..d6f39fa 100644
--- a/Lib/test/test_importlib/frozen/test_loader.py
+++ b/Lib/test/test_importlib/frozen/test_loader.py
@@ -32,17 +32,19 @@ def fresh(name, *, oldapi=False):
 
 class ExecModuleTests(abc.LoaderTests):
 
-    def exec_module(self, name):
+    def exec_module(self, name, origname=None):
         with import_helper.frozen_modules():
             is_package = self.machinery.FrozenImporter.is_package(name)
             code = _imp.get_frozen_object(name)
-        data = marshal.dumps(code)
         spec = self.machinery.ModuleSpec(
             name,
             self.machinery.FrozenImporter,
             origin='frozen',
             is_package=is_package,
-            loader_state=data,
+            loader_state=types.SimpleNamespace(
+                data=marshal.dumps(code),
+                origname=origname or name,
+            ),
         )
         module = types.ModuleType(name)
         module.__spec__ = spec
@@ -66,7 +68,8 @@ class ExecModuleTests(abc.LoaderTests):
             self.assertEqual(getattr(module, attr), value)
         self.assertEqual(output, 'Hello world!\n')
         self.assertTrue(hasattr(module, '__spec__'))
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)
 
     def test_package(self):
         name = '__phello__'
@@ -79,7 +82,8 @@ class ExecModuleTests(abc.LoaderTests):
                                  name=name, attr=attr, given=attr_value,
                                  expected=value))
         self.assertEqual(output, 'Hello world!\n')
-        self.assertIsNone(module.__spec__.loader_state)
+        self.assertIsNone(module.__spec__.loader_state.data)
+        self.assertEqual(module.__spec__.loader_state.origname, name)
 
     def test_lacking_parent(self):
         name = '__phello__.spam'