bpo-45703: Invalidate _NamespacePath cache on importlib.invalidate_ca… (GH-29384)

Consider the following directory structure:

    .
    └── PATH1
        └── namespace
            └── sub1
                └── __init__.py

And both PATH1 and PATH2 in sys path:

    $ PYTHONPATH=PATH1:PATH2 python3.11
    >>> import namespace
    >>> import namespace.sub1
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> ...

While this interpreter still runs, PATH2/namespace/sub2 is created:

    .
    ├── PATH1
    │   └── namespace
    │       └── sub1
    │           └── __init__.py
    └── PATH2
        └── namespace
            └── sub2
                └── __init__.py

The newly created module cannot be imported:

    >>> ...
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> import namespace.sub2
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
    ModuleNotFoundError: No module named 'namespace.sub2'

Calling importlib.invalidate_caches() now newly allows to import it:

    >>> import importlib
    >>> importlib.invalidate_caches()
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> import namespace.sub2
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace', '.../PATH2/namespace'])

This was not previously possible.

1 files changed, 35 insertions, 0 deletions

diff --git a/Lib/test/test_importlib/test_namespace_pkgs.py b/Lib/test/test_importlib/test_namespace_pkgs.py
index f802832..2ea41b7 100644
--- a/Lib/test/test_importlib/test_namespace_pkgs.py
+++ b/Lib/test/test_importlib/test_namespace_pkgs.py
@@ -4,6 +4,7 @@ import importlib.abc
 import importlib.machinery
 import os
 import sys
+import tempfile
 import unittest
 import warnings
 
@@ -130,6 +131,40 @@ class SeparatedNamespacePackages(NamespacePackageTest):
         self.assertEqual(foo.two.attr, 'portion2 foo two')
 
 
+class SeparatedNamespacePackagesCreatedWhileRunning(NamespacePackageTest):
+    paths = ['portion1']
+
+    def test_invalidate_caches(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # we manipulate sys.path before anything is imported to avoid
+            # accidental cache invalidation when changing it
+            sys.path.append(temp_dir)
+
+            import foo.one
+            self.assertEqual(foo.one.attr, 'portion1 foo one')
+
+            # the module does not exist, so it cannot be imported
+            with self.assertRaises(ImportError):
+                import foo.just_created
+
+            # util.create_modules() manipulates sys.path
+            # so we must create the modules manually instead
+            namespace_path = os.path.join(temp_dir, 'foo')
+            os.mkdir(namespace_path)
+            module_path = os.path.join(namespace_path, 'just_created.py')
+            with open(module_path, 'w', encoding='utf-8') as file:
+                file.write('attr = "just_created foo"')
+
+            # the module is not known, so it cannot be imported yet
+            with self.assertRaises(ImportError):
+                import foo.just_created
+
+            # but after explicit cache invalidation, it is importable
+            importlib.invalidate_caches()
+            import foo.just_created
+            self.assertEqual(foo.just_created.attr, 'just_created foo')
+
+
 class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
     paths = ['portion1', 'both_portions']