[3.10] bpo-45703: Invalidate _NamespacePath cache on importlib.invalidate_cache (GH-29384) (GH-30922)

Consider the following directory structure:

    .
    └── PATH1
        └── namespace
            └── sub1
                └── __init__.py

And both PATH1 and PATH2 in sys path:

    $ PYTHONPATH=PATH1:PATH2 python3.11
    >>> import namespace
    >>> import namespace.sub1
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> ...

While this interpreter still runs, PATH2/namespace/sub2 is created:

    .
    ├── PATH1
    │   └── namespace
    │       └── sub1
    │           └── __init__.py
    └── PATH2
        └── namespace
            └── sub2
                └── __init__.py

The newly created module cannot be imported:

    >>> ...
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> import namespace.sub2
    Traceback (most recent call last):
      File "<stdin>", line 1, in <module>
    ModuleNotFoundError: No module named 'namespace.sub2'

Calling importlib.invalidate_caches() now newly allows to import it:

    >>> import importlib
    >>> importlib.invalidate_caches()
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace'])
    >>> import namespace.sub2
    >>> namespace.__path__
    _NamespacePath(['.../PATH1/namespace', '.../PATH2/namespace'])

This was not previously possible.

1 files changed, 10 insertions, 1 deletions

diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
index 9a73c7b..49bcaea 100644
--- a/Lib/importlib/_bootstrap_external.py
+++ b/Lib/importlib/_bootstrap_external.py
@@ -1212,10 +1212,15 @@ class _NamespacePath:
     using path_finder.  For top-level modules, the parent module's path
     is sys.path."""
 
+    # When invalidate_caches() is called, this epoch is incremented
+    # https://bugs.python.org/issue45703
+    _epoch = 0
+
     def __init__(self, name, path, path_finder):
         self._name = name
         self._path = path
         self._last_parent_path = tuple(self._get_parent_path())
+        self._last_epoch = self._epoch
         self._path_finder = path_finder
 
     def _find_parent_path_names(self):
@@ -1235,7 +1240,7 @@ class _NamespacePath:
     def _recalculate(self):
         # If the parent's path has changed, recalculate _path
         parent_path = tuple(self._get_parent_path()) # Make a copy
-        if parent_path != self._last_parent_path:
+        if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
             spec = self._path_finder(self._name, parent_path)
             # Note that no changes are made if a loader is returned, but we
             #  do remember the new parent path
@@ -1243,6 +1248,7 @@ class _NamespacePath:
                 if spec.submodule_search_locations:
                     self._path = spec.submodule_search_locations
             self._last_parent_path = parent_path     # Save the copy
+            self._last_epoch = self._epoch
         return self._path
 
     def __iter__(self):
@@ -1330,6 +1336,9 @@ class PathFinder:
                 del sys.path_importer_cache[name]
             elif hasattr(finder, 'invalidate_caches'):
                 finder.invalidate_caches()
+        # Also invalidate the caches of _NamespacePaths
+        # https://bugs.python.org/issue45703
+        _NamespacePath._epoch += 1
 
     @staticmethod
     def _path_hooks(path):