Issue #18214: Improve finalization of Python modules to avoid setting their globals to None, in most cases.

6 files changed, 88 insertions, 12 deletions

diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py
index d3a4437..8775730 100644
--- a/Lib/rlcompleter.py
+++ b/Lib/rlcompleter.py
@@ -29,6 +29,7 @@ Notes:
 
 """
 
+import atexit
 import builtins
 import __main__
 
@@ -158,3 +159,8 @@ except ImportError:
     pass
 else:
     readline.set_completer(Completer().complete)
+    # Release references early at shutdown (the readline module's
+    # contents are quasi-immortal, and the completer function holds a
+    # reference to globals).
+    atexit.register(lambda: readline.set_completer(None))
+
diff --git a/Lib/site.py b/Lib/site.py
index 96a4bef..08b98eb 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -68,6 +68,7 @@ site-specific customizations.  If this import fails with an
 ImportError exception, it is silently ignored.
 """
 
+import atexit
 import sys
 import os
 import re
@@ -86,6 +87,25 @@ USER_SITE = None
 USER_BASE = None
 
 
+_no_builtin = object()
+
+def _patch_builtins(**items):
+    # When patching builtins, we make some objects almost immortal
+    # (builtins are only reclaimed at the very end of the interpreter
+    #  shutdown sequence).  To avoid keeping to many references alive,
+    # we register callbacks to undo our builtins additions.
+    old_items = {k: getattr(builtins, k, _no_builtin) for k in items}
+    def unpatch(old_items=old_items):
+        for k, v in old_items.items():
+            if v is _no_builtin:
+                delattr(builtins, k)
+            else:
+                setattr(builtins, k, v)
+    for k, v in items.items():
+        setattr(builtins, k, v)
+    atexit.register(unpatch)
+
+
 def makepath(*paths):
     dir = os.path.join(*paths)
     try:
@@ -357,8 +377,7 @@ def setquit():
             except:
                 pass
             raise SystemExit(code)
-    builtins.quit = Quitter('quit')
-    builtins.exit = Quitter('exit')
+    _patch_builtins(quit=Quitter('quit'), exit=Quitter('exit'))
 
 
 class _Printer(object):
@@ -423,20 +442,20 @@ class _Printer(object):
 
 def setcopyright():
     """Set 'copyright' and 'credits' in builtins"""
-    builtins.copyright = _Printer("copyright", sys.copyright)
+    _patch_builtins(copyright=_Printer("copyright", sys.copyright))
     if sys.platform[:4] == 'java':
-        builtins.credits = _Printer(
+        _patch_builtins(credits=_Printer(
             "credits",
-            "Jython is maintained by the Jython developers (www.jython.org).")
+            "Jython is maintained by the Jython developers (www.jython.org)."))
     else:
-        builtins.credits = _Printer("credits", """\
+        _patch_builtins(credits=_Printer("credits", """\
     Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
-    for supporting Python development.  See www.python.org for more information.""")
+    for supporting Python development.  See www.python.org for more information."""))
     here = os.path.dirname(os.__file__)
-    builtins.license = _Printer(
+    _patch_builtins(license=_Printer(
         "license", "See http://www.python.org/%.3s/license.html" % sys.version,
         ["LICENSE.txt", "LICENSE"],
-        [os.path.join(here, os.pardir), here, os.curdir])
+        [os.path.join(here, os.pardir), here, os.curdir]))
 
 
 class _Helper(object):
@@ -453,7 +472,7 @@ class _Helper(object):
         return pydoc.help(*args, **kwds)
 
 def sethelper():
-    builtins.help = _Helper()
+    _patch_builtins(help=_Helper())
 
 def enablerlcompleter():
     """Enable default readline configuration on interactive prompts, by
diff --git a/Lib/test/final_a.py b/Lib/test/final_a.py
new file mode 100644
index 0000000..390ee88
--- /dev/null
+++ b/Lib/test/final_a.py
@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.final_b
+
+x = 'a'
+
+class C:
+    def __del__(self):
+        # Inspect module globals and builtins
+        print("x =", x)
+        print("final_b.x =", test.final_b.x)
+        print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+        print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()
diff --git a/Lib/test/final_b.py b/Lib/test/final_b.py
new file mode 100644
index 0000000..7228d82
--- /dev/null
+++ b/Lib/test/final_b.py
@@ -0,0 +1,19 @@
+"""
+Fodder for module finalization tests in test_module.
+"""
+
+import shutil
+import test.final_a
+
+x = 'b'
+
+class C:
+    def __del__(self):
+        # Inspect module globals and builtins
+        print("x =", x)
+        print("final_a.x =", test.final_a.x)
+        print("shutil.rmtree =", getattr(shutil.rmtree, '__name__', None))
+        print("len =", getattr(len, '__name__', None))
+
+c = C()
+_underscored = C()
diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py
index b34b30f..3000cec 100644
--- a/Lib/test/test_module.py
+++ b/Lib/test/test_module.py
@@ -1,6 +1,7 @@
 # Test the module type
 import unittest
 from test.support import run_unittest, gc_collect
+from test.script_helper import assert_python_ok
 
 import sys
 ModuleType = type(sys)
@@ -70,7 +71,6 @@ class ModuleTests(unittest.TestCase):
                "__loader__": None, "__package__": None})
         self.assertTrue(foo.__dict__ is d)
 
-    @unittest.expectedFailure
     def test_dont_clear_dict(self):
         # See issue 7140.
         def f():
@@ -181,6 +181,19 @@ a = A(destroyed)"""
         self.assertEqual(r[:25], "<module 'unittest' from '")
         self.assertEqual(r[-13:], "__init__.py'>")
 
+    def test_module_finalization_at_shutdown(self):
+        # Module globals and builtins should still be available during shutdown
+        rc, out, err = assert_python_ok("-c", "from test import final_a")
+        self.assertFalse(err)
+        lines = out.splitlines()
+        self.assertEqual(set(lines), {
+            b"x = a",
+            b"x = b",
+            b"final_a.x = a",
+            b"final_b.x = b",
+            b"len = len",
+            b"shutil.rmtree = rmtree"})
+
     # frozen and namespace module reprs are tested in importlib.
 
 
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index 3f093e6..26c7ae7 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -810,7 +810,7 @@ class SizeofTest(unittest.TestCase):
         # memoryview
         check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn'))
         # module
-        check(unittest, size('PnP'))
+        check(unittest, size('PnPPP'))
         # None
         check(None, size(''))
         # NotImplementedType