gh-101659: Isolate "obmalloc" State to Each Interpreter (gh-101660)

This is strictly about moving the "obmalloc" runtime state from
`_PyRuntimeState` to `PyInterpreterState`.  Doing so improves isolation
between interpreters, specifically most of the memory (incl. objects)
allocated for each interpreter's use.  This is important for a
per-interpreter GIL, but such isolation is valuable even without it.

FWIW, a per-interpreter obmalloc is the proverbial
canary-in-the-coalmine when it comes to the isolation of objects between
interpreters.  Any object that leaks (unintentionally) to another
interpreter is highly likely to cause a crash (on debug builds at
least).  That's a useful thing to know, relative to interpreter
isolation.

4 files changed, 53 insertions, 11 deletions

diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index 637adc0..eab6930 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -1211,20 +1211,25 @@ class SubinterpreterTest(unittest.TestCase):
         """
         import json
 
+        OBMALLOC = 1<<5
         EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
 
-        features = ['fork', 'exec', 'threads', 'daemon_threads', 'extensions']
+        features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
+                    'extensions']
         kwlist = [f'allow_{n}' for n in features]
+        kwlist[0] = 'use_main_obmalloc'
         kwlist[-1] = 'check_multi_interp_extensions'
+
+        # expected to work
         for config, expected in {
-            (True, True, True, True, True):
-                FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
-            (False, False, False, False, False): 0,
-            (False, False, True, False, True): THREADS | EXTENSIONS,
+            (True, True, True, True, True, True):
+                OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
+            (True, False, False, False, False, False): OBMALLOC,
+            (False, False, False, True, False, True): THREADS | EXTENSIONS,
         }.items():
             kwargs = dict(zip(kwlist, config))
             expected = {
@@ -1246,6 +1251,20 @@ class SubinterpreterTest(unittest.TestCase):
 
                 self.assertEqual(settings, expected)
 
+        # expected to fail
+        for config in [
+            (False, False, False, False, False, False),
+        ]:
+            kwargs = dict(zip(kwlist, config))
+            with self.subTest(config):
+                script = textwrap.dedent(f'''
+                    import _testinternalcapi
+                    _testinternalcapi.get_interp_settings()
+                    raise NotImplementedError('unreachable')
+                    ''')
+                with self.assertRaises(RuntimeError):
+                    support.run_in_subinterp_with_config(script, **kwargs)
+
     @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
     @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
     def test_overridden_setting_extensions_subinterp_check(self):
@@ -1257,13 +1276,15 @@ class SubinterpreterTest(unittest.TestCase):
         """
         import json
 
+        OBMALLOC = 1<<5
         EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
         FORK = 1<<15
         EXEC = 1<<16
-        BASE_FLAGS = FORK | EXEC | THREADS | DAEMON_THREADS
+        BASE_FLAGS = OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS
         base_kwargs = {
+            'use_main_obmalloc': True,
             'allow_fork': True,
             'allow_exec': True,
             'allow_threads': True,
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e56d0db..f702ffb 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -1656,6 +1656,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                                        api=API_PYTHON, env=env)
 
     def test_init_main_interpreter_settings(self):
+        OBMALLOC = 1<<5
         EXTENSIONS = 1<<8
         THREADS = 1<<10
         DAEMON_THREADS = 1<<11
@@ -1664,7 +1665,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         expected = {
             # All optional features should be enabled.
             'feature_flags':
-                FORK | EXEC | THREADS | DAEMON_THREADS,
+                OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS,
         }
         out, err = self.run_embedded_interpreter(
             'test_init_main_interpreter_settings',
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index 66ae554..f206e52 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -1636,7 +1636,12 @@ class SubinterpImportTests(unittest.TestCase):
         allow_exec=False,
         allow_threads=True,
         allow_daemon_threads=False,
+        # Isolation-related config values aren't included here.
     )
+    ISOLATED = dict(
+        use_main_obmalloc=False,
+    )
+    NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()}
 
     @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
     def pipe(self):
@@ -1669,6 +1674,7 @@ class SubinterpImportTests(unittest.TestCase):
     def run_here(self, name, *,
                  check_singlephase_setting=False,
                  check_singlephase_override=None,
+                 isolated=False,
                  ):
         """
         Try importing the named module in a subinterpreter.
@@ -1689,6 +1695,7 @@ class SubinterpImportTests(unittest.TestCase):
 
         kwargs = dict(
             **self.RUN_KWARGS,
+            **(self.ISOLATED if isolated else self.NOT_ISOLATED),
             check_multi_interp_extensions=check_singlephase_setting,
         )
 
@@ -1699,33 +1706,36 @@ class SubinterpImportTests(unittest.TestCase):
         self.assertEqual(ret, 0)
         return os.read(r, 100)
 
-    def check_compatible_here(self, name, *, strict=False):
+    def check_compatible_here(self, name, *, strict=False, isolated=False):
         # Verify that the named module may be imported in a subinterpreter.
         # (See run_here() for more info.)
         out = self.run_here(name,
                             check_singlephase_setting=strict,
+                            isolated=isolated,
                             )
         self.assertEqual(out, b'okay')
 
-    def check_incompatible_here(self, name):
+    def check_incompatible_here(self, name, *, isolated=False):
         # Differences from check_compatible_here():
         #  * verify that import fails
         #  * "strict" is always True
         out = self.run_here(name,
                             check_singlephase_setting=True,
+                            isolated=isolated,
                             )
         self.assertEqual(
             out.decode('utf-8'),
             f'ImportError: module {name} does not support loading in subinterpreters',
         )
 
-    def check_compatible_fresh(self, name, *, strict=False):
+    def check_compatible_fresh(self, name, *, strict=False, isolated=False):
         # Differences from check_compatible_here():
         #  * subinterpreter in a new process
         #  * module has never been imported before in that process
         #  * this tests importing the module for the first time
         kwargs = dict(
             **self.RUN_KWARGS,
+            **(self.ISOLATED if isolated else self.NOT_ISOLATED),
             check_multi_interp_extensions=strict,
         )
         _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
@@ -1743,12 +1753,13 @@ class SubinterpImportTests(unittest.TestCase):
         self.assertEqual(err, b'')
         self.assertEqual(out, b'okay')
 
-    def check_incompatible_fresh(self, name):
+    def check_incompatible_fresh(self, name, *, isolated=False):
         # Differences from check_compatible_fresh():
         #  * verify that import fails
         #  * "strict" is always True
         kwargs = dict(
             **self.RUN_KWARGS,
+            **(self.ISOLATED if isolated else self.NOT_ISOLATED),
             check_multi_interp_extensions=True,
         )
         _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f'''
@@ -1854,6 +1865,14 @@ class SubinterpImportTests(unittest.TestCase):
         with self.subTest('config: check disabled; override: disabled'):
             check_compatible(False, -1)
 
+    def test_isolated_config(self):
+        module = 'threading'
+        require_pure_python(module)
+        with self.subTest(f'{module}: strict, not fresh'):
+            self.check_compatible_here(module, strict=True, isolated=True)
+        with self.subTest(f'{module}: strict, fresh'):
+            self.check_compatible_fresh(module, strict=True, isolated=True)
+
 
 class TestSinglePhaseSnapshot(ModuleSnapshot):
 
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
index a39a267..fdd74c3 100644
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -1343,6 +1343,7 @@ class SubinterpThreadingTests(BaseTestCase):
             import test.support
             test.support.run_in_subinterp_with_config(
                 {subinterp_code!r},
+                use_main_obmalloc=True,
                 allow_fork=True,
                 allow_exec=True,
                 allow_threads={allowed},