summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Include/internal/pycore_gc.h17
-rw-r--r--Lib/concurrent/futures/process.py5
-rw-r--r--Lib/test/libregrtest/main.py8
-rw-r--r--Lib/test/libregrtest/single.py5
-rw-r--r--Lib/test/support/__init__.py19
-rw-r--r--Lib/test/test_capi/test_watchers.py6
-rw-r--r--Lib/test/test_code.py6
-rw-r--r--Lib/test/test_functools.py1
-rw-r--r--Lib/test/test_weakref.py5
-rw-r--r--Modules/_testinternalcapi.c22
-rw-r--r--Objects/object.c7
-rw-r--r--Python/gc_free_threading.c30
-rw-r--r--Python/pystate.c11
13 files changed, 134 insertions, 8 deletions
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
index 9e465fd..281094d 100644
--- a/Include/internal/pycore_gc.h
+++ b/Include/internal/pycore_gc.h
@@ -312,6 +312,18 @@ struct _gc_runtime_state {
collections, and are awaiting to undergo a full collection for
the first time. */
Py_ssize_t long_lived_pending;
+
+ /* gh-117783: Deferred reference counting is not fully implemented yet, so
+ as a temporary measure we treat objects using deferred referenence
+ counting as immortal. */
+ struct {
+ /* Immortalize objects instead of marking them as using deferred
+ reference counting. */
+ int enabled;
+
+ /* Set enabled=1 when the first background thread is created. */
+ int enable_on_thread_created;
+ } immortalize;
#endif
};
@@ -343,6 +355,11 @@ extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp);
extern void _Py_ScheduleGC(PyThreadState *tstate);
extern void _Py_RunGC(PyThreadState *tstate);
+#ifdef Py_GIL_DISABLED
+// gh-117783: Immortalize objects that use deferred reference counting
+extern void _PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp);
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index ca843e1..bb4892e 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -296,8 +296,9 @@ class _ExecutorManagerThread(threading.Thread):
# if there is no pending work item.
def weakref_cb(_,
thread_wakeup=self.thread_wakeup,
- shutdown_lock=self.shutdown_lock):
- mp.util.debug('Executor collected: triggering callback for'
+ shutdown_lock=self.shutdown_lock,
+ mp_util_debug=mp.util.debug):
+ mp_util_debug('Executor collected: triggering callback for'
' QueueManager wakeup')
with shutdown_lock:
thread_wakeup.wakeup()
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 3c9d962..9e7a7d6 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -7,7 +7,8 @@ import sysconfig
import time
import trace
-from test.support import os_helper, MS_WINDOWS, flush_std_streams
+from test.support import (os_helper, MS_WINDOWS, flush_std_streams,
+ suppress_immortalization)
from .cmdline import _parse_args, Namespace
from .findtests import findtests, split_test_packages, list_cases
@@ -526,7 +527,10 @@ class Regrtest:
if self.num_workers:
self._run_tests_mp(runtests, self.num_workers)
else:
- self.run_tests_sequentially(runtests)
+ # gh-117783: don't immortalize deferred objects when tracking
+ # refleaks. Only releveant for the free-threaded build.
+ with suppress_immortalization(runtests.hunt_refleak):
+ self.run_tests_sequentially(runtests)
coverage = self.results.get_coverage_results()
self.display_result(runtests)
diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py
index 235029d..fc2f271 100644
--- a/Lib/test/libregrtest/single.py
+++ b/Lib/test/libregrtest/single.py
@@ -303,7 +303,10 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult:
result = TestResult(test_name)
pgo = runtests.pgo
try:
- _runtest(result, runtests)
+ # gh-117783: don't immortalize deferred objects when tracking
+ # refleaks. Only releveant for the free-threaded build.
+ with support.suppress_immortalization(runtests.hunt_refleak):
+ _runtest(result, runtests)
except:
if not pgo:
msg = traceback.format_exc()
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 70d2610..15f6543 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -516,6 +516,25 @@ def has_no_debug_ranges():
def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
return unittest.skipIf(has_no_debug_ranges(), reason)
+@contextlib.contextmanager
+def suppress_immortalization(suppress=True):
+ """Suppress immortalization of deferred objects."""
+ try:
+ import _testinternalcapi
+ except ImportError:
+ yield
+ return
+
+ if not suppress:
+ yield
+ return
+
+ old_values = _testinternalcapi.set_immortalize_deferred(False)
+ try:
+ yield
+ finally:
+ _testinternalcapi.set_immortalize_deferred(*old_values)
+
MS_WINDOWS = (sys.platform == 'win32')
# Is not actually used in tests, but is kept for compatibility.
diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py
index 8e84d00..90665a7 100644
--- a/Lib/test/test_capi/test_watchers.py
+++ b/Lib/test/test_capi/test_watchers.py
@@ -1,7 +1,9 @@
import unittest
from contextlib import contextmanager, ExitStack
-from test.support import catch_unraisable_exception, import_helper, gc_collect
+from test.support import (
+ catch_unraisable_exception, import_helper,
+ gc_collect, suppress_immortalization)
# Skip this test if the _testcapi module isn't available.
@@ -382,6 +384,7 @@ class TestCodeObjectWatchers(unittest.TestCase):
self.assertEqual(
exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1))
+ @suppress_immortalization()
def test_code_object_events_dispatched(self):
# verify that all counts are zero before any watchers are registered
self.assert_event_counts(0, 0, 0, 0)
@@ -428,6 +431,7 @@ class TestCodeObjectWatchers(unittest.TestCase):
self.assertIsNone(cm.unraisable.object)
self.assertEqual(str(cm.unraisable.exc_value), "boom!")
+ @suppress_immortalization()
def test_dealloc_error(self):
co = _testcapi.code_newempty("test_watchers", "dummy0", 0)
with self.code_watcher(2):
diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
index fe8c672..aa793f5 100644
--- a/Lib/test/test_code.py
+++ b/Lib/test/test_code.py
@@ -141,7 +141,8 @@ except ImportError:
ctypes = None
from test.support import (cpython_only,
check_impl_detail, requires_debug_ranges,
- gc_collect, Py_GIL_DISABLED)
+ gc_collect, Py_GIL_DISABLED,
+ suppress_immortalization)
from test.support.script_helper import assert_python_ok
from test.support import threading_helper, import_helper
from test.support.bytecode_helper import instructions_with_positions
@@ -577,6 +578,7 @@ class CodeConstsTest(unittest.TestCase):
class CodeWeakRefTest(unittest.TestCase):
+ @suppress_immortalization()
def test_basic(self):
# Create a code object in a clean environment so that we know we have
# the only reference to it left.
@@ -827,6 +829,7 @@ if check_impl_detail(cpython=True) and ctypes is not None:
self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100,
ctypes.c_voidp(100)), 0)
+ @suppress_immortalization()
def test_free_called(self):
# Verify that the provided free function gets invoked
# when the code object is cleaned up.
@@ -854,6 +857,7 @@ if check_impl_detail(cpython=True) and ctypes is not None:
del f
@threading_helper.requires_working_threading()
+ @suppress_immortalization()
def test_free_different_thread(self):
# Freeing a code object on a different thread then
# where the co_extra was set should be safe.
diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py
index ec5f6af..bb4c7cc 100644
--- a/Lib/test/test_functools.py
+++ b/Lib/test/test_functools.py
@@ -1833,6 +1833,7 @@ class TestLRU:
return 1
self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True})
+ @support.suppress_immortalization()
def test_lru_cache_weakrefable(self):
@self.module.lru_cache
def test_function(x):
diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py
index a2f5b9b..df90647 100644
--- a/Lib/test/test_weakref.py
+++ b/Lib/test/test_weakref.py
@@ -13,7 +13,7 @@ import random
import textwrap
from test import support
-from test.support import script_helper, ALWAYS_EQ
+from test.support import script_helper, ALWAYS_EQ, suppress_immortalization
from test.support import gc_collect
from test.support import import_helper
from test.support import threading_helper
@@ -651,6 +651,7 @@ class ReferencesTestCase(TestBase):
# deallocation of c2.
del c2
+ @suppress_immortalization()
def test_callback_in_cycle(self):
import gc
@@ -743,6 +744,7 @@ class ReferencesTestCase(TestBase):
del c1, c2, C, D
gc.collect()
+ @suppress_immortalization()
def test_callback_in_cycle_resurrection(self):
import gc
@@ -878,6 +880,7 @@ class ReferencesTestCase(TestBase):
# No exception should be raised here
gc.collect()
+ @suppress_immortalization()
def test_classes(self):
# Check that classes are weakrefable.
class A(object):
diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c
index b0bba34..99e80ba 100644
--- a/Modules/_testinternalcapi.c
+++ b/Modules/_testinternalcapi.c
@@ -1958,6 +1958,27 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored))
#endif
static PyObject *
+set_immortalize_deferred(PyObject *self, PyObject *value)
+{
+#ifdef Py_GIL_DISABLED
+ PyInterpreterState *interp = PyInterpreterState_Get();
+ int old_enabled = interp->gc.immortalize.enabled;
+ int old_enabled_on_thread = interp->gc.immortalize.enable_on_thread_created;
+ int enabled_on_thread = 0;
+ if (!PyArg_ParseTuple(value, "i|i",
+ &interp->gc.immortalize.enabled,
+ &enabled_on_thread))
+ {
+ return NULL;
+ }
+ interp->gc.immortalize.enable_on_thread_created = enabled_on_thread;
+ return Py_BuildValue("ii", old_enabled, old_enabled_on_thread);
+#else
+ return Py_BuildValue("OO", Py_False, Py_False);
+#endif
+}
+
+static PyObject *
has_inline_values(PyObject *self, PyObject *obj)
{
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
@@ -2050,6 +2071,7 @@ static PyMethodDef module_functions[] = {
#ifdef Py_GIL_DISABLED
{"py_thread_id", get_py_thread_id, METH_NOARGS},
#endif
+ {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS},
{"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
{NULL, NULL} /* sentinel */
};
diff --git a/Objects/object.c b/Objects/object.c
index 91bb011..8d85693 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -2430,6 +2430,13 @@ _PyObject_SetDeferredRefcount(PyObject *op)
assert(PyType_IS_GC(Py_TYPE(op)));
assert(_Py_IsOwnedByCurrentThread(op));
assert(op->ob_ref_shared == 0);
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ if (interp->gc.immortalize.enabled) {
+ // gh-117696: immortalize objects instead of using deferred reference
+ // counting for now.
+ _Py_SetImmortal(op);
+ return;
+ }
op->ob_gc_bits |= _PyGC_BITS_DEFERRED;
op->ob_ref_local += 1;
op->ob_ref_shared = _Py_REF_QUEUED;
diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
index 9cf0e98..8c0940d 100644
--- a/Python/gc_free_threading.c
+++ b/Python/gc_free_threading.c
@@ -704,6 +704,12 @@ _PyGC_Init(PyInterpreterState *interp)
{
GCState *gcstate = &interp->gc;
+ if (_Py_IsMainInterpreter(interp)) {
+ // gh-117783: immortalize objects that would use deferred refcounting
+ // once the first non-main thread is created.
+ gcstate->immortalize.enable_on_thread_created = 1;
+ }
+
gcstate->garbage = PyList_New(0);
if (gcstate->garbage == NULL) {
return _PyStatus_NO_MEMORY();
@@ -1781,6 +1787,30 @@ custom_visitor_wrapper(const mi_heap_t *heap, const mi_heap_area_t *area,
return true;
}
+// gh-117783: Immortalize objects that use deferred reference counting to
+// temporarily work around scaling bottlenecks.
+static bool
+immortalize_visitor(const mi_heap_t *heap, const mi_heap_area_t *area,
+ void *block, size_t block_size, void *args)
+{
+ PyObject *op = op_from_block(block, args, false);
+ if (op != NULL && _PyObject_HasDeferredRefcount(op)) {
+ _Py_SetImmortal(op);
+ op->ob_gc_bits &= ~_PyGC_BITS_DEFERRED;
+ }
+ return true;
+}
+
+void
+_PyGC_ImmortalizeDeferredObjects(PyInterpreterState *interp)
+{
+ struct visitor_args args;
+ _PyEval_StopTheWorld(interp);
+ gc_visit_heaps(interp, &immortalize_visitor, &args);
+ interp->gc.immortalize.enabled = 1;
+ _PyEval_StartTheWorld(interp);
+}
+
void
PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg)
{
diff --git a/Python/pystate.c b/Python/pystate.c
index bca28ce..78b39c9 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -1568,6 +1568,17 @@ new_threadstate(PyInterpreterState *interp, int whence)
// Must be called with lock unlocked to avoid re-entrancy deadlock.
PyMem_RawFree(new_tstate);
}
+ else {
+#ifdef Py_GIL_DISABLED
+ if (interp->gc.immortalize.enable_on_thread_created &&
+ !interp->gc.immortalize.enabled)
+ {
+ // Immortalize objects marked as using deferred reference counting
+ // the first time a non-main thread is created.
+ _PyGC_ImmortalizeDeferredObjects(interp);
+ }
+#endif
+ }
#ifdef Py_GIL_DISABLED
// Must be called with lock unlocked to avoid lock ordering deadlocks.
generated by cgit v0.12 at 2025-08-06 22:50:37 (GMT)