summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Doc/library/sys.rst13
-rw-r--r--Include/cpython/optimizer.h3
-rw-r--r--Lib/test/libregrtest/refleak.py4
-rw-r--r--Lib/test/test_capi/test_opt.py16
-rw-r--r--Lib/test/test_mailbox.py4
-rw-r--r--Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst3
-rw-r--r--Objects/codeobject.c22
-rw-r--r--Python/bytecodes.c23
-rw-r--r--Python/clinic/sysmodule.c.h20
-rw-r--r--Python/generated_cases.c.h25
-rw-r--r--Python/optimizer.c64
-rw-r--r--Python/sysmodule.c17
12 files changed, 130 insertions, 84 deletions
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index a97a369..ad8857f 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -195,6 +195,17 @@ always available.
This function should be used for internal and specialized purposes only.
+ .. deprecated:: 3.13
+ Use the more general :func:`_clear_internal_caches` function instead.
+
+
+.. function:: _clear_internal_caches()
+
+ Clear all internal performance-related caches. Use this function *only* to
+ release unnecessary references and memory blocks when hunting for leaks.
+
+ .. versionadded:: 3.13
+
.. function:: _current_frames()
@@ -724,7 +735,7 @@ always available.
regardless of their size. This function is mainly useful for tracking
and debugging memory leaks. Because of the interpreter's internal
caches, the result can vary from call to call; you may have to call
- :func:`_clear_type_cache()` and :func:`gc.collect()` to get more
+ :func:`_clear_internal_caches()` and :func:`gc.collect()` to get more
predictable results.
If a Python build or implementation cannot reasonably compute this
diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h
index 5a9ccae..3928eca 100644
--- a/Include/cpython/optimizer.h
+++ b/Include/cpython/optimizer.h
@@ -24,9 +24,10 @@ typedef struct {
uint8_t opcode;
uint8_t oparg;
uint8_t valid;
- uint8_t linked;
+ int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below).
_PyBloomFilter bloom;
_PyExecutorLinkListNode links;
+ PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR).
} _PyVMData;
typedef struct {
diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py
index 7da16cf..71a70af 100644
--- a/Lib/test/libregrtest/refleak.py
+++ b/Lib/test/libregrtest/refleak.py
@@ -201,8 +201,8 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
# Clear caches
clear_caches()
- # Clear type cache at the end: previous function calls can modify types
- sys._clear_type_cache()
+ # Clear other caches last (previous function calls can re-populate them):
+ sys._clear_internal_caches()
def warm_caches():
diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index 5c8c059..e6b1b55 100644
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -1,5 +1,6 @@
import contextlib
import opcode
+import sys
import textwrap
import unittest
@@ -181,6 +182,21 @@ class TestExecutorInvalidation(unittest.TestCase):
_testinternalcapi.invalidate_executors(f.__code__)
self.assertFalse(exe.is_valid())
+ def test_sys__clear_internal_caches(self):
+ def f():
+ for _ in range(1000):
+ pass
+ opt = _testinternalcapi.get_uop_optimizer()
+ with temporary_optimizer(opt):
+ f()
+ exe = get_first_executor(f)
+ self.assertIsNotNone(exe)
+ self.assertTrue(exe.is_valid())
+ sys._clear_internal_caches()
+ self.assertFalse(exe.is_valid())
+ exe = get_first_executor(f)
+ self.assertIsNone(exe)
+
class TestUops(unittest.TestCase):
def test_basic_loop(self):
diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py
index c52c014..d4628f9 100644
--- a/Lib/test/test_mailbox.py
+++ b/Lib/test/test_mailbox.py
@@ -10,6 +10,7 @@ import io
import tempfile
from test import support
from test.support import os_helper
+from test.support import refleak_helper
from test.support import socket_helper
import unittest
import textwrap
@@ -2443,6 +2444,9 @@ class MiscTestCase(unittest.TestCase):
def tearDownModule():
support.reap_children()
+ # reap_children may have re-populated caches:
+ if refleak_helper.hunting_for_refleaks():
+ sys._clear_internal_caches()
if __name__ == '__main__':
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst
new file mode 100644
index 0000000..a1db4de
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-07-18-04-36.gh-issue-114695.o9wP5P.rst
@@ -0,0 +1,3 @@
+Add :func:`sys._clear_internal_caches`, which clears all internal
+performance-related caches (and deprecate the less-general
+:func:`sys._clear_type_cache` function).
diff --git a/Objects/codeobject.c b/Objects/codeobject.c
index dc46b77..30336fa 100644
--- a/Objects/codeobject.c
+++ b/Objects/codeobject.c
@@ -1489,27 +1489,19 @@ PyCode_GetFreevars(PyCodeObject *code)
static void
clear_executors(PyCodeObject *co)
{
+ assert(co->co_executors);
for (int i = 0; i < co->co_executors->size; i++) {
- Py_CLEAR(co->co_executors->executors[i]);
+ if (co->co_executors->executors[i]) {
+ _Py_ExecutorClear(co->co_executors->executors[i]);
+ }
}
PyMem_Free(co->co_executors);
co->co_executors = NULL;
}
void
-_PyCode_Clear_Executors(PyCodeObject *code) {
- int code_len = (int)Py_SIZE(code);
- for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {
- _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
- uint8_t opcode = instr->op.code;
- uint8_t oparg = instr->op.arg;
- if (opcode == ENTER_EXECUTOR) {
- _PyExecutorObject *exec = code->co_executors->executors[oparg];
- assert(exec->vm_data.opcode != ENTER_EXECUTOR);
- instr->op.code = exec->vm_data.opcode;
- instr->op.arg = exec->vm_data.oparg;
- }
- }
+_PyCode_Clear_Executors(PyCodeObject *code)
+{
clear_executors(code);
}
@@ -2360,10 +2352,10 @@ _PyCode_ConstantKey(PyObject *op)
void
_PyStaticCode_Fini(PyCodeObject *co)
{
- deopt_code(co, _PyCode_CODE(co));
if (co->co_executors != NULL) {
clear_executors(co);
}
+ deopt_code(co, _PyCode_CODE(co));
PyMem_Free(co->co_extra);
if (co->_co_cached != NULL) {
Py_CLEAR(co->_co_cached->_co_code);
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 6fb4d71..197dff4 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2370,23 +2370,12 @@ dummy_func(
CHECK_EVAL_BREAKER();
PyCodeObject *code = _PyFrame_GetCode(frame);
- _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
- if (executor->vm_data.valid) {
- Py_INCREF(executor);
- current_executor = executor;
- GOTO_TIER_TWO();
- }
- else {
- /* ENTER_EXECUTOR will be the first code unit of the instruction */
- assert(oparg < 256);
- code->co_executors->executors[oparg] = NULL;
- opcode = this_instr->op.code = executor->vm_data.opcode;
- this_instr->op.arg = executor->vm_data.oparg;
- oparg = executor->vm_data.oparg;
- Py_DECREF(executor);
- next_instr = this_instr;
- DISPATCH_GOTO();
- }
+ current_executor = code->co_executors->executors[oparg & 255];
+ assert(current_executor->vm_data.index == INSTR_OFFSET() - 1);
+ assert(current_executor->vm_data.code == code);
+ assert(current_executor->vm_data.valid);
+ Py_INCREF(current_executor);
+ GOTO_TIER_TWO();
}
replaced op(_POP_JUMP_IF_FALSE, (cond -- )) {
diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h
index 93b8385..13f4ea8 100644
--- a/Python/clinic/sysmodule.c.h
+++ b/Python/clinic/sysmodule.c.h
@@ -1131,6 +1131,24 @@ sys__clear_type_cache(PyObject *module, PyObject *Py_UNUSED(ignored))
return sys__clear_type_cache_impl(module);
}
+PyDoc_STRVAR(sys__clear_internal_caches__doc__,
+"_clear_internal_caches($module, /)\n"
+"--\n"
+"\n"
+"Clear all internal performance-related caches.");
+
+#define SYS__CLEAR_INTERNAL_CACHES_METHODDEF \
+ {"_clear_internal_caches", (PyCFunction)sys__clear_internal_caches, METH_NOARGS, sys__clear_internal_caches__doc__},
+
+static PyObject *
+sys__clear_internal_caches_impl(PyObject *module);
+
+static PyObject *
+sys__clear_internal_caches(PyObject *module, PyObject *Py_UNUSED(ignored))
+{
+ return sys__clear_internal_caches_impl(module);
+}
+
PyDoc_STRVAR(sys_is_finalizing__doc__,
"is_finalizing($module, /)\n"
"--\n"
@@ -1486,4 +1504,4 @@ exit:
#ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
#define SYS_GETANDROIDAPILEVEL_METHODDEF
#endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=3dc3b2cb0ce38ebb input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b8b1c53e04c3b20c input=a9049054013a1b77]*/
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 16f1db3..e524414 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -2363,29 +2363,18 @@
}
TARGET(ENTER_EXECUTOR) {
- _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
+ frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(ENTER_EXECUTOR);
TIER_ONE_ONLY
CHECK_EVAL_BREAKER();
PyCodeObject *code = _PyFrame_GetCode(frame);
- _PyExecutorObject *executor = code->co_executors->executors[oparg & 255];
- if (executor->vm_data.valid) {
- Py_INCREF(executor);
- current_executor = executor;
- GOTO_TIER_TWO();
- }
- else {
- /* ENTER_EXECUTOR will be the first code unit of the instruction */
- assert(oparg < 256);
- code->co_executors->executors[oparg] = NULL;
- opcode = this_instr->op.code = executor->vm_data.opcode;
- this_instr->op.arg = executor->vm_data.oparg;
- oparg = executor->vm_data.oparg;
- Py_DECREF(executor);
- next_instr = this_instr;
- DISPATCH_GOTO();
- }
+ current_executor = code->co_executors->executors[oparg & 255];
+ assert(current_executor->vm_data.index == INSTR_OFFSET() - 1);
+ assert(current_executor->vm_data.code == code);
+ assert(current_executor->vm_data.valid);
+ Py_INCREF(current_executor);
+ GOTO_TIER_TWO();
DISPATCH();
}
diff --git a/Python/optimizer.c b/Python/optimizer.c
index d71ca0a..ad9ac38 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -73,25 +73,21 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO
Py_INCREF(executor);
if (instr->op.code == ENTER_EXECUTOR) {
assert(index == instr->op.arg);
- _PyExecutorObject *old = code->co_executors->executors[index];
- executor->vm_data.opcode = old->vm_data.opcode;
- executor->vm_data.oparg = old->vm_data.oparg;
- old->vm_data.opcode = 0;
- code->co_executors->executors[index] = executor;
- Py_DECREF(old);
+ _Py_ExecutorClear(code->co_executors->executors[index]);
}
else {
assert(code->co_executors->size == index);
assert(code->co_executors->capacity > index);
- executor->vm_data.opcode = instr->op.code;
- executor->vm_data.oparg = instr->op.arg;
- code->co_executors->executors[index] = executor;
- assert(index < MAX_EXECUTORS_SIZE);
- instr->op.code = ENTER_EXECUTOR;
- instr->op.arg = index;
code->co_executors->size++;
}
- return;
+ executor->vm_data.opcode = instr->op.code;
+ executor->vm_data.oparg = instr->op.arg;
+ executor->vm_data.code = code;
+ executor->vm_data.index = (int)(instr - _PyCode_CODE(code));
+ code->co_executors->executors[index] = executor;
+ assert(index < MAX_EXECUTORS_SIZE);
+ instr->op.code = ENTER_EXECUTOR;
+ instr->op.arg = index;
}
int
@@ -1071,7 +1067,7 @@ link_executor(_PyExecutorObject *executor)
}
head->vm_data.links.next = executor;
}
- executor->vm_data.linked = true;
+ executor->vm_data.valid = true;
/* executor_list_head must be first in list */
assert(interp->executor_list_head->vm_data.links.previous == NULL);
}
@@ -1079,7 +1075,7 @@ link_executor(_PyExecutorObject *executor)
static void
unlink_executor(_PyExecutorObject *executor)
{
- if (!executor->vm_data.linked) {
+ if (!executor->vm_data.valid) {
return;
}
_PyExecutorLinkListNode *links = &executor->vm_data.links;
@@ -1097,7 +1093,7 @@ unlink_executor(_PyExecutorObject *executor)
assert(interp->executor_list_head == executor);
interp->executor_list_head = next;
}
- executor->vm_data.linked = false;
+ executor->vm_data.valid = false;
}
/* This must be called by optimizers before using the executor */
@@ -1116,12 +1112,24 @@ void
_Py_ExecutorClear(_PyExecutorObject *executor)
{
unlink_executor(executor);
+ PyCodeObject *code = executor->vm_data.code;
+ if (code == NULL) {
+ return;
+ }
+ _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index];
+ assert(instruction->op.code == ENTER_EXECUTOR);
+ int index = instruction->op.arg;
+ assert(code->co_executors->executors[index] == executor);
+ instruction->op.code = executor->vm_data.opcode;
+ instruction->op.arg = executor->vm_data.oparg;
+ executor->vm_data.code = NULL;
+ Py_CLEAR(code->co_executors->executors[index]);
}
void
_Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj)
{
- assert(executor->vm_data.valid = true);
+ assert(executor->vm_data.valid);
_Py_BloomFilter_Add(&executor->vm_data.bloom, obj);
}
@@ -1140,8 +1148,7 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
assert(exec->vm_data.valid);
_PyExecutorObject *next = exec->vm_data.links.next;
if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) {
- exec->vm_data.valid = false;
- unlink_executor(exec);
+ _Py_ExecutorClear(exec);
}
exec = next;
}
@@ -1151,15 +1158,14 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj)
void
_Py_Executors_InvalidateAll(PyInterpreterState *interp)
{
- /* Walk the list of executors */
- for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) {
- assert(exec->vm_data.valid);
- _PyExecutorObject *next = exec->vm_data.links.next;
- exec->vm_data.links.next = NULL;
- exec->vm_data.links.previous = NULL;
- exec->vm_data.valid = false;
- exec->vm_data.linked = false;
- exec = next;
+ while (interp->executor_list_head) {
+ _PyExecutorObject *executor = interp->executor_list_head;
+ if (executor->vm_data.code) {
+ // Clear the entire code object so its co_executors array be freed:
+ _PyCode_Clear_Executors(executor->vm_data.code);
+ }
+ else {
+ _Py_ExecutorClear(executor);
+ }
}
- interp->executor_list_head = NULL;
}
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 437d7f8..69b6d88 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2127,6 +2127,22 @@ sys__clear_type_cache_impl(PyObject *module)
Py_RETURN_NONE;
}
+/*[clinic input]
+sys._clear_internal_caches
+
+Clear all internal performance-related caches.
+[clinic start generated code]*/
+
+static PyObject *
+sys__clear_internal_caches_impl(PyObject *module)
+/*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/
+{
+ PyInterpreterState *interp = _PyInterpreterState_GET();
+ _Py_Executors_InvalidateAll(interp);
+ PyType_ClearCache();
+ Py_RETURN_NONE;
+}
+
/* Note that, for now, we do not have a per-interpreter equivalent
for sys.is_finalizing(). */
@@ -2461,6 +2477,7 @@ static PyMethodDef sys_methods[] = {
{"audit", _PyCFunction_CAST(sys_audit), METH_FASTCALL, audit_doc },
{"breakpointhook", _PyCFunction_CAST(sys_breakpointhook),
METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc},
+ SYS__CLEAR_INTERNAL_CACHES_METHODDEF
SYS__CLEAR_TYPE_CACHE_METHODDEF
SYS__CURRENT_FRAMES_METHODDEF
SYS__CURRENT_EXCEPTIONS_METHODDEF
generated by cgit v0.12 at 2025-02-16 23:43:18 (GMT)