summaryrefslogtreecommitdiffstats
path: root/Python/bytecodes.c
diff options
context:
space:
mode:
Diffstat (limited to 'Python/bytecodes.c')
-rw-r--r--Python/bytecodes.c442
1 files changed, 347 insertions, 95 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 72f85cc..5c6398a 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -14,6 +14,7 @@
#include "pycore_function.h"
#include "pycore_intrinsics.h"
#include "pycore_long.h" // _PyLong_GetZero()
+#include "pycore_instruments.h"
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_opcode.h" // EXTRA_CASES
@@ -134,11 +135,45 @@ dummy_func(
inst(RESUME, (--)) {
assert(tstate->cframe == &cframe);
assert(frame == cframe.current_frame);
- if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
+ /* Possibly combine this with eval breaker */
+ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) {
+ int err = _Py_Instrument(frame->f_code, tstate->interp);
+ ERROR_IF(err, error);
+ next_instr--;
+ }
+ else if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
goto handle_eval_breaker;
}
}
+ inst(INSTRUMENTED_RESUME, (--)) {
+ /* Possible performance enhancement:
+ * We need to check the eval breaker anyway, can we
+ * combine the instrument verison check and the eval breaker test?
+ */
+ if (frame->f_code->_co_instrumentation_version != tstate->interp->monitoring_version) {
+ if (_Py_Instrument(frame->f_code, tstate->interp)) {
+ goto error;
+ }
+ next_instr--;
+ }
+ else {
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ int err = _Py_call_instrumentation(
+ tstate, oparg > 0, frame, next_instr-1);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ ERROR_IF(err, error);
+ if (frame->prev_instr != next_instr-1) {
+ /* Instrumentation has jumped */
+ next_instr = frame->prev_instr;
+ DISPATCH();
+ }
+ if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
+ goto handle_eval_breaker;
+ }
+ }
+ }
+
inst(LOAD_CLOSURE, (-- value)) {
/* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
value = GETLOCAL(oparg);
@@ -183,6 +218,34 @@ dummy_func(
macro(END_FOR) = POP_TOP + POP_TOP;
+ inst(INSTRUMENTED_END_FOR, (receiver, value --)) {
+ /* Need to create a fake StopIteration error here,
+ * to conform to PEP 380 */
+ if (PyGen_Check(receiver)) {
+ PyErr_SetObject(PyExc_StopIteration, value);
+ if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+ goto error;
+ }
+ PyErr_SetRaisedException(NULL);
+ }
+ DECREF_INPUTS();
+ }
+
+ inst(END_SEND, (receiver, value -- value)) {
+ Py_DECREF(receiver);
+ }
+
+ inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) {
+ if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {
+ PyErr_SetObject(PyExc_StopIteration, value);
+ if (monitor_stop_iteration(tstate, frame, next_instr-1)) {
+ goto error;
+ }
+ PyErr_SetRaisedException(NULL);
+ }
+ Py_DECREF(receiver);
+ }
+
inst(UNARY_NEGATIVE, (value -- res)) {
res = PyNumber_Negative(value);
DECREF_INPUTS();
@@ -222,7 +285,6 @@ dummy_func(
inst(BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- prod)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -233,7 +295,6 @@ dummy_func(
}
inst(BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- prod)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -243,7 +304,6 @@ dummy_func(
}
inst(BINARY_OP_SUBTRACT_INT, (unused/1, left, right -- sub)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -254,7 +314,6 @@ dummy_func(
}
inst(BINARY_OP_SUBTRACT_FLOAT, (unused/1, left, right -- sub)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -263,7 +322,6 @@ dummy_func(
}
inst(BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -280,7 +338,6 @@ dummy_func(
// specializations, but there is no output.
// At the end we just skip over the STORE_FAST.
inst(BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
_Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
@@ -310,7 +367,6 @@ dummy_func(
}
inst(BINARY_OP_ADD_FLOAT, (unused/1, left, right -- sum)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -320,7 +376,6 @@ dummy_func(
}
inst(BINARY_OP_ADD_INT, (unused/1, left, right -- sum)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
STAT_INC(BINARY_OP, hit);
@@ -342,7 +397,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
@@ -386,7 +440,6 @@ dummy_func(
}
inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);
@@ -403,7 +456,6 @@ dummy_func(
}
inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);
@@ -420,7 +472,6 @@ dummy_func(
}
inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR);
STAT_INC(BINARY_SUBSCR, hit);
res = PyDict_GetItemWithError(dict, sub);
@@ -479,7 +530,6 @@ dummy_func(
inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) {
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG();
@@ -497,7 +547,6 @@ dummy_func(
}
inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);
@@ -517,7 +566,6 @@ dummy_func(
}
inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
STAT_INC(STORE_SUBSCR, hit);
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
@@ -573,7 +621,6 @@ dummy_func(
assert(EMPTY());
/* Restore previous cframe and return. */
tstate->cframe = cframe.previous;
- tstate->cframe->use_tracing = cframe.use_tracing;
assert(tstate->cframe->current_frame == frame->previous);
assert(!_PyErr_Occurred(tstate));
_Py_LeaveRecursiveCallTstate(tstate);
@@ -584,8 +631,24 @@ dummy_func(
STACK_SHRINK(1);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
- TRACE_FUNCTION_EXIT();
- DTRACE_FUNCTION_EXIT();
+ _Py_LeaveRecursiveCallPy(tstate);
+ assert(frame != &entry_frame);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
+ frame = cframe.current_frame = dying->previous;
+ _PyEvalFrameClearAndPop(tstate, dying);
+ _PyFrame_StackPush(frame, retval);
+ goto resume_frame;
+ }
+
+ inst(INSTRUMENTED_RETURN_VALUE, (retval --)) {
+ int err = _Py_call_instrumentation_arg(
+ tstate, PY_MONITORING_EVENT_PY_RETURN,
+ frame, next_instr-1, retval);
+ if (err) goto error;
+ STACK_SHRINK(1);
+ assert(EMPTY());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
@@ -601,8 +664,25 @@ dummy_func(
Py_INCREF(retval);
assert(EMPTY());
_PyFrame_SetStackPointer(frame, stack_pointer);
- TRACE_FUNCTION_EXIT();
- DTRACE_FUNCTION_EXIT();
+ _Py_LeaveRecursiveCallPy(tstate);
+ assert(frame != &entry_frame);
+ // GH-99729: We need to unlink the frame *before* clearing it:
+ _PyInterpreterFrame *dying = frame;
+ frame = cframe.current_frame = dying->previous;
+ _PyEvalFrameClearAndPop(tstate, dying);
+ _PyFrame_StackPush(frame, retval);
+ goto resume_frame;
+ }
+
+ inst(INSTRUMENTED_RETURN_CONST, (--)) {
+ PyObject *retval = GETITEM(frame->f_code->co_consts, oparg);
+ int err = _Py_call_instrumentation_arg(
+ tstate, PY_MONITORING_EVENT_PY_RETURN,
+ frame, next_instr-1, retval);
+ if (err) goto error;
+ Py_INCREF(retval);
+ assert(EMPTY());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
_Py_LeaveRecursiveCallPy(tstate);
assert(frame != &entry_frame);
// GH-99729: We need to unlink the frame *before* clearing it:
@@ -730,7 +810,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PySendCache *cache = (_PySendCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_Send(receiver, next_instr);
DISPATCH_SAME_OPARG();
@@ -739,6 +818,20 @@ dummy_func(
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
assert(frame != &entry_frame);
+ if ((Py_TYPE(receiver) == &PyGen_Type ||
+ Py_TYPE(receiver) == &PyCoro_Type) && ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING)
+ {
+ PyGenObject *gen = (PyGenObject *)receiver;
+ _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe;
+ frame->yield_offset = oparg;
+ STACK_SHRINK(1);
+ _PyFrame_StackPush(gen_frame, v);
+ gen->gi_frame_state = FRAME_EXECUTING;
+ gen->gi_exc_state.previous_item = tstate->exc_info;
+ tstate->exc_info = &gen->gi_exc_state;
+ JUMPBY(INLINE_CACHE_ENTRIES_SEND + oparg);
+ DISPATCH_INLINED(gen_frame);
+ }
if (Py_IsNone(v) && PyIter_Check(receiver)) {
retval = Py_TYPE(receiver)->tp_iternext(receiver);
}
@@ -746,26 +839,22 @@ dummy_func(
retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
}
if (retval == NULL) {
- if (tstate->c_tracefunc != NULL
- && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
- call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
+ if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)
+ ) {
+ monitor_raise(tstate, frame, next_instr-1);
+ }
if (_PyGen_FetchStopIterationValue(&retval) == 0) {
assert(retval != NULL);
JUMPBY(oparg);
}
else {
- assert(retval == NULL);
goto error;
}
}
- else {
- assert(retval != NULL);
- }
Py_DECREF(v);
}
inst(SEND_GEN, (unused/1, receiver, v -- receiver)) {
- assert(cframe.use_tracing == 0);
PyGenObject *gen = (PyGenObject *)receiver;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type &&
Py_TYPE(gen) != &PyCoro_Type, SEND);
@@ -782,6 +871,26 @@ dummy_func(
DISPATCH_INLINED(gen_frame);
}
+ inst(INSTRUMENTED_YIELD_VALUE, (retval -- unused)) {
+ assert(frame != &entry_frame);
+ PyGenObject *gen = _PyFrame_GetGenerator(frame);
+ gen->gi_frame_state = FRAME_SUSPENDED;
+ _PyFrame_SetStackPointer(frame, stack_pointer - 1);
+ int err = _Py_call_instrumentation_arg(
+ tstate, PY_MONITORING_EVENT_PY_YIELD,
+ frame, next_instr-1, retval);
+ if (err) goto error;
+ tstate->exc_info = gen->gi_exc_state.previous_item;
+ gen->gi_exc_state.previous_item = NULL;
+ _Py_LeaveRecursiveCallPy(tstate);
+ _PyInterpreterFrame *gen_frame = frame;
+ frame = cframe.current_frame = frame->previous;
+ gen_frame->previous = NULL;
+ frame->prev_instr -= frame->yield_offset;
+ _PyFrame_StackPush(frame, retval);
+ goto resume_frame;
+ }
+
inst(YIELD_VALUE, (retval -- unused)) {
// NOTE: It's important that YIELD_VALUE never raises an exception!
// The compiler treats any exception raised here as a failed close()
@@ -790,8 +899,6 @@ dummy_func(
PyGenObject *gen = _PyFrame_GetGenerator(frame);
gen->gi_frame_state = FRAME_SUSPENDED;
_PyFrame_SetStackPointer(frame, stack_pointer - 1);
- TRACE_FUNCTION_EXIT();
- DTRACE_FUNCTION_EXIT();
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
_Py_LeaveRecursiveCallPy(tstate);
@@ -930,7 +1037,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
@@ -994,7 +1100,6 @@ dummy_func(
inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) {
#if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
- assert(cframe.use_tracing == 0);
PyObject *name = GETITEM(frame->f_code->co_names, oparg);
next_instr--;
_Py_Specialize_StoreAttr(owner, next_instr, name);
@@ -1111,7 +1216,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1);
next_instr--;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
@@ -1163,7 +1267,6 @@ dummy_func(
}
inst(LOAD_GLOBAL_MODULE, (unused/1, index/1, version/1, unused/1 -- null if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
PyDictObject *dict = (PyDictObject *)GLOBALS();
DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL);
@@ -1177,11 +1280,11 @@ dummy_func(
}
inst(LOAD_GLOBAL_BUILTIN, (unused/1, index/1, mod_version/1, bltn_version/1 -- null if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyDict_CheckExact(GLOBALS()), LOAD_GLOBAL);
DEOPT_IF(!PyDict_CheckExact(BUILTINS()), LOAD_GLOBAL);
PyDictObject *mdict = (PyDictObject *)GLOBALS();
PyDictObject *bdict = (PyDictObject *)BUILTINS();
+ assert(opcode == LOAD_GLOBAL_BUILTIN);
DEOPT_IF(mdict->ma_keys->dk_version != mod_version, LOAD_GLOBAL);
DEOPT_IF(bdict->ma_keys->dk_version != bltn_version, LOAD_GLOBAL);
assert(DK_IS_UNICODE(bdict->ma_keys));
@@ -1465,7 +1568,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
PyObject *name = GETITEM(frame->f_code->co_names, oparg>>1);
next_instr--;
_Py_Specialize_LoadAttr(owner, next_instr, name);
@@ -1511,7 +1613,6 @@ dummy_func(
}
inst(LOAD_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -1528,7 +1629,6 @@ dummy_func(
}
inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict;
assert(dict != NULL);
@@ -1545,7 +1645,6 @@ dummy_func(
}
inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -1576,7 +1675,6 @@ dummy_func(
}
inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
@@ -1590,7 +1688,6 @@ dummy_func(
}
inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyType_Check(cls), LOAD_ATTR);
DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version,
@@ -1606,7 +1703,6 @@ dummy_func(
}
inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner);
@@ -1632,7 +1728,6 @@ dummy_func(
}
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
@@ -1660,7 +1755,6 @@ dummy_func(
}
inst(STORE_ATTR_INSTANCE_VALUE, (unused/1, type_version/2, index/1, value, owner --)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -1681,7 +1775,6 @@ dummy_func(
}
inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -1723,7 +1816,6 @@ dummy_func(
}
inst(STORE_ATTR_SLOT, (unused/1, type_version/2, index/1, value, owner --)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *tp = Py_TYPE(owner);
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
@@ -1746,7 +1838,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
@@ -1761,7 +1852,6 @@ dummy_func(
}
inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
@@ -1777,7 +1867,6 @@ dummy_func(
// Similar to COMPARE_OP_FLOAT
inst(COMPARE_OP_INT, (unused/1, left, right -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP);
DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP);
@@ -1797,7 +1886,6 @@ dummy_func(
// Similar to COMPARE_OP_FLOAT, but for ==, != only
inst(COMPARE_OP_STR, (unused/1, left, right -- res)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP);
STAT_INC(COMPARE_OP, hit);
@@ -2044,7 +2132,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyForIterCache *cache = (_PyForIterCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_ForIter(iter, next_instr, oparg);
DISPATCH_SAME_OPARG();
@@ -2059,13 +2146,12 @@ dummy_func(
if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
goto error;
}
- else if (tstate->c_tracefunc != NULL) {
- call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
- }
+ monitor_raise(tstate, frame, next_instr-1);
_PyErr_Clear(tstate);
}
/* iterator ended normally */
- assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR);
+ assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
+ next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
Py_DECREF(iter);
STACK_SHRINK(1);
/* Jump forward oparg, then skip following END_FOR instruction */
@@ -2075,8 +2161,35 @@ dummy_func(
// Common case: no jump, leave it to the code generator
}
+ inst(INSTRUMENTED_FOR_ITER, ( -- )) {
+ _Py_CODEUNIT *here = next_instr-1;
+ _Py_CODEUNIT *target;
+ PyObject *iter = TOP();
+ PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter);
+ if (next != NULL) {
+ PUSH(next);
+ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER;
+ }
+ else {
+ if (_PyErr_Occurred(tstate)) {
+ if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) {
+ goto error;
+ }
+ monitor_raise(tstate, frame, here);
+ _PyErr_Clear(tstate);
+ }
+ /* iterator ended normally */
+ assert(next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == END_FOR ||
+ next_instr[INLINE_CACHE_ENTRIES_FOR_ITER + oparg].op.code == INSTRUMENTED_END_FOR);
+ STACK_SHRINK(1);
+ Py_DECREF(iter);
+ /* Skip END_FOR */
+ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1;
+ }
+ INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH);
+ }
+
inst(FOR_ITER_LIST, (unused/1, iter -- iter, next)) {
- assert(cframe.use_tracing == 0);
DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER);
_PyListIterObject *it = (_PyListIterObject *)iter;
STAT_INC(FOR_ITER, hit);
@@ -2099,7 +2212,6 @@ dummy_func(
}
inst(FOR_ITER_TUPLE, (unused/1, iter -- iter, next)) {
- assert(cframe.use_tracing == 0);
_PyTupleIterObject *it = (_PyTupleIterObject *)iter;
DEOPT_IF(Py_TYPE(it) != &PyTupleIter_Type, FOR_ITER);
STAT_INC(FOR_ITER, hit);
@@ -2122,7 +2234,6 @@ dummy_func(
}
inst(FOR_ITER_RANGE, (unused/1, iter -- iter, next)) {
- assert(cframe.use_tracing == 0);
_PyRangeIterObject *r = (_PyRangeIterObject *)iter;
DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER);
STAT_INC(FOR_ITER, hit);
@@ -2143,7 +2254,6 @@ dummy_func(
}
inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) {
- assert(cframe.use_tracing == 0);
PyGenObject *gen = (PyGenObject *)iter;
DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER);
DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER);
@@ -2155,7 +2265,8 @@ dummy_func(
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
- assert(next_instr->op.code == END_FOR);
+ assert(next_instr->op.code == END_FOR ||
+ next_instr->op.code == INSTRUMENTED_END_FOR);
DISPATCH_INLINED(gen_frame);
}
@@ -2264,7 +2375,6 @@ dummy_func(
inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (oparg & 1), res)) {
/* Cached method object */
- assert(cframe.use_tracing == 0);
PyTypeObject *self_cls = Py_TYPE(self);
assert(type_version != 0);
DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
@@ -2283,7 +2393,6 @@ dummy_func(
}
inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *self_cls = Py_TYPE(self);
DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
assert(self_cls->tp_dictoffset == 0);
@@ -2296,7 +2405,6 @@ dummy_func(
}
inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (oparg & 1), res)) {
- assert(cframe.use_tracing == 0);
PyTypeObject *self_cls = Py_TYPE(self);
DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR);
Py_ssize_t dictoffset = self_cls->tp_dictoffset;
@@ -2318,6 +2426,21 @@ dummy_func(
kwnames = GETITEM(frame->f_code->co_consts, oparg);
}
+ inst(INSTRUMENTED_CALL, ( -- )) {
+ int is_meth = PEEK(oparg+2) != NULL;
+ int total_args = oparg + is_meth;
+ PyObject *function = PEEK(total_args + 1);
+ PyObject *arg = total_args == 0 ?
+ &_PyInstrumentation_MISSING : PEEK(total_args);
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, next_instr-1, function, arg);
+ ERROR_IF(err, error);
+ _PyCallCache *cache = (_PyCallCache *)next_instr;
+ INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+ GO_TO_INSTRUCTION(CALL);
+ }
+
// Cache layout: counter/1, func_version/2
// Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members!
family(call, INLINE_CACHE_ENTRIES_CALL) = {
@@ -2359,7 +2482,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_Call(callable, next_instr, total_args, kwnames);
DISPATCH_SAME_OPARG();
@@ -2402,16 +2524,26 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
/* Callable is not a normal Python function */
- if (cframe.use_tracing) {
- res = trace_call_function(
- tstate, callable, args,
- positional_args, kwnames);
- }
- else {
- res = PyObject_Vectorcall(
- callable, args,
- positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
- kwnames);
+ res = PyObject_Vectorcall(
+ callable, args,
+ positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+ kwnames);
+ if (opcode == INSTRUMENTED_CALL) {
+ PyObject *arg = total_args == 0 ?
+ &_PyInstrumentation_MISSING : PEEK(total_args);
+ if (res == NULL) {
+ _Py_call_instrumentation_exc2(
+ tstate, PY_MONITORING_EVENT_C_RAISE,
+ frame, next_instr-1, callable, arg);
+ }
+ else {
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_C_RETURN,
+ frame, next_instr-1, callable, arg);
+ if (err < 0) {
+ Py_CLEAR(res);
+ }
+ }
}
kwnames = NULL;
assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
@@ -2504,7 +2636,6 @@ dummy_func(
inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
- assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
PyObject *obj = args[0];
@@ -2517,7 +2648,6 @@ dummy_func(
inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) {
assert(kwnames == NULL);
- assert(cframe.use_tracing == 0);
assert(oparg == 1);
DEOPT_IF(null != NULL, CALL);
DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL);
@@ -2570,7 +2700,6 @@ dummy_func(
}
inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) {
- assert(cframe.use_tracing == 0);
/* Builtin METH_O functions */
assert(kwnames == NULL);
int is_meth = method != NULL;
@@ -2602,7 +2731,6 @@ dummy_func(
}
inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) {
- assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL functions, without keywords */
assert(kwnames == NULL);
int is_meth = method != NULL;
@@ -2638,7 +2766,6 @@ dummy_func(
}
inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, callable, args[oparg] -- res)) {
- assert(cframe.use_tracing == 0);
/* Builtin METH_FASTCALL | METH_KEYWORDS functions */
int is_meth = method != NULL;
int total_args = oparg;
@@ -2674,7 +2801,6 @@ dummy_func(
}
inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) {
- assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* len(o) */
int is_meth = method != NULL;
@@ -2702,7 +2828,6 @@ dummy_func(
}
inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) {
- assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
/* isinstance(o, o2) */
int is_meth = method != NULL;
@@ -2733,7 +2858,6 @@ dummy_func(
// This is secretly a super-instruction
inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) {
- assert(cframe.use_tracing == 0);
assert(kwnames == NULL);
assert(oparg == 1);
assert(method != NULL);
@@ -2882,12 +3006,14 @@ dummy_func(
CHECK_EVAL_BREAKER();
}
+ inst(INSTRUMENTED_CALL_FUNCTION_EX, ( -- )) {
+ GO_TO_INSTRUCTION(CALL_FUNCTION_EX);
+ }
+
inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) {
- if (oparg & 1) {
- // DICT_MERGE is called before this opcode if there are kwargs.
- // It converts all dict subtypes in kwargs into regular dicts.
- assert(PyDict_CheckExact(kwargs));
- }
+ // DICT_MERGE is called before this opcode if there are kwargs.
+ // It converts all dict subtypes in kwargs into regular dicts.
+ assert(kwargs == NULL || PyDict_CheckExact(kwargs));
if (!PyTuple_CheckExact(callargs)) {
if (check_args_iterable(tstate, func, callargs) < 0) {
goto error;
@@ -2899,10 +3025,35 @@ dummy_func(
Py_SETREF(callargs, tuple);
}
assert(PyTuple_CheckExact(callargs));
-
- result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing);
+ EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func);
+ if (opcode == INSTRUMENTED_CALL_FUNCTION_EX &&
+ !PyFunction_Check(func) && !PyMethod_Check(func)
+ ) {
+ PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ?
+ PyTuple_GET_ITEM(callargs, 0) : Py_None;
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_CALL,
+ frame, next_instr-1, func, arg);
+ if (err) goto error;
+ result = PyObject_Call(func, callargs, kwargs);
+ if (result == NULL) {
+ _Py_call_instrumentation_exc2(
+ tstate, PY_MONITORING_EVENT_C_RAISE,
+ frame, next_instr-1, func, arg);
+ }
+ else {
+ int err = _Py_call_instrumentation_2args(
+ tstate, PY_MONITORING_EVENT_C_RETURN,
+ frame, next_instr-1, func, arg);
+ if (err < 0) {
+ Py_CLEAR(result);
+ }
+ }
+ }
+ else {
+ result = PyObject_Call(func, callargs, kwargs);
+ }
DECREF_INPUTS();
-
assert(PEEK(3 + (oparg & 1)) == NULL);
ERROR_IF(result == NULL, error);
CHECK_EVAL_BREAKER();
@@ -3018,7 +3169,6 @@ dummy_func(
#if ENABLE_SPECIALIZATION
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
- assert(cframe.use_tracing == 0);
next_instr--;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
DISPATCH_SAME_OPARG();
@@ -3039,9 +3189,105 @@ dummy_func(
assert(oparg >= 2);
}
- inst(EXTENDED_ARG, (--)) {
+ inst(INSTRUMENTED_LINE, ( -- )) {
+ _Py_CODEUNIT *here = next_instr-1;
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ int original_opcode = _Py_call_instrumentation_line(
+ tstate, frame, here);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (original_opcode < 0) {
+ next_instr = here+1;
+ goto error;
+ }
+ next_instr = frame->prev_instr;
+ if (next_instr != here) {
+ DISPATCH();
+ }
+ if (_PyOpcode_Caches[original_opcode]) {
+ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
+ INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+ }
+ opcode = original_opcode;
+ DISPATCH_GOTO();
+ }
+
+ inst(INSTRUMENTED_INSTRUCTION, ( -- )) {
+ int next_opcode = _Py_call_instrumentation_instruction(
+ tstate, frame, next_instr-1);
+ ERROR_IF(next_opcode < 0, error);
+ next_instr--;
+ if (_PyOpcode_Caches[next_opcode]) {
+ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
+ INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+ }
+ assert(next_opcode > 0 && next_opcode < 256);
+ opcode = next_opcode;
+ DISPATCH_GOTO();
+ }
+
+ inst(INSTRUMENTED_JUMP_FORWARD, ( -- )) {
+ INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP);
+ }
+
+ inst(INSTRUMENTED_JUMP_BACKWARD, ( -- )) {
+ INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP);
+ CHECK_EVAL_BREAKER();
+ }
+
+ inst(INSTRUMENTED_POP_JUMP_IF_TRUE, ( -- )) {
+ PyObject *cond = POP();
+ int err = PyObject_IsTrue(cond);
+ Py_DECREF(cond);
+ ERROR_IF(err < 0, error);
+ _Py_CODEUNIT *here = next_instr-1;
+ assert(err == 0 || err == 1);
+ int offset = err*oparg;
+ INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+ }
+
+ inst(INSTRUMENTED_POP_JUMP_IF_FALSE, ( -- )) {
+ PyObject *cond = POP();
+ int err = PyObject_IsTrue(cond);
+ Py_DECREF(cond);
+ ERROR_IF(err < 0, error);
+ _Py_CODEUNIT *here = next_instr-1;
+ assert(err == 0 || err == 1);
+ int offset = (1-err)*oparg;
+ INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+ }
+
+ inst(INSTRUMENTED_POP_JUMP_IF_NONE, ( -- )) {
+ PyObject *value = POP();
+ _Py_CODEUNIT *here = next_instr-1;
+ int offset;
+ if (Py_IsNone(value)) {
+ _Py_DECREF_NO_DEALLOC(value);
+ offset = oparg;
+ }
+ else {
+ Py_DECREF(value);
+ offset = 0;
+ }
+ INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+ }
+
+ inst(INSTRUMENTED_POP_JUMP_IF_NOT_NONE, ( -- )) {
+ PyObject *value = POP();
+ _Py_CODEUNIT *here = next_instr-1;
+ int offset;
+ if (Py_IsNone(value)) {
+ _Py_DECREF_NO_DEALLOC(value);
+ offset = 0;
+ }
+ else {
+ Py_DECREF(value);
+ offset = oparg;
+ }
+ INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
+ }
+
+ inst(EXTENDED_ARG, ( -- )) {
assert(oparg);
- assert(cframe.use_tracing == 0);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
PRE_DISPATCH_GOTO();
@@ -3049,6 +3295,12 @@ dummy_func(
}
inst(CACHE, (--)) {
+ assert(0 && "Executing a cache.");
+ Py_UNREACHABLE();
+ }
+
+ inst(RESERVED, (--)) {
+ assert(0 && "Executing RESERVED instruction.");
Py_UNREACHABLE();
}
generated by cgit v0.12 at 2025-11-04 02:38:04 (GMT)