#include "Python.h"
#include "opcode_ids.h"
#include "pycore_bitutils.h" // _Py_popcount32
#include "pycore_call.h"
#include "pycore_ceval.h" // _PY_EVAL_EVENTS_BITS
#include "pycore_code.h" // _PyCode_Clear_Executors()
#include "pycore_critical_section.h"
#include "pycore_frame.h"
#include "pycore_interp.h"
#include "pycore_long.h"
#include "pycore_modsupport.h" // _PyModule_CreateInitialized()
#include "pycore_namespace.h"
#include "pycore_object.h"
#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_UINTPTR_RELEASE
#include "pycore_pyerrors.h"
#include "pycore_pystate.h" // _PyInterpreterState_GET()
/* Uncomment this to dump debugging output when assertions fail */
// #define INSTRUMENT_DEBUG 1
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj) \
if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); \
}
#define ASSERT_WORLD_STOPPED() assert(_PyInterpreterState_GET()->stoptheworld.world_stopped);
#else
#define ASSERT_WORLD_STOPPED_OR_LOCKED(obj)
#define ASSERT_WORLD_STOPPED()
#endif
#ifdef Py_GIL_DISABLED
#define LOCK_CODE(code) \
assert(!_PyInterpreterState_GET()->stoptheworld.world_stopped); \
Py_BEGIN_CRITICAL_SECTION(code)
#define UNLOCK_CODE() Py_END_CRITICAL_SECTION()
#define MODIFY_BYTECODE(code, func, ...) \
do { \
PyCodeObject *co = (code); \
for (Py_ssize_t i = 0; i < code->co_tlbc->size; i++) { \
char *bc = co->co_tlbc->entries[i]; \
if (bc == NULL) { \
continue; \
} \
(func)((_Py_CODEUNIT *)bc, __VA_ARGS__); \
} \
} while (0)
#else
#define LOCK_CODE(code)
#define UNLOCK_CODE()
#define MODIFY_BYTECODE(code, func, ...) \
(func)(_PyCode_CODE(code), __VA_ARGS__)
#endif
PyObject _PyInstrumentation_DISABLE = _PyObject_HEAD_INIT(&PyBaseObject_Type);
PyObject _PyInstrumentation_MISSING = _PyObject_HEAD_INIT(&PyBaseObject_Type);
static const int8_t EVENT_FOR_OPCODE[256] = {
[RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
[INSTRUMENTED_RETURN_VALUE] = PY_MONITORING_EVENT_PY_RETURN,
[CALL] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL,
[CALL_KW] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL,
[CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL,
[LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[INSTRUMENTED_LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL,
[RESUME] = -1,
[YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
[INSTRUMENTED_YIELD_VALUE] = PY_MONITORING_EVENT_PY_YIELD,
[JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
[JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
[POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_JUMP_FORWARD] = PY_MONITORING_EVENT_JUMP,
[INSTRUMENTED_JUMP_BACKWARD] = PY_MONITORING_EVENT_JUMP,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[INSTRUMENTED_FOR_ITER] = PY_MONITORING_EVENT_BRANCH_RIGHT,
[END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
[INSTRUMENTED_END_FOR] = PY_MONITORING_EVENT_STOP_ITERATION,
[END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
[INSTRUMENTED_END_SEND] = PY_MONITORING_EVENT_STOP_ITERATION,
[NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,
[INSTRUMENTED_NOT_TAKEN] = PY_MONITORING_EVENT_BRANCH_LEFT,
};
static const uint8_t DE_INSTRUMENT[256] = {
[INSTRUMENTED_RESUME] = RESUME,
[INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE,
[INSTRUMENTED_CALL] = CALL,
[INSTRUMENTED_CALL_KW] = CALL_KW,
[INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
[INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE,
[INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD,
[INSTRUMENTED_JUMP_BACKWARD] = JUMP_BACKWARD,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
[INSTRUMENTED_POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
[INSTRUMENTED_FOR_ITER] = FOR_ITER,
[INSTRUMENTED_END_FOR] = END_FOR,
[INSTRUMENTED_END_SEND] = END_SEND,
[INSTRUMENTED_LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR,
[INSTRUMENTED_NOT_TAKEN] = NOT_TAKEN,
};
static const uint8_t INSTRUMENTED_OPCODES[256] = {
[RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
[INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE,
[CALL] = INSTRUMENTED_CALL,
[INSTRUMENTED_CALL] = INSTRUMENTED_CALL,
[CALL_KW] = INSTRUMENTED_CALL_KW,
[INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW,
[CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX,
[YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
[INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE,
[RESUME] = INSTRUMENTED_RESUME,
[INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME,
[JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
[INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD,
[JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
[INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD,
[POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
[INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE,
[POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
[INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE,
[POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
[INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE,
[POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
[INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE,
[END_FOR] = INSTRUMENTED_END_FOR,
[INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR,
[END_SEND] = INSTRUMENTED_END_SEND,
[INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND,
[FOR_ITER] = INSTRUMENTED_FOR_ITER,
[INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER,
[LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR,
[NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,
[INSTRUMENTED_NOT_TAKEN] = INSTRUMENTED_NOT_TAKEN,
[INSTRUMENTED_LINE] = INSTRUMENTED_LINE,
[INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION,
};
static inline bool
opcode_has_event(int opcode)
{
return (
opcode != INSTRUMENTED_LINE &&
INSTRUMENTED_OPCODES[opcode] > 0
);
}
static inline bool
is_instrumented(int opcode)
{
assert(opcode != 0);
assert(opcode != RESERVED);
assert(opcode != ENTER_EXECUTOR);
return opcode >= MIN_INSTRUMENTED_OPCODE;
}
#ifndef NDEBUG
static inline bool
monitors_equals(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (a.tools[i] != b.tools[i]) {
return false;
}
}
return true;
}
#endif
static inline _Py_LocalMonitors
monitors_sub(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] & ~b.tools[i];
}
return res;
}
#ifndef NDEBUG
static inline _Py_LocalMonitors
monitors_and(_Py_LocalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] & b.tools[i];
}
return res;
}
#endif
/* The union of the *local* events in a and b.
* Global events like RAISE are ignored.
* Used for instrumentation, as only local
* events get instrumented.
*/
static inline _Py_LocalMonitors
local_union(_Py_GlobalMonitors a, _Py_LocalMonitors b)
{
_Py_LocalMonitors res;
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
res.tools[i] = a.tools[i] | b.tools[i];
}
return res;
}
static inline bool
monitors_are_empty(_Py_LocalMonitors m)
{
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (m.tools[i]) {
return false;
}
}
return true;
}
static inline bool
multiple_tools(_Py_LocalMonitors *m)
{
for (int i = 0; i < _PY_MONITORING_LOCAL_EVENTS; i++) {
if (_Py_popcount32(m->tools[i]) > 1) {
return true;
}
}
return false;
}
static inline _PyMonitoringEventSet
get_local_events(_Py_LocalMonitors *m, int tool_id)
{
_PyMonitoringEventSet result = 0;
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
if ((m->tools[e] >> tool_id) & 1) {
result |= (1 << e);
}
}
return result;
}
static inline _PyMonitoringEventSet
get_events(_Py_GlobalMonitors *m, int tool_id)
{
_PyMonitoringEventSet result = 0;
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
if ((m->tools[e] >> tool_id) & 1) {
result |= (1 << e);
}
}
return result;
}
/* Line delta.
* 8 bit value.
* if line_delta == -128:
* line = None # represented as -1
* elif line_delta == -127 or line_delta == -126:
* line = PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
* else:
* line = first_line + (offset >> OFFSET_SHIFT) + line_delta;
*/
#define NO_LINE -128
#define COMPUTED_LINE_LINENO_CHANGE -127
#define COMPUTED_LINE -126
#define OFFSET_SHIFT 4
static int8_t
compute_line_delta(PyCodeObject *code, int offset, int line)
{
if (line < 0) {
return NO_LINE;
}
int delta = line - code->co_firstlineno - (offset >> OFFSET_SHIFT);
if (delta <= INT8_MAX && delta > COMPUTED_LINE) {
return delta;
}
return COMPUTED_LINE;
}
static int
compute_line(PyCodeObject *code, int offset, int8_t line_delta)
{
if (line_delta > COMPUTED_LINE) {
return code->co_firstlineno + (offset >> OFFSET_SHIFT) + line_delta;
}
if (line_delta == NO_LINE) {
return -1;
}
assert(line_delta == COMPUTED_LINE || line_delta == COMPUTED_LINE_LINENO_CHANGE);
/* Look it up */
return PyCode_Addr2Line(code, offset * sizeof(_Py_CODEUNIT));
}
int
_PyInstruction_GetLength(PyCodeObject *code, int offset)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_Py_CODEUNIT inst = _Py_GetBaseCodeUnit(code, offset);
return 1 + _PyOpcode_Caches[inst.op.code];
}
#ifdef INSTRUMENT_DEBUG
static void
dump_instrumentation_data_tools(PyCodeObject *code, uint8_t *tools, int i, FILE*out)
{
if (tools == NULL) {
fprintf(out, "tools = NULL");
}
else {
fprintf(out, "tools = %d", tools[i]);
}
}
static void
dump_instrumentation_data_lines(PyCodeObject *code, _PyCoLineInstrumentationData *lines, int i, FILE*out)
{
if (lines == NULL) {
fprintf(out, ", lines = NULL");
}
else if (lines[i].original_opcode == 0) {
fprintf(out, ", lines = {original_opcode = No LINE (0), line_delta = %d)", lines[i].line_delta);
}
else {
fprintf(out, ", lines = {original_opcode = %s, line_delta = %d)", _PyOpcode_OpName[lines[i].original_opcode], lines[i].line_delta);
}
}
static void
dump_instrumentation_data_line_tools(PyCodeObject *code, uint8_t *line_tools, int i, FILE*out)
{
if (line_tools == NULL) {
fprintf(out, ", line_tools = NULL");
}
else {
fprintf(out, ", line_tools = %d", line_tools[i]);
}
}
static void
dump_instrumentation_data_per_instruction(PyCodeObject *code, _PyCoMonitoringData *data, int i, FILE*out)
{
if (data->per_instruction_opcodes == NULL) {
fprintf(out, ", per-inst opcode = NULL");
}
else {
fprintf(out, ", per-inst opcode = %s", _PyOpcode_OpName[data->per_instruction_opcodes[i]]);
}
if (data->per_instruction_tools == NULL) {
fprintf(out, ", per-inst tools = NULL");
}
else {
fprintf(out, ", per-inst tools = %d", data->per_instruction_tools[i]);
}
}
static void
dump_global_monitors(const char *prefix, _Py_GlobalMonitors monitors, FILE*out)
{
fprintf(out, "%s monitors:\n", prefix);
for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) {
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
}
}
static void
dump_local_monitors(const char *prefix, _Py_LocalMonitors monitors, FILE*out)
{
fprintf(out, "%s monitors:\n", prefix);
for (int event = 0; event < _PY_MONITORING_LOCAL_EVENTS; event++) {
fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]);
}
}
/* No error checking -- Don't use this for anything but experimental debugging */
static void
dump_instrumentation_data(PyCodeObject *code, int star, FILE*out)
{
_PyCoMonitoringData *data = code->_co_monitoring;
fprintf(out, "\n");
PyObject_Print(code->co_name, out, Py_PRINT_RAW);
fprintf(out, "\n");
if (data == NULL) {
fprintf(out, "NULL\n");
return;
}
dump_global_monitors("Global", _PyInterpreterState_GET()->monitors, out);
dump_local_monitors("Code", data->local_monitors, out);
dump_local_monitors("Active", data->active_monitors, out);
int code_len = (int)Py_SIZE(code);
bool starred = false;
for (int i = 0; i < code_len; i += _PyInstruction_GetLength(code, i)) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
if (i == star) {
fprintf(out, "** ");
starred = true;
}
fprintf(out, "Offset: %d, line: %d %s: ", i, PyCode_Addr2Line(code, i*2), _PyOpcode_OpName[opcode]);
dump_instrumentation_data_tools(code, data->tools, i, out);
dump_instrumentation_data_lines(code, data->lines, i, out);
dump_instrumentation_data_line_tools(code, data->line_tools, i, out);
dump_instrumentation_data_per_instruction(code, data, i, out);
fprintf(out, "\n");
;
}
if (!starred && star >= 0) {
fprintf(out, "Error offset not at valid instruction offset: %d\n", star);
fprintf(out, " ");
dump_instrumentation_data_tools(code, data->tools, star, out);
dump_instrumentation_data_lines(code, data->lines, star, out);
dump_instrumentation_data_line_tools(code, data->line_tools, star, out);
dump_instrumentation_data_per_instruction(code, data, star, out);
fprintf(out, "\n");
}
}
#define CHECK(test) do { \
if (!(test)) { \
dump_instrumentation_data(code, i, stderr); \
} \
assert(test); \
} while (0)
static bool
valid_opcode(int opcode)
{
if (opcode == INSTRUMENTED_LINE) {
return true;
}
if (IS_VALID_OPCODE(opcode) &&
opcode != CACHE &&
opcode != RESERVED &&
opcode < 255)
{
return true;
}
return false;
}
static void
sanity_check_instrumentation(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *data = code->_co_monitoring;
if (data == NULL) {
return;
}
_Py_GlobalMonitors global_monitors = _PyInterpreterState_GET()->monitors;
_Py_LocalMonitors active_monitors;
if (code->_co_monitoring) {
_Py_LocalMonitors local_monitors = code->_co_monitoring->local_monitors;
active_monitors = local_union(global_monitors, local_monitors);
}
else {
_Py_LocalMonitors empty = (_Py_LocalMonitors) { 0 };
active_monitors = local_union(global_monitors, empty);
}
assert(monitors_equals(
code->_co_monitoring->active_monitors,
active_monitors));
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len;) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
int base_opcode = _Py_GetBaseCodeUnit(code, offset).op.code;
CHECK(valid_opcode(opcode));
CHECK(valid_opcode(base_opcode));
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = data->per_instruction_opcodes[i];
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);
}
}
if (opcode == INSTRUMENTED_LINE) {
CHECK(data->lines);
CHECK(valid_opcode(data->lines[i].original_opcode));
opcode = data->lines[i].original_opcode;
CHECK(opcode != END_FOR);
CHECK(opcode != RESUME);
CHECK(opcode != RESUME_CHECK);
CHECK(opcode != INSTRUMENTED_RESUME);
if (!is_instrumented(opcode)) {
CHECK(_PyOpcode_Deopt[opcode] == opcode);
}
CHECK(opcode != INSTRUMENTED_LINE);
}
else if (data->lines) {
/* If original_opcode is INSTRUMENTED_INSTRUCTION
* *and* we are executing a INSTRUMENTED_LINE instruction
* that has de-instrumented itself, then we will execute
* an invalid INSTRUMENTED_INSTRUCTION */
CHECK(data->lines[i].original_opcode != INSTRUMENTED_INSTRUCTION);
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
CHECK(data->per_instruction_opcodes[i] != 0);
opcode = data->per_instruction_opcodes[i];
}
if (is_instrumented(opcode)) {
CHECK(DE_INSTRUMENT[opcode] == base_opcode);
int event = EVENT_FOR_OPCODE[DE_INSTRUMENT[opcode]];
if (event < 0) {
/* RESUME fixup */
event = instr->op.arg ? 1: 0;
}
CHECK(active_monitors.tools[event] != 0);
}
if (data->lines && base_opcode != END_FOR) {
int line1 = compute_line(code, i, data->lines[i].line_delta);
int line2 = PyCode_Addr2Line(code, i*sizeof(_Py_CODEUNIT));
CHECK(line1 == line2);
}
CHECK(valid_opcode(opcode));
if (data->tools) {
uint8_t local_tools = data->tools[i];
if (opcode_has_event(base_opcode)) {
int event = EVENT_FOR_OPCODE[base_opcode];
if (event == -1) {
/* RESUME fixup */
event = _PyCode_CODE(code)[i].op.arg;
}
CHECK((active_monitors.tools[event] & local_tools) == local_tools);
}
else {
CHECK(local_tools == 0xff);
}
}
i += _PyInstruction_GetLength(code, i);
assert(i <= code_len);
}
}
#else
#define CHECK(test) assert(test)
#endif
/* Get the underlying code unit, stripping instrumentation and ENTER_EXECUTOR */
_Py_CODEUNIT
_Py_GetBaseCodeUnit(PyCodeObject *code, int i)
{
_Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i;
_Py_CODEUNIT inst = {
.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)};
int opcode = inst.op.code;
if (opcode < MIN_INSTRUMENTED_OPCODE) {
inst.op.code = _PyOpcode_Deopt[opcode];
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
return inst;
}
if (opcode == ENTER_EXECUTOR) {
_PyExecutorObject *exec = code->co_executors->executors[inst.op.arg];
opcode = _PyOpcode_Deopt[exec->vm_data.opcode];
inst.op.code = opcode;
inst.op.arg = exec->vm_data.oparg;
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
return inst;
}
if (opcode == INSTRUMENTED_LINE) {
opcode = code->_co_monitoring->lines[i].original_opcode;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
}
CHECK(opcode != INSTRUMENTED_INSTRUCTION);
CHECK(opcode != INSTRUMENTED_LINE);
int deinstrumented = DE_INSTRUMENT[opcode];
if (deinstrumented) {
inst.op.code = deinstrumented;
}
else {
inst.op.code = _PyOpcode_Deopt[opcode];
}
assert(inst.op.code < MIN_SPECIALIZED_OPCODE);
return inst;
}
static void
de_instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i,
int event)
{
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(event != PY_MONITORING_EVENT_LINE);
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = &monitoring->lines[i].original_opcode;
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode_ptr = &monitoring->per_instruction_opcodes[i];
opcode = *opcode_ptr;
}
int deinstrumented = DE_INSTRUMENT[opcode];
if (deinstrumented == 0) {
return;
}
CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented);
if (_PyOpcode_Caches[deinstrumented]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
}
static void
de_instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring,
int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
int opcode = instr->op.code;
if (opcode != INSTRUMENTED_LINE) {
return;
}
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
int original_opcode = lines->original_opcode;
if (original_opcode == INSTRUMENTED_INSTRUCTION) {
lines->original_opcode = monitoring->per_instruction_opcodes[i];
}
CHECK(original_opcode != 0);
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
FT_ATOMIC_STORE_UINT8(instr->op.code, original_opcode);
if (_PyOpcode_Caches[original_opcode]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
assert(instr->op.code != INSTRUMENTED_LINE);
}
static void
de_instrument_per_instruction(_Py_CODEUNIT *bytecode,
_PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
opcode_ptr = &monitoring->lines[i].original_opcode;
opcode = *opcode_ptr;
}
if (opcode != INSTRUMENTED_INSTRUCTION) {
return;
}
int original_opcode = monitoring->per_instruction_opcodes[i];
CHECK(original_opcode != 0);
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, original_opcode);
if (_PyOpcode_Caches[original_opcode]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);
assert(instr->op.code != INSTRUMENTED_INSTRUCTION);
}
static void
instrument(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode =*opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
opcode_ptr = &lines->original_opcode;
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode_ptr = &monitoring->per_instruction_opcodes[i];
opcode = *opcode_ptr;
CHECK(opcode != INSTRUMENTED_INSTRUCTION && opcode != INSTRUMENTED_LINE);
CHECK(opcode == _PyOpcode_Deopt[opcode]);
}
CHECK(opcode != 0);
if (!is_instrumented(opcode)) {
int deopt = _PyOpcode_Deopt[opcode];
int instrumented = INSTRUMENTED_OPCODES[deopt];
assert(instrumented);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented);
if (_PyOpcode_Caches[deopt]) {
FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.value_and_backoff,
adaptive_counter_warmup().value_and_backoff);
}
}
}
static void
instrument_line(_Py_CODEUNIT *bytecode, _PyCoMonitoringData *monitoring, int i)
{
uint8_t *opcode_ptr = &bytecode[i].op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
return;
}
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
lines->original_opcode = _PyOpcode_Deopt[opcode];
CHECK(lines->original_opcode > 0);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_LINE);
}
static void
instrument_per_instruction(_Py_CODEUNIT *bytecode,
_PyCoMonitoringData *monitoring, int i)
{
_Py_CODEUNIT *instr = &bytecode[i];
uint8_t *opcode_ptr = &instr->op.code;
int opcode = *opcode_ptr;
if (opcode == INSTRUMENTED_LINE) {
_PyCoLineInstrumentationData *lines = &monitoring->lines[i];
opcode_ptr = &lines->original_opcode;
opcode = *opcode_ptr;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
assert(monitoring->per_instruction_opcodes[i] > 0);
return;
}
CHECK(opcode != 0);
if (is_instrumented(opcode)) {
monitoring->per_instruction_opcodes[i] = opcode;
}
else {
assert(opcode != 0);
assert(_PyOpcode_Deopt[opcode] != 0);
assert(_PyOpcode_Deopt[opcode] != RESUME);
monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
}
assert(monitoring->per_instruction_opcodes[i] > 0);
FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, INSTRUMENTED_INSTRUCTION);
}
static void
remove_tools(PyCodeObject * code, int offset, int event, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
assert(opcode_has_event(_Py_GetBaseCodeUnit(code, offset).op.code));
_PyCoMonitoringData *monitoring = code->_co_monitoring;
bool should_de_instrument;
if (monitoring && monitoring->tools) {
monitoring->tools[offset] &= ~tools;
should_de_instrument = (monitoring->tools[offset] == 0);
}
else {
/* Single tool */
uint8_t single_tool = code->_co_monitoring->active_monitors.tools[event];
assert(_Py_popcount32(single_tool) <= 1);
should_de_instrument = ((single_tool & tools) == single_tool);
}
if (should_de_instrument) {
MODIFY_BYTECODE(code, de_instrument, monitoring, offset, event);
}
}
#ifndef NDEBUG
static bool
tools_is_subset_for_event(PyCodeObject * code, int event, int tools)
{
int global_tools = _PyInterpreterState_GET()->monitors.tools[event];
int local_tools = code->_co_monitoring->local_monitors.tools[event];
return tools == ((global_tools | local_tools) & tools);
}
#endif
static void
remove_line_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(monitoring);
bool should_de_instrument;
if (monitoring->line_tools)
{
uint8_t *toolsptr = &monitoring->line_tools[offset];
*toolsptr &= ~tools;
should_de_instrument = (*toolsptr == 0);
}
else {
/* Single tool */
uint8_t single_tool = monitoring->active_monitors.tools[PY_MONITORING_EVENT_LINE];
assert(_Py_popcount32(single_tool) <= 1);
should_de_instrument = ((single_tool & tools) == single_tool);
}
if (should_de_instrument) {
MODIFY_BYTECODE(code, de_instrument_line, monitoring, offset);
}
}
static void
add_tools(PyCodeObject * code, int offset, int event, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
assert(code->_co_monitoring);
if (code->_co_monitoring &&
code->_co_monitoring->tools
) {
code->_co_monitoring->tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
assert(tools_is_subset_for_event(code, event, tools));
}
MODIFY_BYTECODE(code, instrument, code->_co_monitoring, offset);
}
static void
add_line_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->line_tools) {
code->_co_monitoring->line_tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
}
MODIFY_BYTECODE(code, instrument_line, code->_co_monitoring, offset);
}
static void
add_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools));
assert(code->_co_monitoring);
if (code->_co_monitoring->per_instruction_tools) {
code->_co_monitoring->per_instruction_tools[offset] |= tools;
}
else {
/* Single tool */
assert(_Py_popcount32(tools) == 1);
}
MODIFY_BYTECODE(code, instrument_per_instruction, code->_co_monitoring, offset);
}
static void
remove_per_instruction_tools(PyCodeObject * code, int offset, int tools)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(code->_co_monitoring);
bool should_de_instrument;
if (code->_co_monitoring->per_instruction_tools) {
uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset];
*toolsptr &= ~tools;
should_de_instrument = (*toolsptr == 0);
}
else {
/* Single tool */
uint8_t single_tool = code->_co_monitoring->active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION];
assert(_Py_popcount32(single_tool) <= 1);
should_de_instrument = ((single_tool & tools) == single_tool);
}
if (should_de_instrument) {
MODIFY_BYTECODE(code, de_instrument_per_instruction, monitoring, offset);
}
}
/* Return 1 if DISABLE returned, -1 if error, 0 otherwise */
static int
call_one_instrument(
PyInterpreterState *interp, PyThreadState *tstate, PyObject **args,
size_t nargsf, int8_t tool, int event)
{
assert(0 <= tool && tool < 8);
assert(tstate->tracing == 0);
PyObject *instrument = interp->monitoring_callables[tool][event];
if (instrument == NULL) {
return 0;
}
int old_what = tstate->what_event;
tstate->what_event = event;
tstate->tracing++;
PyObject *res = _PyObject_VectorcallTstate(tstate, instrument, args, nargsf, NULL);
tstate->tracing--;
tstate->what_event = old_what;
if (res == NULL) {
return -1;
}
Py_DECREF(res);
return (res == &_PyInstrumentation_DISABLE);
}
static const int8_t MOST_SIGNIFICANT_BITS[16] = {
-1, 0, 1, 1,
2, 2, 2, 2,
3, 3, 3, 3,
3, 3, 3, 3,
};
/* We could use _Py_bit_length here, but that is designed for larger (32/64)
* bit ints, and can perform relatively poorly on platforms without the
* necessary intrinsics. */
static inline int most_significant_bit(uint8_t bits) {
assert(bits != 0);
if (bits > 15) {
return MOST_SIGNIFICANT_BITS[bits>>4]+4;
}
return MOST_SIGNIFICANT_BITS[bits];
}
static uint32_t
global_version(PyInterpreterState *interp)
{
uint32_t version = (uint32_t)_Py_atomic_load_uintptr_relaxed(
&interp->ceval.instrumentation_version);
#ifdef Py_DEBUG
PyThreadState *tstate = _PyThreadState_GET();
uint32_t thread_version =
(uint32_t)(_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
~_PY_EVAL_EVENTS_MASK);
assert(thread_version == version);
#endif
return version;
}
/* Atomically set the given version in the given location, without touching
anything in _PY_EVAL_EVENTS_MASK. */
static void
set_version_raw(uintptr_t *ptr, uint32_t version)
{
uintptr_t old = _Py_atomic_load_uintptr_relaxed(ptr);
uintptr_t new;
do {
new = (old & _PY_EVAL_EVENTS_MASK) | version;
} while (!_Py_atomic_compare_exchange_uintptr(ptr, &old, new));
}
static void
set_global_version(PyThreadState *tstate, uint32_t version)
{
assert((version & _PY_EVAL_EVENTS_MASK) == 0);
PyInterpreterState *interp = tstate->interp;
set_version_raw(&interp->ceval.instrumentation_version, version);
#ifdef Py_GIL_DISABLED
// Set the version on all threads in free-threaded builds.
_Py_FOR_EACH_TSTATE_BEGIN(interp, tstate) {
set_version_raw(&tstate->eval_breaker, version);
};
_Py_FOR_EACH_TSTATE_END(interp);
#else
// Normal builds take the current version from instrumentation_version when
// attaching a thread, so we only have to set the current thread's version.
set_version_raw(&tstate->eval_breaker, version);
#endif
}
static bool
is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
return global_version(interp) == code->_co_instrumentation_version;
}
#ifndef NDEBUG
static bool
instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_Py_LocalMonitors expected = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
return monitors_equals(code->_co_monitoring->active_monitors, expected);
}
static int
debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code)
{
int res;
LOCK_CODE(code);
res = is_version_up_to_date(code, interp) &&
instrumentation_cross_checks(interp, code);
UNLOCK_CODE();
return res;
}
#endif
static inline uint8_t
get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, int event)
{
uint8_t tools;
assert(event != PY_MONITORING_EVENT_LINE);
assert(event != PY_MONITORING_EVENT_INSTRUCTION);
if (event >= _PY_MONITORING_UNGROUPED_EVENTS) {
assert(event == PY_MONITORING_EVENT_C_RAISE ||
event == PY_MONITORING_EVENT_C_RETURN);
event = PY_MONITORING_EVENT_CALL;
}
if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
CHECK(debug_check_sanity(interp, code));
if (code->_co_monitoring->tools) {
tools = code->_co_monitoring->tools[i];
}
else {
tools = code->_co_monitoring->active_monitors.tools[event];
}
}
else {
tools = interp->monitors.tools[event];
}
return tools;
}
static const char *const event_names [] = {
[PY_MONITORING_EVENT_PY_START] = "PY_START",
[PY_MONITORING_EVENT_PY_RESUME] = "PY_RESUME",
[PY_MONITORING_EVENT_PY_RETURN] = "PY_RETURN",
[PY_MONITORING_EVENT_PY_YIELD] = "PY_YIELD",
[PY_MONITORING_EVENT_CALL] = "CALL",
[PY_MONITORING_EVENT_LINE] = "LINE",
[PY_MONITORING_EVENT_INSTRUCTION] = "INSTRUCTION",
[PY_MONITORING_EVENT_JUMP] = "JUMP",
[PY_MONITORING_EVENT_BRANCH] = "BRANCH",
[PY_MONITORING_EVENT_BRANCH_LEFT] = "BRANCH_LEFT",
[PY_MONITORING_EVENT_BRANCH_RIGHT] = "BRANCH_RIGHT",
[PY_MONITORING_EVENT_C_RETURN] = "C_RETURN",
[PY_MONITORING_EVENT_PY_THROW] = "PY_THROW",
[PY_MONITORING_EVENT_RAISE] = "RAISE",
[PY_MONITORING_EVENT_RERAISE] = "RERAISE",
[PY_MONITORING_EVENT_EXCEPTION_HANDLED] = "EXCEPTION_HANDLED",
[PY_MONITORING_EVENT_C_RAISE] = "C_RAISE",
[PY_MONITORING_EVENT_PY_UNWIND] = "PY_UNWIND",
[PY_MONITORING_EVENT_STOP_ITERATION] = "STOP_ITERATION",
};
static int
call_instrumentation_vector(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])
{
if (tstate->tracing) {
return 0;
}
assert(!_PyErr_Occurred(tstate));
assert(args[0] == NULL);
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(args[1] == NULL);
args[1] = (PyObject *)code;
int offset = (int)(instr - _PyFrame_GetBytecode(frame));
/* Offset visible to user should be the offset in bytes, as that is the
* convention for APIs involving code offsets. */
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
if (event == PY_MONITORING_EVENT_BRANCH_LEFT) {
assert(EVENT_FOR_OPCODE[_Py_GetBaseCodeUnit(code, offset-2).op.code] == PY_MONITORING_EVENT_BRANCH_RIGHT);
bytes_offset -= 4;
}
PyObject *offset_obj = PyLong_FromLong(bytes_offset);
if (offset_obj == NULL) {
return -1;
}
assert(args[2] == NULL);
args[2] = offset_obj;
PyInterpreterState *interp = tstate->interp;
uint8_t tools = get_tools_for_instruction(code, interp, offset, event);
size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject **callargs = &args[1];
int err = 0;
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
assert(tools & (1 << tool));
tools ^= (1 << tool);
int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
if (res == 0) {
/* Nothing to do */
}
else if (res < 0) {
/* error */
err = -1;
break;
}
else {
/* DISABLE */
if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
PyErr_Format(PyExc_ValueError,
"Cannot disable %s events. Callback removed.",
event_names[event]);
/* Clear tool to prevent infinite loop */
Py_CLEAR(interp->monitoring_callables[tool][event]);
err = -1;
break;
}
else {
LOCK_CODE(code);
remove_tools(code, offset, event, 1 << tool);
UNLOCK_CODE();
}
}
}
Py_DECREF(offset_obj);
return err;
}
int
_Py_call_instrumentation(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr)
{
PyObject *args[3] = { NULL, NULL, NULL };
return call_instrumentation_vector(tstate, event, frame, instr, 2, args);
}
int
_Py_call_instrumentation_arg(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg)
{
PyObject *args[4] = { NULL, NULL, NULL, arg };
return call_instrumentation_vector(tstate, event, frame, instr, 3, args);
}
int
_Py_call_instrumentation_2args(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1)
{
PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 };
return call_instrumentation_vector(tstate, event, frame, instr, 4, args);
}
_Py_CODEUNIT *
_Py_call_instrumentation_jump(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target)
{
assert(event == PY_MONITORING_EVENT_JUMP ||
event == PY_MONITORING_EVENT_BRANCH_RIGHT ||
event == PY_MONITORING_EVENT_BRANCH_LEFT);
assert(frame->instr_ptr == instr);
int to = (int)(target - _PyFrame_GetBytecode(frame));
PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT));
if (to_obj == NULL) {
return NULL;
}
PyObject *args[4] = { NULL, NULL, NULL, to_obj };
int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args);
Py_DECREF(to_obj);
if (err) {
return NULL;
}
if (frame->instr_ptr != instr) {
/* The callback has caused a jump (by setting the line number) */
return frame->instr_ptr;
}
return target;
}
static void
call_instrumentation_vector_protected(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, Py_ssize_t nargs, PyObject *args[])
{
assert(_PyErr_Occurred(tstate));
PyObject *exc = _PyErr_GetRaisedException(tstate);
int err = call_instrumentation_vector(tstate, event, frame, instr, nargs, args);
if (err) {
Py_XDECREF(exc);
}
else {
_PyErr_SetRaisedException(tstate, exc);
}
assert(_PyErr_Occurred(tstate));
}
void
_Py_call_instrumentation_exc2(
PyThreadState *tstate, int event,
_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1)
{
assert(_PyErr_Occurred(tstate));
PyObject *args[5] = { NULL, NULL, NULL, arg0, arg1 };
call_instrumentation_vector_protected(tstate, event, frame, instr, 4, args);
}
int
_Py_Instrumentation_GetLine(PyCodeObject *code, int index)
{
_PyCoMonitoringData *monitoring = code->_co_monitoring;
assert(monitoring != NULL);
assert(monitoring->lines != NULL);
assert(index >= code->_co_firsttraceable);
assert(index < Py_SIZE(code));
_PyCoLineInstrumentationData *line_data = &monitoring->lines[index];
int8_t line_delta = line_data->line_delta;
int line = compute_line(code, index, line_delta);
return line;
}
int
_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev)
{
PyCodeObject *code = _PyFrame_GetCode(frame);
assert(tstate->tracing == 0);
assert(debug_check_sanity(tstate->interp, code));
_Py_CODEUNIT *bytecode = _PyFrame_GetBytecode(frame);
int i = (int)(instr - bytecode);
_PyCoMonitoringData *monitoring = code->_co_monitoring;
_PyCoLineInstrumentationData *line_data = &monitoring->lines[i];
PyInterpreterState *interp = tstate->interp;
int8_t line_delta = line_data->line_delta;
int line = 0;
if (line_delta == COMPUTED_LINE_LINENO_CHANGE) {
// We know the line number must have changed, don't need to calculate
// the line number for now because we might not need it.
line = -1;
} else {
line = compute_line(code, i, line_delta);
assert(line >= 0);
assert(prev != NULL);
int prev_index = (int)(prev - bytecode);
int prev_line = _Py_Instrumentation_GetLine(code, prev_index);
if (prev_line == line) {
int prev_opcode = bytecode[prev_index].op.code;
/* RESUME and INSTRUMENTED_RESUME are needed for the operation of
* instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
*/
if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) {
goto done;
}
}
}
uint8_t tools = code->_co_monitoring->line_tools != NULL ?
code->_co_monitoring->line_tools[i] :
(interp->monitors.tools[PY_MONITORING_EVENT_LINE] |
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE]
);
/* Special case sys.settrace to avoid boxing the line number,
* only to immediately unbox it. */
if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) {
if (tstate->c_tracefunc != NULL) {
PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame);
if (frame_obj == NULL) {
return -1;
}
if (frame_obj->f_trace_lines) {
/* Need to set tracing and what_event as if using
* the instrumentation call. */
int old_what = tstate->what_event;
tstate->what_event = PY_MONITORING_EVENT_LINE;
tstate->tracing++;
/* Call c_tracefunc directly, having set the line number. */
Py_INCREF(frame_obj);
if (line == -1 && line_delta > COMPUTED_LINE) {
/* Only assign f_lineno if it's easy to calculate, otherwise
* do lazy calculation by setting the f_lineno to 0.
*/
line = compute_line(code, i, line_delta);
}
frame_obj->f_lineno = line;
int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None);
frame_obj->f_lineno = 0;
tstate->tracing--;
tstate->what_event = old_what;
Py_DECREF(frame_obj);
if (err) {
return -1;
}
}
}
tools &= (255 - (1 << PY_MONITORING_SYS_TRACE_ID));
}
if (tools == 0) {
goto done;
}
if (line == -1) {
/* Need to calculate the line number now for monitoring events */
line = compute_line(code, i, line_delta);
}
PyObject *line_obj = PyLong_FromLong(line);
if (line_obj == NULL) {
return -1;
}
PyObject *args[3] = { NULL, (PyObject *)code, line_obj };
do {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < PY_MONITORING_SYS_PROFILE_ID);
assert(tools & (1 << tool));
tools &= ~(1 << tool);
int res = call_one_instrument(interp, tstate, &args[1],
2 | PY_VECTORCALL_ARGUMENTS_OFFSET,
tool, PY_MONITORING_EVENT_LINE);
if (res == 0) {
/* Nothing to do */
}
else if (res < 0) {
/* error */
Py_DECREF(line_obj);
return -1;
}
else {
/* DISABLE */
LOCK_CODE(code);
remove_line_tools(code, i, 1 << tool);
UNLOCK_CODE();
}
} while (tools);
Py_DECREF(line_obj);
uint8_t original_opcode;
done:
original_opcode = line_data->original_opcode;
assert(original_opcode != 0);
assert(original_opcode != INSTRUMENTED_LINE);
assert(_PyOpcode_Deopt[original_opcode] == original_opcode);
return original_opcode;
}
int
_Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr)
{
PyCodeObject *code = _PyFrame_GetCode(frame);
int offset = (int)(instr - _PyFrame_GetBytecode(frame));
_PyCoMonitoringData *instrumentation_data = code->_co_monitoring;
assert(instrumentation_data->per_instruction_opcodes);
int next_opcode = instrumentation_data->per_instruction_opcodes[offset];
if (tstate->tracing) {
return next_opcode;
}
assert(debug_check_sanity(tstate->interp, code));
PyInterpreterState *interp = tstate->interp;
uint8_t tools = instrumentation_data->per_instruction_tools != NULL ?
instrumentation_data->per_instruction_tools[offset] :
(interp->monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] |
code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]
);
int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT);
PyObject *offset_obj = PyLong_FromLong(bytes_offset);
if (offset_obj == NULL) {
return -1;
}
PyObject *args[3] = { NULL, (PyObject *)code, offset_obj };
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
assert(tools & (1 << tool));
tools &= ~(1 << tool);
int res = call_one_instrument(interp, tstate, &args[1],
2 | PY_VECTORCALL_ARGUMENTS_OFFSET,
tool, PY_MONITORING_EVENT_INSTRUCTION);
if (res == 0) {
/* Nothing to do */
}
else if (res < 0) {
/* error */
Py_DECREF(offset_obj);
return -1;
}
else {
/* DISABLE */
LOCK_CODE(code);
remove_per_instruction_tools(code, offset, 1 << tool);
UNLOCK_CODE();
}
}
Py_DECREF(offset_obj);
assert(next_opcode != 0);
return next_opcode;
}
static void
initialize_tools(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
uint8_t* tools = code->_co_monitoring->tools;
assert(tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
_Py_CODEUNIT *instr = &_PyCode_CODE(code)[i];
int opcode = instr->op.code;
assert(opcode != ENTER_EXECUTOR);
if (opcode == INSTRUMENTED_LINE) {
opcode = code->_co_monitoring->lines[i].original_opcode;
}
if (opcode == INSTRUMENTED_INSTRUCTION) {
opcode = code->_co_monitoring->per_instruction_opcodes[i];
}
bool instrumented = is_instrumented(opcode);
if (instrumented) {
opcode = DE_INSTRUMENT[opcode];
assert(opcode != 0);
}
opcode = _PyOpcode_Deopt[opcode];
if (opcode_has_event(opcode)) {
if (instrumented) {
int8_t event;
if (opcode == RESUME) {
event = instr->op.arg != 0;
}
else {
event = EVENT_FOR_OPCODE[opcode];
assert(event > 0);
}
assert(event >= 0);
assert(PY_MONITORING_IS_INSTRUMENTED_EVENT(event));
tools[i] = code->_co_monitoring->active_monitors.tools[event];
CHECK(tools[i] != 0);
}
else {
tools[i] = 0;
}
}
#ifdef Py_DEBUG
/* Initialize tools for invalid locations to all ones to try to catch errors */
else {
tools[i] = 0xff;
}
for (int j = 1; j <= _PyOpcode_Caches[opcode]; j++) {
tools[i+j] = 0xff;
}
#endif
i += _PyOpcode_Caches[opcode];
}
}
#define NO_LINE -128
static void
initialize_lines(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
assert(line_data != NULL);
int code_len = (int)Py_SIZE(code);
PyCodeAddressRange range;
_PyCode_InitAddressRange(code, &range);
for (int i = 0; i < code->_co_firsttraceable && i < code_len; i++) {
line_data[i].original_opcode = 0;
line_data[i].line_delta = -127;
}
int current_line = -1;
for (int i = code->_co_firsttraceable; i < code_len; ) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
int line = _PyCode_CheckLineNumber(i*(int)sizeof(_Py_CODEUNIT), &range);
line_data[i].line_delta = compute_line_delta(code, i, line);
int length = _PyInstruction_GetLength(code, i);
switch (opcode) {
case END_ASYNC_FOR:
case END_FOR:
case END_SEND:
case RESUME:
/* END_FOR cannot start a line, as it is skipped by FOR_ITER
* END_SEND cannot start a line, as it is skipped by SEND
* RESUME must not be instrumented with INSTRUMENT_LINE */
line_data[i].original_opcode = 0;
break;
default:
/* Set original_opcode to the opcode iff the instruction
* starts a line, and thus should be instrumented.
* This saves having to perform this check every time the
* we turn instrumentation on or off, and serves as a sanity
* check when debugging.
*/
if (line != current_line && line >= 0) {
line_data[i].original_opcode = opcode;
if (line_data[i].line_delta == COMPUTED_LINE) {
/* Label this line as a line with a line number change
* which could help the monitoring callback to quickly
* identify the line number change.
*/
line_data[i].line_delta = COMPUTED_LINE_LINENO_CHANGE;
}
}
else {
line_data[i].original_opcode = 0;
}
current_line = line;
}
for (int j = 1; j < length; j++) {
line_data[i+j].original_opcode = 0;
line_data[i+j].line_delta = NO_LINE;
}
i += length;
}
for (int i = code->_co_firsttraceable; i < code_len; ) {
_Py_CODEUNIT inst =_Py_GetBaseCodeUnit(code, i);
int opcode = inst.op.code;
int oparg = 0;
while (opcode == EXTENDED_ARG) {
oparg = (oparg << 8) | inst.op.arg;
i++;
inst =_Py_GetBaseCodeUnit(code, i);
opcode = inst.op.code;
}
oparg = (oparg << 8) | inst.op.arg;
i += _PyInstruction_GetLength(code, i);
int target = -1;
switch (opcode) {
case POP_JUMP_IF_FALSE:
case POP_JUMP_IF_TRUE:
case POP_JUMP_IF_NONE:
case POP_JUMP_IF_NOT_NONE:
case JUMP_FORWARD:
{
target = i + oparg;
break;
}
case FOR_ITER:
case SEND:
{
/* Skip over END_FOR/END_SEND */
target = i + oparg + 1;
break;
}
case JUMP_BACKWARD:
case JUMP_BACKWARD_NO_INTERRUPT:
{
target = i - oparg;
break;
}
default:
continue;
}
assert(target >= 0);
if (line_data[target].line_delta != NO_LINE) {
line_data[target].original_opcode = _Py_GetBaseCodeUnit(code, target).op.code;
if (line_data[target].line_delta == COMPUTED_LINE_LINENO_CHANGE) {
// If the line is a jump target, we are not sure if the line
// number changes, so we set it to COMPUTED_LINE.
line_data[target].line_delta = COMPUTED_LINE;
}
}
}
/* Scan exception table */
unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
unsigned char *scan = start;
while (scan < end) {
int start_offset, size, handler;
scan = parse_varint(scan, &start_offset);
assert(start_offset >= 0 && start_offset < code_len);
scan = parse_varint(scan, &size);
assert(size >= 0 && start_offset+size <= code_len);
scan = parse_varint(scan, &handler);
assert(handler >= 0 && handler < code_len);
int depth_and_lasti;
scan = parse_varint(scan, &depth_and_lasti);
int original_opcode = _Py_GetBaseCodeUnit(code, handler).op.code;
/* Skip if not the start of a line.
* END_ASYNC_FOR is a bit special as it marks the end of
* an `async for` loop, which should not generate its own
* line event. */
if (line_data[handler].line_delta != NO_LINE &&
original_opcode != END_ASYNC_FOR) {
line_data[handler].original_opcode = original_opcode;
}
}
}
static void
initialize_line_tools(PyCodeObject *code, _Py_LocalMonitors *all_events)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
uint8_t *line_tools = code->_co_monitoring->line_tools;
assert(line_tools != NULL);
int code_len = (int)Py_SIZE(code);
for (int i = 0; i < code_len; i++) {
line_tools[i] = all_events->tools[PY_MONITORING_EVENT_LINE];
}
}
static int
allocate_instrumentation_data(PyCodeObject *code)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
if (code->_co_monitoring == NULL) {
code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData));
if (code->_co_monitoring == NULL) {
PyErr_NoMemory();
return -1;
}
code->_co_monitoring->local_monitors = (_Py_LocalMonitors){ 0 };
code->_co_monitoring->active_monitors = (_Py_LocalMonitors){ 0 };
code->_co_monitoring->tools = NULL;
code->_co_monitoring->lines = NULL;
code->_co_monitoring->line_tools = NULL;
code->_co_monitoring->per_instruction_opcodes = NULL;
code->_co_monitoring->per_instruction_tools = NULL;
}
return 0;
}
static int
update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
int code_len = (int)Py_SIZE(code);
if (allocate_instrumentation_data(code)) {
return -1;
}
// If the local monitors are out of date, clear them up
_Py_LocalMonitors *local_monitors = &code->_co_monitoring->local_monitors;
for (int i = 0; i < PY_MONITORING_TOOL_IDS; i++) {
if (code->_co_monitoring->tool_versions[i] != interp->monitoring_tool_versions[i]) {
for (int j = 0; j < _PY_MONITORING_LOCAL_EVENTS; j++) {
local_monitors->tools[j] &= ~(1 << i);
}
}
}
_Py_LocalMonitors all_events = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
bool multitools = multiple_tools(&all_events);
if (code->_co_monitoring->tools == NULL && multitools) {
code->_co_monitoring->tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->tools == NULL) {
PyErr_NoMemory();
return -1;
}
initialize_tools(code);
}
if (all_events.tools[PY_MONITORING_EVENT_LINE]) {
if (code->_co_monitoring->lines == NULL) {
code->_co_monitoring->lines = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
if (code->_co_monitoring->lines == NULL) {
PyErr_NoMemory();
return -1;
}
initialize_lines(code);
}
if (multitools && code->_co_monitoring->line_tools == NULL) {
code->_co_monitoring->line_tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->line_tools == NULL) {
PyErr_NoMemory();
return -1;
}
initialize_line_tools(code, &all_events);
}
}
if (all_events.tools[PY_MONITORING_EVENT_INSTRUCTION]) {
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
code->_co_monitoring->per_instruction_opcodes = PyMem_Malloc(code_len * sizeof(_PyCoLineInstrumentationData));
if (code->_co_monitoring->per_instruction_opcodes == NULL) {
PyErr_NoMemory();
return -1;
}
// Initialize all of the instructions so if local events change while another thread is executing
// we know what the original opcode was.
for (int i = 0; i < code_len; i++) {
int opcode = _PyCode_CODE(code)[i].op.code;
code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode];
}
}
if (multitools && code->_co_monitoring->per_instruction_tools == NULL) {
code->_co_monitoring->per_instruction_tools = PyMem_Malloc(code_len);
if (code->_co_monitoring->per_instruction_tools == NULL) {
PyErr_NoMemory();
return -1;
}
for (int i = 0; i < code_len; i++) {
code->_co_monitoring->per_instruction_tools[i] = 0;
}
}
}
return 0;
}
static int
force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
#ifdef _Py_TIER2
if (code->co_executors != NULL) {
_PyCode_Clear_Executors(code);
}
_Py_Executors_InvalidateDependency(interp, code, 1);
#endif
int code_len = (int)Py_SIZE(code);
/* Exit early to avoid creating instrumentation
* data for potential statically allocated code
* objects.
* See https://github.com/python/cpython/issues/108390 */
if (code->co_flags & CO_NO_MONITORING_EVENTS) {
return 0;
}
if (update_instrumentation_data(code, interp)) {
return -1;
}
_Py_LocalMonitors active_events = local_union(
interp->monitors,
code->_co_monitoring->local_monitors);
_Py_LocalMonitors new_events;
_Py_LocalMonitors removed_events;
bool restarted = interp->last_restart_version > code->_co_instrumentation_version;
if (restarted) {
removed_events = code->_co_monitoring->active_monitors;
new_events = active_events;
}
else {
removed_events = monitors_sub(code->_co_monitoring->active_monitors, active_events);
new_events = monitors_sub(active_events, code->_co_monitoring->active_monitors);
assert(monitors_are_empty(monitors_and(new_events, removed_events)));
}
code->_co_monitoring->active_monitors = active_events;
if (monitors_are_empty(new_events) && monitors_are_empty(removed_events)) {
goto done;
}
/* Insert instrumentation */
for (int i = code->_co_firsttraceable; i < code_len; i+= _PyInstruction_GetLength(code, i)) {
assert(_PyCode_CODE(code)[i].op.code != ENTER_EXECUTOR);
_Py_CODEUNIT instr = _Py_GetBaseCodeUnit(code, i);
CHECK(instr.op.code != 0);
int base_opcode = instr.op.code;
if (opcode_has_event(base_opcode)) {
int8_t event;
if (base_opcode == RESUME) {
event = instr.op.arg > 0;
}
else {
event = EVENT_FOR_OPCODE[base_opcode];
assert(event > 0);
}
uint8_t removed_tools = removed_events.tools[event];
if (removed_tools) {
remove_tools(code, i, event, removed_tools);
}
uint8_t new_tools = new_events.tools[event];
if (new_tools) {
add_tools(code, i, event, new_tools);
}
}
}
// GH-103845: We need to remove both the line and instruction instrumentation before
// adding new ones, otherwise we may remove the newly added instrumentation.
uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (removed_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
if (line_data[i].original_opcode) {
remove_line_tools(code, i, removed_line_tools);
}
i += _PyInstruction_GetLength(code, i);
}
}
if (removed_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
if (opcode == RESUME || opcode == END_FOR) {
i += _PyInstruction_GetLength(code, i);
continue;
}
remove_per_instruction_tools(code, i, removed_per_instruction_tools);
i += _PyInstruction_GetLength(code, i);
}
}
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE];
uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION];
if (new_line_tools) {
_PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines;
for (int i = code->_co_firsttraceable; i < code_len;) {
if (line_data[i].original_opcode) {
add_line_tools(code, i, new_line_tools);
}
i += _PyInstruction_GetLength(code, i);
}
}
if (new_per_instruction_tools) {
for (int i = code->_co_firsttraceable; i < code_len;) {
int opcode = _Py_GetBaseCodeUnit(code, i).op.code;
if (opcode == RESUME || opcode == END_FOR) {
i += _PyInstruction_GetLength(code, i);
continue;
}
add_per_instruction_tools(code, i, new_per_instruction_tools);
i += _PyInstruction_GetLength(code, i);
}
}
done:
FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version,
global_version(interp));
#ifdef INSTRUMENT_DEBUG
sanity_check_instrumentation(code);
#endif
return 0;
}
static int
instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
{
ASSERT_WORLD_STOPPED_OR_LOCKED(code);
if (is_version_up_to_date(code, interp)) {
assert(
interp->ceval.instrumentation_version == 0 ||
instrumentation_cross_checks(interp, code)
);
return 0;
}
return force_instrument_lock_held(code, interp);
}
int
_Py_Instrument(PyCodeObject *code, PyInterpreterState *interp)
{
int res;
LOCK_CODE(code);
res = instrument_lock_held(code, interp);
UNLOCK_CODE();
return res;
}
#define C_RETURN_EVENTS \
((1 << PY_MONITORING_EVENT_C_RETURN) | \
(1 << PY_MONITORING_EVENT_C_RAISE))
#define C_CALL_EVENTS \
(C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL))
static int
instrument_all_executing_code_objects(PyInterpreterState *interp) {
ASSERT_WORLD_STOPPED();
_PyRuntimeState *runtime = &_PyRuntime;
HEAD_LOCK(runtime);
PyThreadState* ts = PyInterpreterState_ThreadHead(interp);
HEAD_UNLOCK(runtime);
while (ts) {
_PyInterpreterFrame *frame = ts->current_frame;
while (frame) {
if (frame->owner != FRAME_OWNED_BY_CSTACK) {
if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) {
return -1;
}
}
frame = frame->previous;
}
HEAD_LOCK(runtime);
ts = PyThreadState_Next(ts);
HEAD_UNLOCK(runtime);
}
return 0;
}
static void
set_events(_Py_GlobalMonitors *m, int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
uint8_t *tools = &m->tools[e];
int active = (events >> e) & 1;
*tools &= ~(1 << tool_id);
*tools |= (active << tool_id);
}
}
static void
set_local_events(_Py_LocalMonitors *m, int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
uint8_t *tools = &m->tools[e];
int val = (events >> e) & 1;
*tools &= ~(1 << tool_id);
*tools |= (val << tool_id);
}
}
static int
check_tool(PyInterpreterState *interp, int tool_id)
{
if (tool_id < PY_MONITORING_SYS_PROFILE_ID &&
interp->monitoring_tool_names[tool_id] == NULL)
{
PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id);
return -1;
}
return 0;
}
/* We share the eval-breaker with flags, so the monitoring
* version goes in the top 24 bits */
#define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS)
int
_PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
PyThreadState *tstate = _PyThreadState_GET();
PyInterpreterState *interp = tstate->interp;
assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS));
if (check_tool(interp, tool_id)) {
return -1;
}
int res;
_PyEval_StopTheWorld(interp);
uint32_t existing_events = get_events(&interp->monitors, tool_id);
if (existing_events == events) {
res = 0;
goto done;
}
set_events(&interp->monitors, tool_id, events);
uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
if (new_version == 0) {
PyErr_Format(PyExc_OverflowError, "events set too many times");
res = -1;
goto done;
}
set_global_version(tstate, new_version);
#ifdef _Py_TIER2
_Py_Executors_InvalidateAll(interp, 1);
#endif
res = instrument_all_executing_code_objects(interp);
done:
_PyEval_StartTheWorld(interp);
return res;
}
int
_PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
PyInterpreterState *interp = _PyInterpreterState_GET();
assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS));
if (code->_co_firsttraceable >= Py_SIZE(code)) {
PyErr_Format(PyExc_SystemError, "cannot instrument shim code object '%U'", code->co_name);
return -1;
}
if (check_tool(interp, tool_id)) {
return -1;
}
int res;
_PyEval_StopTheWorld(interp);
if (allocate_instrumentation_data(code)) {
res = -1;
goto done;
}
code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id];
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
uint32_t existing_events = get_local_events(local, tool_id);
if (existing_events == events) {
res = 0;
goto done;
}
set_local_events(local, tool_id, events);
res = force_instrument_lock_held(code, interp);
done:
_PyEval_StartTheWorld(interp);
return res;
}
int
_PyMonitoring_GetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet *events)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
PyInterpreterState *interp = _PyInterpreterState_GET();
if (check_tool(interp, tool_id)) {
return -1;
}
if (code->_co_monitoring == NULL) {
*events = 0;
return 0;
}
_Py_LocalMonitors *local = &code->_co_monitoring->local_monitors;
*events = get_local_events(local, tool_id);
return 0;
}
int _PyMonitoring_ClearToolId(int tool_id)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
PyInterpreterState *interp = _PyInterpreterState_GET();
for (int i = 0; i < _PY_MONITORING_EVENTS; i++) {
PyObject *func = _PyMonitoring_RegisterCallback(tool_id, i, NULL);
if (func != NULL) {
Py_DECREF(func);
}
}
if (_PyMonitoring_SetEvents(tool_id, 0) < 0) {
return -1;
}
_PyEval_StopTheWorld(interp);
uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT;
if (version == 0) {
PyErr_Format(PyExc_OverflowError, "events set too many times");
_PyEval_StartTheWorld(interp);
return -1;
}
// monitoring_tool_versions[tool_id] is set to latest global version here to
// 1. invalidate local events on all existing code objects
// 2. be ready for the next call to set local events
interp->monitoring_tool_versions[tool_id] = version;
// Set the new global version so all the code objects can refresh the
// instrumentation.
set_global_version(_PyThreadState_GET(), version);
int res = instrument_all_executing_code_objects(interp);
_PyEval_StartTheWorld(interp);
return res;
}
/*[clinic input]
module monitoring
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=37257f5987a360cf]*/
/*[clinic end generated code]*/
#include "clinic/instrumentation.c.h"
static int
check_valid_tool(int tool_id)
{
if (tool_id < 0 || tool_id >= PY_MONITORING_SYS_PROFILE_ID) {
PyErr_Format(PyExc_ValueError, "invalid tool %d (must be between 0 and 5)", tool_id);
return -1;
}
return 0;
}
/*[clinic input]
monitoring.use_tool_id
tool_id: int
name: object
/
[clinic start generated code]*/
static PyObject *
monitoring_use_tool_id_impl(PyObject *module, int tool_id, PyObject *name)
/*[clinic end generated code: output=30d76dc92b7cd653 input=ebc453761c621be1]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (!PyUnicode_Check(name)) {
PyErr_SetString(PyExc_ValueError, "tool name must be a str");
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
PyErr_Format(PyExc_ValueError, "tool %d is already in use", tool_id);
return NULL;
}
interp->monitoring_tool_names[tool_id] = Py_NewRef(name);
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.clear_tool_id
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_clear_tool_id_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=04defc23470b1be7 input=af643d6648a66163]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
return NULL;
}
}
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.free_tool_id
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_free_tool_id_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=86c2d2a1219a8591 input=a23fb6be3a8618e9]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
if (interp->monitoring_tool_names[tool_id] != NULL) {
if (_PyMonitoring_ClearToolId(tool_id) < 0) {
return NULL;
}
}
Py_CLEAR(interp->monitoring_tool_names[tool_id]);
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.get_tool
tool_id: int
/
[clinic start generated code]*/
static PyObject *
monitoring_get_tool_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=1c05a98b404a9a16 input=eeee9bebd0bcae9d]*/
/*[clinic end generated code]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *name = interp->monitoring_tool_names[tool_id];
if (name == NULL) {
Py_RETURN_NONE;
}
return Py_NewRef(name);
}
/*[clinic input]
monitoring.register_callback
tool_id: int
event: int
func: object
/
[clinic start generated code]*/
static PyObject *
monitoring_register_callback_impl(PyObject *module, int tool_id, int event,
PyObject *func)
/*[clinic end generated code: output=e64daa363004030c input=df6d70ea4cf81007]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (_Py_popcount32(event) != 1) {
PyErr_SetString(PyExc_ValueError, "The callback can only be set for one event at a time");
return NULL;
}
int event_id = _Py_bit_length(event)-1;
if (event_id < 0 || event_id >= _PY_MONITORING_EVENTS) {
PyErr_Format(PyExc_ValueError, "invalid event %d", event);
return NULL;
}
if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) {
return NULL;
}
if (func == Py_None) {
func = NULL;
}
func = _PyMonitoring_RegisterCallback(tool_id, event_id, func);
if (func == NULL) {
Py_RETURN_NONE;
}
return func;
}
/*[clinic input]
monitoring.get_events -> int
tool_id: int
/
[clinic start generated code]*/
static int
monitoring_get_events_impl(PyObject *module, int tool_id)
/*[clinic end generated code: output=4450cc13f826c8c0 input=a64b238f76c4b2f7]*/
{
if (check_valid_tool(tool_id)) {
return -1;
}
_Py_GlobalMonitors *m = &_PyInterpreterState_GET()->monitors;
_PyMonitoringEventSet event_set = get_events(m, tool_id);
return event_set;
}
/*[clinic input]
monitoring.set_events
tool_id: int
event_set: int
/
[clinic start generated code]*/
static PyObject *
monitoring_set_events_impl(PyObject *module, int tool_id, int event_set)
/*[clinic end generated code: output=1916c1e49cfb5bdb input=a77ba729a242142b]*/
{
if (check_valid_tool(tool_id)) {
return NULL;
}
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) {
PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set);
return NULL;
}
if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) {
PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently");
return NULL;
}
event_set &= ~C_RETURN_EVENTS;
if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) {
event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH);
event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT);
}
if (_PyMonitoring_SetEvents(tool_id, event_set)) {
return NULL;
}
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.get_local_events -> int
tool_id: int
code: object
/
[clinic start generated code]*/
static int
monitoring_get_local_events_impl(PyObject *module, int tool_id,
PyObject *code)
/*[clinic end generated code: output=d3e92c1c9c1de8f9 input=bb0f927530386a94]*/
{
if (!PyCode_Check(code)) {
PyErr_Format(
PyExc_TypeError,
"code must be a code object"
);
return -1;
}
if (check_valid_tool(tool_id)) {
return -1;
}
_PyMonitoringEventSet event_set = 0;
_PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring;
if (data != NULL) {
for (int e = 0; e < _PY_MONITORING_LOCAL_EVENTS; e++) {
if ((data->local_monitors.tools[e] >> tool_id) & 1) {
event_set |= (1 << e);
}
}
}
return event_set;
}
/*[clinic input]
monitoring.set_local_events
tool_id: int
code: object
event_set: int
/
[clinic start generated code]*/
static PyObject *
monitoring_set_local_events_impl(PyObject *module, int tool_id,
PyObject *code, int event_set)
/*[clinic end generated code: output=68cc755a65dfea99 input=5655ecd78d937a29]*/
{
if (!PyCode_Check(code)) {
PyErr_Format(
PyExc_TypeError,
"code must be a code object"
);
return NULL;
}
if (check_valid_tool(tool_id)) {
return NULL;
}
if ((event_set & C_RETURN_EVENTS) && (event_set & C_CALL_EVENTS) != C_CALL_EVENTS) {
PyErr_Format(PyExc_ValueError, "cannot set C_RETURN or C_RAISE events independently");
return NULL;
}
event_set &= ~C_RETURN_EVENTS;
if (event_set & (1 << PY_MONITORING_EVENT_BRANCH)) {
event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH);
event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT);
}
if (event_set < 0 || event_set >= (1 << _PY_MONITORING_LOCAL_EVENTS)) {
PyErr_Format(PyExc_ValueError, "invalid local event set 0x%x", event_set);
return NULL;
}
if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) {
return NULL;
}
Py_RETURN_NONE;
}
/*[clinic input]
monitoring.restart_events
[clinic start generated code]*/
static PyObject *
monitoring_restart_events_impl(PyObject *module)
/*[clinic end generated code: output=e025dd5ba33314c4 input=add8a855063c8008]*/
{
/* We want to ensure that:
* last restart version > instrumented version for all code objects
* last restart version < current version
*/
PyThreadState *tstate = _PyThreadState_GET();
PyInterpreterState *interp = tstate->interp;
_PyEval_StopTheWorld(interp);
uint32_t restart_version = global_version(interp) + MONITORING_VERSION_INCREMENT;
uint32_t new_version = restart_version + MONITORING_VERSION_INCREMENT;
if (new_version <= MONITORING_VERSION_INCREMENT) {
_PyEval_StartTheWorld(interp);
PyErr_Format(PyExc_OverflowError, "events set too many times");
return NULL;
}
interp->last_restart_version = restart_version;
set_global_version(tstate, new_version);
int res = instrument_all_executing_code_objects(interp);
_PyEval_StartTheWorld(interp);
if (res) {
return NULL;
}
Py_RETURN_NONE;
}
static int
add_power2_constant(PyObject *obj, const char *name, int i)
{
PyObject *val = PyLong_FromLong(1<<i);
if (val == NULL) {
return -1;
}
int err = PyObject_SetAttrString(obj, name, val);
Py_DECREF(val);
return err;
}
/*[clinic input]
monitoring._all_events
[clinic start generated code]*/
static PyObject *
monitoring__all_events_impl(PyObject *module)
/*[clinic end generated code: output=6b7581e2dbb690f6 input=62ee9672c17b7f0e]*/
{
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *res = PyDict_New();
if (res == NULL) {
return NULL;
}
for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) {
uint8_t tools = interp->monitors.tools[e];
if (tools == 0) {
continue;
}
PyObject *tools_obj = PyLong_FromLong(tools);
assert(tools_obj != NULL);
int err = PyDict_SetItemString(res, event_names[e], tools_obj);
Py_DECREF(tools_obj);
if (err < 0) {
Py_DECREF(res);
return NULL;
}
}
return res;
}
static PyMethodDef methods[] = {
MONITORING_USE_TOOL_ID_METHODDEF
MONITORING_CLEAR_TOOL_ID_METHODDEF
MONITORING_FREE_TOOL_ID_METHODDEF
MONITORING_GET_TOOL_METHODDEF
MONITORING_REGISTER_CALLBACK_METHODDEF
MONITORING_GET_EVENTS_METHODDEF
MONITORING_SET_EVENTS_METHODDEF
MONITORING_GET_LOCAL_EVENTS_METHODDEF
MONITORING_SET_LOCAL_EVENTS_METHODDEF
MONITORING_RESTART_EVENTS_METHODDEF
MONITORING__ALL_EVENTS_METHODDEF
{NULL, NULL} // sentinel
};
static struct PyModuleDef monitoring_module = {
PyModuleDef_HEAD_INIT,
.m_name = "sys.monitoring",
.m_size = -1, /* multiple "initialization" just copies the module dict. */
.m_methods = methods,
};
PyObject *_Py_CreateMonitoringObject(void)
{
PyObject *mod = _PyModule_CreateInitialized(&monitoring_module, PYTHON_API_VERSION);
if (mod == NULL) {
return NULL;
}
if (PyObject_SetAttrString(mod, "DISABLE", &_PyInstrumentation_DISABLE)) {
goto error;
}
if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) {
goto error;
}
PyObject *events = _PyNamespace_New(NULL);
if (events == NULL) {
goto error;
}
int err = PyObject_SetAttrString(mod, "events", events);
Py_DECREF(events);
if (err) {
goto error;
}
for (int i = 0; i < _PY_MONITORING_EVENTS; i++) {
if (add_power2_constant(events, event_names[i], i)) {
goto error;
}
}
err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero());
if (err) goto error;
PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID);
err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID);
err = PyObject_SetAttrString(mod, "COVERAGE_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_PROFILER_ID);
err = PyObject_SetAttrString(mod, "PROFILER_ID", val);
Py_DECREF(val);
if (err) goto error;
val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID);
err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val);
Py_DECREF(val);
if (err) goto error;
return mod;
error:
Py_DECREF(mod);
return NULL;
}
static int
capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject **args, Py_ssize_t nargs, int event)
{
PyThreadState *tstate = _PyThreadState_GET();
PyInterpreterState *interp = tstate->interp;
uint8_t tools = state->active;
assert(args[1] == NULL);
args[1] = codelike;
if (offset < 0) {
PyErr_SetString(PyExc_ValueError, "offset must be non-negative");
return -1;
}
if (event != PY_MONITORING_EVENT_LINE) {
PyObject *offset_obj = PyLong_FromLong(offset);
if (offset_obj == NULL) {
return -1;
}
assert(args[2] == NULL);
args[2] = offset_obj;
}
size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject **callargs = &args[1];
int err = 0;
while (tools) {
int tool = most_significant_bit(tools);
assert(tool >= 0 && tool < 8);
assert(tools & (1 << tool));
tools ^= (1 << tool);
int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
if (res == 0) {
/* Nothing to do */
}
else if (res < 0) {
/* error */
err = -1;
break;
}
else {
/* DISABLE */
if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
PyErr_Format(PyExc_ValueError,
"Cannot disable %s events. Callback removed.",
event_names[event]);
/* Clear tool to prevent infinite loop */
Py_CLEAR(interp->monitoring_callables[tool][event]);
err = -1;
break;
}
else {
state->active &= ~(1 << tool);
}
}
}
return err;
}
int
PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
const uint8_t *event_types, Py_ssize_t length)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
if (global_version(interp) == *version) {
return 0;
}
_Py_GlobalMonitors *m = &interp->monitors;
for (Py_ssize_t i = 0; i < length; i++) {
int event = event_types[i];
state_array[i].active = m->tools[event];
}
*version = global_version(interp);
return 0;
}
int
PyMonitoring_ExitScope(void)
{
return 0;
}
int
_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
assert(state->active);
PyObject *args[3] = { NULL, NULL, NULL };
return capi_call_instrumentation(state, codelike, offset, args, 2,
PY_MONITORING_EVENT_PY_START);
}
int
_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
assert(state->active);
PyObject *args[3] = { NULL, NULL, NULL };
return capi_call_instrumentation(state, codelike, offset, args, 2,
PY_MONITORING_EVENT_PY_RESUME);
}
int
_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject* retval)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, retval };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_PY_RETURN);
}
int
_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject* retval)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, retval };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_PY_YIELD);
}
int
_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject* callable, PyObject *arg0)
{
assert(state->active);
PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 };
return capi_call_instrumentation(state, codelike, offset, args, 4,
PY_MONITORING_EVENT_CALL);
}
int
_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
int lineno)
{
assert(state->active);
PyObject *lno = PyLong_FromLong(lineno);
if (lno == NULL) {
return -1;
}
PyObject *args[3] = { NULL, NULL, lno };
int res= capi_call_instrumentation(state, codelike, offset, args, 2,
PY_MONITORING_EVENT_LINE);
Py_DECREF(lno);
return res;
}
int
_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject *target_offset)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_JUMP);
}
int
_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject *target_offset)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_BRANCH_RIGHT);
}
int
_PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject *target_offset)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_BRANCH_RIGHT);
}
int
_PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject *target_offset)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, target_offset };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_BRANCH_LEFT);
}
int
_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
PyObject *retval)
{
assert(state->active);
PyObject *args[4] = { NULL, NULL, NULL, retval };
return capi_call_instrumentation(state, codelike, offset, args, 3,
PY_MONITORING_EVENT_C_RETURN);
}
static inline int
exception_event_setup(PyObject **exc, int event) {
*exc = PyErr_GetRaisedException();
if (*exc == NULL) {
PyErr_Format(PyExc_ValueError,
"Firing event %d with no exception set",
event);
return -1;
}
return 0;
}
static inline int
exception_event_teardown(int err, PyObject *exc) {
if (err == 0) {
PyErr_SetRaisedException(exc);
}
else {
assert(PyErr_Occurred());
Py_XDECREF(exc);
}
return err;
}
int
_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_PY_THROW;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_RAISE;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_C_RAISE;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_RERAISE;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
{
int event = PY_MONITORING_EVENT_PY_UNWIND;
assert(state->active);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
return exception_event_teardown(err, exc);
}
int
_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value)
{
int event = PY_MONITORING_EVENT_STOP_ITERATION;
assert(state->active);
assert(!PyErr_Occurred());
PyErr_SetObject(PyExc_StopIteration, value);
PyObject *exc;
if (exception_event_setup(&exc, event) < 0) {
return -1;
}
PyObject *args[4] = { NULL, NULL, NULL, exc };
int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
Py_DECREF(exc);
return exception_event_teardown(err, NULL);
}
/* Handle legacy BRANCH event */
typedef struct _PyLegacyBranchEventHandler {
PyObject_HEAD
vectorcallfunc vectorcall;
PyObject *handler;
bool right;
int tool_id;
} _PyLegacyBranchEventHandler;
static void
dealloc_branch_handler(_PyLegacyBranchEventHandler *self)
{
Py_CLEAR(self->handler);
PyObject_Free((PyObject *)self);
}
static PyTypeObject _PyLegacyBranchEventHandler_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"sys.monitoring.branch_event_handler",
sizeof(_PyLegacyBranchEventHandler),
.tp_dealloc = (destructor)dealloc_branch_handler,
.tp_vectorcall_offset = offsetof(_PyLegacyBranchEventHandler, vectorcall),
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_DISALLOW_INSTANTIATION,
.tp_call = PyVectorcall_Call,
};
static PyObject *
branch_handler(
_PyLegacyBranchEventHandler *self, PyObject *const *args,
size_t nargsf, PyObject *kwnames
) {
PyObject *res = PyObject_Vectorcall(self->handler, args, nargsf, kwnames);
if (res == &_PyInstrumentation_DISABLE) {
// Find the other instrumented instruction and remove tool
assert(PyVectorcall_NARGS(nargsf) >= 2);
PyObject *offset_obj = args[1];
int bytes_offset = PyLong_AsLong(offset_obj);
if (PyErr_Occurred()) {
return NULL;
}
PyCodeObject *code = (PyCodeObject *)args[0];
if (!PyCode_Check(code) || (bytes_offset & 1)) {
return res;
}
int offset = bytes_offset / 2;
/* We need FOR_ITER and POP_JUMP_ to be the same size */
assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1);
if (self->right) {
offset += 2;
}
if (offset >= Py_SIZE(code)) {
return res;
}
int other_event = self->right ?
PY_MONITORING_EVENT_BRANCH_LEFT : PY_MONITORING_EVENT_BRANCH_RIGHT;
LOCK_CODE(code);
remove_tools(code, offset, other_event, 1 << self->tool_id);
UNLOCK_CODE();
}
return res;
}
static PyObject *make_branch_handler(int tool_id, PyObject *handler, bool right)
{
_PyLegacyBranchEventHandler *callback =
PyObject_NEW(_PyLegacyBranchEventHandler, &_PyLegacyBranchEventHandler_Type);
if (callback == NULL) {
return NULL;
}
callback->vectorcall = (vectorcallfunc)branch_handler;
callback->handler = Py_NewRef(handler);
callback->right = right;
callback->tool_id = tool_id;
return (PyObject *)callback;
}
/* Consumes a reference to obj */
static PyObject *exchange_callables(int tool_id, int event_id, PyObject *obj)
{
PyInterpreterState *is = _PyInterpreterState_GET();
return _Py_atomic_exchange_ptr(&is->monitoring_callables[tool_id][event_id], obj);
}
PyObject *
_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj)
{
assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS);
assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS);
PyObject *res;
if (event_id == PY_MONITORING_EVENT_BRANCH) {
PyObject *left, *right;
if (obj == NULL) {
left = NULL;
right = NULL;
}
else {
right = make_branch_handler(tool_id, obj, true);
if (right == NULL) {
return NULL;
}
left = make_branch_handler(tool_id, obj, false);
if (left == NULL) {
Py_DECREF(right);
return NULL;
}
}
Py_XDECREF(exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_RIGHT, right));
res = exchange_callables(tool_id, PY_MONITORING_EVENT_BRANCH_LEFT, left);
}
else {
res = exchange_callables(tool_id, event_id, Py_XNewRef(obj));
}
if (res != NULL && Py_TYPE(res) == &_PyLegacyBranchEventHandler_Type) {
_PyLegacyBranchEventHandler *wrapper = (_PyLegacyBranchEventHandler *)res;
res = Py_NewRef(wrapper->handler);
Py_DECREF(wrapper);
}
return res;
}
/* Branch Iterator */
typedef struct {
PyObject_HEAD
PyCodeObject *bi_code;
int bi_offset;
} branchesiterator;
static PyObject *
int_triple(int a, int b, int c) {
PyObject *obja = PyLong_FromLong(a);
PyObject *objb = NULL;
PyObject *objc = NULL;
if (obja == NULL) {
goto error;
}
objb = PyLong_FromLong(b);
if (objb == NULL) {
goto error;
}
objc = PyLong_FromLong(c);
if (objc == NULL) {
goto error;
}
PyObject *array[3] = { obja, objb, objc };
return _PyTuple_FromArraySteal(array, 3);
error:
Py_XDECREF(obja);
Py_XDECREF(objb);
Py_XDECREF(objc);
return NULL;
}
static PyObject *
branchesiter_next(branchesiterator *bi)
{
int offset = bi->bi_offset;
while (offset < Py_SIZE(bi->bi_code)) {
_Py_CODEUNIT inst = _Py_GetBaseCodeUnit(bi->bi_code, offset);
int next_offset = offset + _PyInstruction_GetLength(bi->bi_code, offset);
int event = EVENT_FOR_OPCODE[inst.op.code];
if (event == PY_MONITORING_EVENT_BRANCH_RIGHT) {
/* Skip NOT_TAKEN */
int not_taken = next_offset + 1;
bi->bi_offset = not_taken;
return int_triple(offset*2, not_taken*2, (next_offset + inst.op.arg)*2);
}
offset = next_offset;
}
return NULL;
}
static void
branchesiter_dealloc(branchesiterator *bi)
{
Py_DECREF(bi->bi_code);
PyObject_Free(bi);
}
static PyTypeObject _PyBranchesIterator = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"line_iterator", /* tp_name */
sizeof(branchesiterator), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
.tp_dealloc = (destructor)branchesiter_dealloc,
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
.tp_iter = PyObject_SelfIter,
.tp_iternext = (iternextfunc)branchesiter_next,
.tp_free = PyObject_Del,
};
PyObject *
_PyInstrumentation_BranchesIterator(PyCodeObject *code)
{
branchesiterator *bi = (branchesiterator *)PyType_GenericAlloc(&_PyBranchesIterator, 0);
if (bi == NULL) {
return NULL;
}
bi->bi_code = (PyCodeObject*)Py_NewRef(code);
bi->bi_offset = 0;
return (PyObject *)bi;
}