1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
|
#include "Python.h"
#include "pycore_backoff.h"
#include "pycore_call.h"
#include "pycore_ceval.h"
#include "pycore_cell.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_long.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_optimizer.h"
#include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_range.h"
#include "pycore_setobject.h"
#include "pycore_sliceobject.h"
#include "pycore_descrobject.h"
#include "ceval_macros.h"
#undef CURRENT_OPARG
#define CURRENT_OPARG() (_oparg)
#undef CURRENT_OPERAND
#define CURRENT_OPERAND() (_operand)
#undef DEOPT_IF
#define DEOPT_IF(COND, INSTNAME) \
do { \
if ((COND)) { \
goto deoptimize; \
} \
} while (0)
#undef ENABLE_SPECIALIZATION
#define ENABLE_SPECIALIZATION (0)
#undef GOTO_ERROR
#define GOTO_ERROR(LABEL) \
do { \
goto LABEL ## _tier_two; \
} while (0)
#undef GOTO_TIER_TWO
#define GOTO_TIER_TWO(EXECUTOR) \
do { \
OPT_STAT_INC(traces_executed); \
__attribute__((musttail)) \
return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \
} while (0)
#undef GOTO_TIER_ONE
#define GOTO_TIER_ONE(TARGET) \
do { \
_PyFrame_SetStackPointer(frame, stack_pointer); \
return TARGET; \
} while (0)
#undef LOAD_IP
#define LOAD_IP(UNUSED) \
do { \
} while (0)
#define PATCH_VALUE(TYPE, NAME, ALIAS) \
PyAPI_DATA(void) ALIAS; \
TYPE NAME = (TYPE)(uintptr_t)&ALIAS;
#define PATCH_JUMP(ALIAS) \
do { \
PyAPI_DATA(void) ALIAS; \
__attribute__((musttail)) \
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
} while (0)
#undef JUMP_TO_JUMP_TARGET
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
#undef JUMP_TO_ERROR
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
_Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate)
{
// Locals that the instruction implementations expect to exist:
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
int oparg;
int uopcode = _JIT_OPCODE;
_Py_CODEUNIT *next_instr;
// Other stuff we need handy:
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
#if SIZEOF_VOID_P == 8
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
#else
assert(SIZEOF_VOID_P == 4);
PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI)
PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO)
uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo;
#endif
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX)
OPT_STAT_INC(uops_executed);
UOP_STAT_INC(uopcode, execution_count);
// The actual instruction definitions (only one will be used):
if (uopcode == _JUMP_TO_TOP) {
PATCH_JUMP(_JIT_TOP);
}
switch (uopcode) {
#include "executor_cases.c.h"
default:
Py_UNREACHABLE();
}
PATCH_JUMP(_JIT_CONTINUE);
// Labels that the instruction implementations expect to exist:
error_tier_two:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(NULL);
exit_to_tier1:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
exit_to_tier1_dynamic:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(frame->instr_ptr);
exit_to_trace:
{
_PyExitData *exit = ¤t_executor->exits[_exit_index];
Py_INCREF(exit->executor);
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_TWO(exit->executor);
}
}
|
