1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
#include "Python.h"
#include "pycore_backoff.h"
#include "pycore_call.h"
#include "pycore_ceval.h"
#include "pycore_cell.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_long.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_optimizer.h"
#include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_range.h"
#include "pycore_setobject.h"
#include "pycore_sliceobject.h"
#include "pycore_descrobject.h"
#include "pycore_stackref.h"
#include "ceval_macros.h"
#undef CURRENT_OPARG
#define CURRENT_OPARG() (_oparg)
#undef CURRENT_OPERAND
#define CURRENT_OPERAND() (_operand)
#undef DEOPT_IF
#define DEOPT_IF(COND, INSTNAME) \
do { \
if ((COND)) { \
goto deoptimize; \
} \
} while (0)
#undef ENABLE_SPECIALIZATION
#define ENABLE_SPECIALIZATION (0)
#undef GOTO_ERROR
#define GOTO_ERROR(LABEL) \
do { \
goto LABEL ## _tier_two; \
} while (0)
#undef GOTO_TIER_TWO
#define GOTO_TIER_TWO(EXECUTOR) \
do { \
OPT_STAT_INC(traces_executed); \
__attribute__((musttail)) \
return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \
} while (0)
#undef GOTO_TIER_ONE
#define GOTO_TIER_ONE(TARGET) \
do { \
_PyFrame_SetStackPointer(frame, stack_pointer); \
return TARGET; \
} while (0)
#undef LOAD_IP
#define LOAD_IP(UNUSED) \
do { \
} while (0)
#define PATCH_VALUE(TYPE, NAME, ALIAS) \
PyAPI_DATA(void) ALIAS; \
TYPE NAME = (TYPE)(uintptr_t)&ALIAS;
#define PATCH_JUMP(ALIAS) \
do { \
PyAPI_DATA(void) ALIAS; \
__attribute__((musttail)) \
return ((jit_func)&ALIAS)(frame, stack_pointer, tstate); \
} while (0)
#undef JUMP_TO_JUMP_TARGET
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
#undef JUMP_TO_ERROR
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
#undef WITHIN_STACK_BOUNDS
#define WITHIN_STACK_BOUNDS() 1
_Py_CODEUNIT *
_JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate)
{
// Locals that the instruction implementations expect to exist:
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
int oparg;
int uopcode = _JIT_OPCODE;
_Py_CODEUNIT *next_instr;
// Other stuff we need handy:
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
#if SIZEOF_VOID_P == 8
PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND)
#else
assert(SIZEOF_VOID_P == 4);
PATCH_VALUE(uint32_t, _operand_hi, _JIT_OPERAND_HI)
PATCH_VALUE(uint32_t, _operand_lo, _JIT_OPERAND_LO)
uint64_t _operand = ((uint64_t)_operand_hi << 32) | _operand_lo;
#endif
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
OPT_STAT_INC(uops_executed);
UOP_STAT_INC(uopcode, execution_count);
// The actual instruction definitions (only one will be used):
switch (uopcode) {
#include "executor_cases.c.h"
default:
Py_UNREACHABLE();
}
PATCH_JUMP(_JIT_CONTINUE);
// Labels that the instruction implementations expect to exist:
error_tier_two:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(NULL);
exit_to_tier1:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target);
exit_to_tier1_dynamic:
tstate->previous_executor = (PyObject *)current_executor;
GOTO_TIER_ONE(frame->instr_ptr);
}
|
