1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
#include "Python.h"
#include "pycore_backoff.h"
#include "pycore_call.h"
#include "pycore_cell.h"
#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_descrobject.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"
#include "pycore_floatobject.h"
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"
#include "pycore_interpframe.h"
#include "pycore_interpolation.h"
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_list.h"
#include "pycore_long.h"
#include "pycore_opcode_metadata.h"
#include "pycore_opcode_utils.h"
#include "pycore_optimizer.h"
#include "pycore_pyatomic_ft_wrappers.h"
#include "pycore_range.h"
#include "pycore_setobject.h"
#include "pycore_sliceobject.h"
#include "pycore_stackref.h"
#include "pycore_template.h"
#include "pycore_tuple.h"
#include "pycore_unicodeobject.h"
#include "ceval_macros.h"
#include "jit.h"
#undef CURRENT_OPERAND0_64
#define CURRENT_OPERAND0_64() (_operand0_64)
#undef CURRENT_OPERAND1_64
#define CURRENT_OPERAND1_64() (_operand1_64)
#undef CURRENT_OPARG
#undef CURRENT_OPERAND0_16
#undef CURRENT_OPERAND0_32
#undef CURRENT_OPERAND1_16
#undef CURRENT_OPERAND1_32
#if SUPPORTS_SMALL_CONSTS
#define CURRENT_OPARG() (_oparg_16)
#define CURRENT_OPERAND0_32() (_operand0_32)
#define CURRENT_OPERAND0_16() (_operand0_16)
#define CURRENT_OPERAND1_32() (_operand1_32)
#define CURRENT_OPERAND1_16() (_operand1_16)
#else
#define CURRENT_OPARG() (_oparg)
#define CURRENT_OPERAND0_32() (_operand0_64)
#define CURRENT_OPERAND0_16() (_operand0_64)
#define CURRENT_OPERAND1_32() (_operand1_64)
#define CURRENT_OPERAND1_16() (_operand1_64)
#endif
#undef CURRENT_TARGET
#define CURRENT_TARGET() (_target)
#undef TIER2_TO_TIER2
#define TIER2_TO_TIER2(EXECUTOR) \
do { \
OPT_STAT_INC(traces_executed); \
_PyExecutorObject *_executor = (EXECUTOR); \
jit_func_preserve_none jitted = _executor->jit_code; \
__attribute__((musttail)) return jitted(frame, stack_pointer, tstate, \
_tos_cache0, _tos_cache1, _tos_cache2); \
} while (0)
#undef GOTO_TIER_ONE_SETUP
#define GOTO_TIER_ONE_SETUP \
tstate->current_executor = NULL; \
_PyFrame_SetStackPointer(frame, stack_pointer);
#undef LOAD_IP
#define LOAD_IP(UNUSED) \
do { \
} while (0)
#undef LLTRACE_RESUME_FRAME
#ifdef Py_DEBUG
#define LLTRACE_RESUME_FRAME() (frame->lltrace = 0)
#else
#define LLTRACE_RESUME_FRAME() do {} while (0)
#endif
#define PATCH_JUMP(ALIAS) \
do { \
DECLARE_TARGET(ALIAS); \
__attribute__((musttail)) return ALIAS(frame, stack_pointer, tstate, \
_tos_cache0, _tos_cache1, _tos_cache2); \
} while (0)
#undef JUMP_TO_JUMP_TARGET
#define JUMP_TO_JUMP_TARGET() PATCH_JUMP(_JIT_JUMP_TARGET)
#undef JUMP_TO_ERROR
#define JUMP_TO_ERROR() PATCH_JUMP(_JIT_ERROR_TARGET)
#define TIER_TWO 2
#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif
__attribute__((preserve_none)) _Py_CODEUNIT *
_JIT_ENTRY(
_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate,
_PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2
) {
// Locals that the instruction implementations expect to exist:
PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR)
int oparg;
int uopcode = _JIT_OPCODE;
_Py_CODEUNIT *next_instr;
// Other stuff we need handy:
#if SIZEOF_VOID_P == 8
PATCH_VALUE(uint64_t, _operand0_64, _JIT_OPERAND0)
PATCH_VALUE(uint64_t, _operand1_64, _JIT_OPERAND1)
#else
assert(SIZEOF_VOID_P == 4);
PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI)
PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO)
uint64_t _operand0_64 = ((uint64_t)_operand0_hi << 32) | _operand0_lo;
PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI)
PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO)
uint64_t _operand1_64 = ((uint64_t)_operand1_hi << 32) | _operand1_lo;
#endif
#if SUPPORTS_SMALL_CONSTS
PATCH_VALUE(uint32_t, _operand0_32, _JIT_OPERAND0_32)
PATCH_VALUE(uint32_t, _operand1_32, _JIT_OPERAND1_32)
PATCH_VALUE(uint16_t, _operand0_16, _JIT_OPERAND0_16)
PATCH_VALUE(uint16_t, _operand1_16, _JIT_OPERAND1_16)
PATCH_VALUE(uint16_t, _oparg_16, _JIT_OPARG_16)
#else
PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG)
#endif
PATCH_VALUE(uint32_t, _target, _JIT_TARGET)
OPT_STAT_INC(uops_executed);
UOP_STAT_INC(uopcode, execution_count);
switch (uopcode) {
// The actual instruction definition gets inserted here:
CASE
default:
Py_UNREACHABLE();
}
PATCH_JUMP(_JIT_CONTINUE);
}
|
