summaryrefslogtreecommitdiffstats
path: root/Python
diff options
context:
space:
mode:
Diffstat (limited to 'Python')
-rw-r--r--Python/bytecodes.c72
-rw-r--r--Python/ceval.c1
-rw-r--r--Python/codegen.c21
-rw-r--r--Python/executor_cases.c.h51
-rw-r--r--Python/flowgraph.c13
-rw-r--r--Python/generated_cases.c.h583
-rw-r--r--Python/opcode_targets.h39
-rw-r--r--Python/optimizer.c2
-rw-r--r--Python/optimizer_bytecodes.c2
-rw-r--r--Python/optimizer_cases.c.h21
-rw-r--r--Python/specialize.c298
11 files changed, 477 insertions, 626 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index bef120b..703d7ec 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -537,6 +537,11 @@ dummy_func(
BINARY_OP_ADD_FLOAT,
BINARY_OP_SUBTRACT_FLOAT,
BINARY_OP_ADD_UNICODE,
+ BINARY_OP_SUBSCR_LIST_INT,
+ BINARY_OP_SUBSCR_TUPLE_INT,
+ BINARY_OP_SUBSCR_STR_INT,
+ BINARY_OP_SUBSCR_DICT,
+ BINARY_OP_SUBSCR_GETITEM,
// BINARY_OP_INPLACE_ADD_UNICODE, // See comments at that opcode.
BINARY_OP_EXTEND,
};
@@ -787,39 +792,6 @@ dummy_func(
macro(BINARY_OP_INPLACE_ADD_UNICODE) =
_GUARD_BOTH_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE;
- family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = {
- BINARY_SUBSCR_DICT,
- BINARY_SUBSCR_GETITEM,
- BINARY_SUBSCR_LIST_INT,
- BINARY_SUBSCR_STR_INT,
- BINARY_SUBSCR_TUPLE_INT,
- };
-
- specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) {
- #if ENABLE_SPECIALIZATION_FT
- assert(frame->stackpointer == NULL);
- if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
- next_instr = this_instr;
- _Py_Specialize_BinarySubscr(container, sub, next_instr);
- DISPATCH_SAME_OPARG();
- }
- OPCODE_DEFERRED_INC(BINARY_SUBSCR);
- ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
- #endif /* ENABLE_SPECIALIZATION_FT */
- }
-
- op(_BINARY_SUBSCR, (container, sub -- res)) {
- PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
- PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
-
- PyObject *res_o = PyObject_GetItem(container_o, sub_o);
- DECREF_INPUTS();
- ERROR_IF(res_o == NULL, error);
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
-
- macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR;
-
specializing op(_SPECIALIZE_BINARY_SLICE, (container, start, stop -- container, start, stop)) {
// Placeholder until we implement BINARY_SLICE specialization
#if ENABLE_SPECIALIZATION
@@ -871,7 +843,7 @@ dummy_func(
macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE;
- inst(BINARY_SUBSCR_LIST_INT, (unused/1, list_st, sub_st -- res)) {
+ inst(BINARY_OP_SUBSCR_LIST_INT, (unused/5, list_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
@@ -884,10 +856,10 @@ dummy_func(
#ifdef Py_GIL_DISABLED
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
DEOPT_IF(res_o == NULL);
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
#else
DEOPT_IF(index >= PyList_GET_SIZE(list));
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -898,7 +870,7 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_STR_INT, (unused/1, str_st, sub_st -- res)) {
+ inst(BINARY_OP_SUBSCR_STR_INT, (unused/5, str_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
@@ -910,7 +882,7 @@ dummy_func(
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
DEAD(sub_st);
@@ -918,7 +890,7 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple_st, sub_st -- res)) {
+ inst(BINARY_OP_SUBSCR_TUPLE_INT, (unused/5, tuple_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
@@ -929,7 +901,7 @@ dummy_func(
DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
DEOPT_IF(index >= PyTuple_GET_SIZE(tuple));
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -939,12 +911,12 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
}
- inst(BINARY_SUBSCR_DICT, (unused/1, dict_st, sub_st -- res)) {
+ inst(BINARY_OP_SUBSCR_DICT, (unused/5, dict_st, sub_st -- res)) {
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
DEOPT_IF(!PyDict_CheckExact(dict));
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o;
int rc = PyDict_GetItemRef(dict, sub, &res_o);
if (rc == 0) {
@@ -955,7 +927,7 @@ dummy_func(
res = PyStackRef_FromPyObjectSteal(res_o);
}
- op(_BINARY_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) {
+ op(_BINARY_OP_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) {
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE));
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
@@ -968,10 +940,10 @@ dummy_func(
assert(code->co_argcount == 2);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
getitem = PyStackRef_FromPyObjectNew(getitem_o);
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
}
- op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) {
+ op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _PyInterpreterFrame* )) {
new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
new_frame->localsplus[0] = container;
new_frame->localsplus[1] = sub;
@@ -979,11 +951,11 @@ dummy_func(
frame->return_offset = INSTRUCTION_SIZE;
}
- macro(BINARY_SUBSCR_GETITEM) =
- unused/1 + // Skip over the counter
+ macro(BINARY_OP_SUBSCR_GETITEM) =
+ unused/5 + // Skip over the counter and cache
_CHECK_PEP_523 +
- _BINARY_SUBSCR_CHECK_FUNC +
- _BINARY_SUBSCR_INIT_CALL +
+ _BINARY_OP_SUBSCR_CHECK_FUNC +
+ _BINARY_OP_SUBSCR_INIT_CALL +
_PUSH_FRAME;
inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) {
@@ -4816,7 +4788,7 @@ dummy_func(
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
assert(NB_ADD <= oparg);
- assert(oparg <= NB_INPLACE_XOR);
+ assert(oparg <= NB_OPARG_LAST);
}
op(_BINARY_OP, (lhs, rhs -- res)) {
diff --git a/Python/ceval.c b/Python/ceval.c
index 6c8e39a..1e4f1f3 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -362,6 +362,7 @@ const binaryfunc _PyEval_BinaryOps[] = {
[NB_INPLACE_SUBTRACT] = PyNumber_InPlaceSubtract,
[NB_INPLACE_TRUE_DIVIDE] = PyNumber_InPlaceTrueDivide,
[NB_INPLACE_XOR] = PyNumber_InPlaceXor,
+ [NB_SUBSCR] = PyObject_GetItem,
};
const conversion_func _PyEval_ConversionFuncs[4] = {
diff --git a/Python/codegen.c b/Python/codegen.c
index e9853d7..cd77b34 100644
--- a/Python/codegen.c
+++ b/Python/codegen.c
@@ -5076,7 +5076,7 @@ codegen_augassign(compiler *c, stmt_ty s)
VISIT(c, expr, e->v.Subscript.slice);
ADDOP_I(c, loc, COPY, 2);
ADDOP_I(c, loc, COPY, 2);
- ADDOP(c, loc, BINARY_SUBSCR);
+ ADDOP_I(c, loc, BINARY_OP, NB_SUBSCR);
}
break;
case Name_kind:
@@ -5242,7 +5242,6 @@ codegen_subscript(compiler *c, expr_ty e)
{
location loc = LOC(e);
expr_context_ty ctx = e->v.Subscript.ctx;
- int op = 0;
if (ctx == Load) {
RETURN_IF_ERROR(check_subscripter(c, e->v.Subscript.value));
@@ -5265,12 +5264,16 @@ codegen_subscript(compiler *c, expr_ty e)
else {
VISIT(c, expr, e->v.Subscript.slice);
switch (ctx) {
- case Load: op = BINARY_SUBSCR; break;
- case Store: op = STORE_SUBSCR; break;
- case Del: op = DELETE_SUBSCR; break;
+ case Load:
+ ADDOP_I(c, loc, BINARY_OP, NB_SUBSCR);
+ break;
+ case Store:
+ ADDOP(c, loc, STORE_SUBSCR);
+ break;
+ case Del:
+ ADDOP(c, loc, DELETE_SUBSCR);
+ break;
}
- assert(op);
- ADDOP(c, loc, op);
}
return SUCCESS;
}
@@ -5502,7 +5505,7 @@ pattern_helper_sequence_unpack(compiler *c, location loc,
return SUCCESS;
}
-// Like pattern_helper_sequence_unpack, but uses BINARY_SUBSCR instead of
+// Like pattern_helper_sequence_unpack, but uses BINARY_OP/NB_SUBSCR instead of
// UNPACK_SEQUENCE / UNPACK_EX. This is more efficient for patterns with a
// starred wildcard like [first, *_] / [first, *_, last] / [*_, last] / etc.
static int
@@ -5533,7 +5536,7 @@ pattern_helper_sequence_subscr(compiler *c, location loc,
ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromSsize_t(size - i));
ADDOP_BINARY(c, loc, Sub);
}
- ADDOP(c, loc, BINARY_SUBSCR);
+ ADDOP_I(c, loc, BINARY_OP, NB_SUBSCR);
RETURN_IF_ERROR(codegen_pattern_subpattern(c, pattern, pc));
}
// Pop the subject, we're done with it:
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index dc65728..96b7386 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -1029,31 +1029,6 @@
break;
}
- case _BINARY_SUBSCR: {
- _PyStackRef sub;
- _PyStackRef container;
- _PyStackRef res;
- sub = stack_pointer[-1];
- container = stack_pointer[-2];
- PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
- PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = PyObject_GetItem(container_o, sub_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- PyStackRef_CLOSE(container);
- PyStackRef_CLOSE(sub);
- if (res_o == NULL) {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- JUMP_TO_ERROR();
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
case _BINARY_SLICE: {
_PyStackRef stop;
_PyStackRef start;
@@ -1136,7 +1111,7 @@
break;
}
- case _BINARY_SUBSCR_LIST_INT: {
+ case _BINARY_OP_SUBSCR_LIST_INT: {
_PyStackRef sub_st;
_PyStackRef list_st;
_PyStackRef res;
@@ -1166,13 +1141,13 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
#else
if (index >= PyList_GET_SIZE(list)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -1190,7 +1165,7 @@
break;
}
- case _BINARY_SUBSCR_STR_INT: {
+ case _BINARY_OP_SUBSCR_STR_INT: {
_PyStackRef sub_st;
_PyStackRef str_st;
_PyStackRef res;
@@ -1221,7 +1196,7 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -2;
@@ -1236,7 +1211,7 @@
break;
}
- case _BINARY_SUBSCR_TUPLE_INT: {
+ case _BINARY_OP_SUBSCR_TUPLE_INT: {
_PyStackRef sub_st;
_PyStackRef tuple_st;
_PyStackRef res;
@@ -1262,7 +1237,7 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -1279,7 +1254,7 @@
break;
}
- case _BINARY_SUBSCR_DICT: {
+ case _BINARY_OP_SUBSCR_DICT: {
_PyStackRef sub_st;
_PyStackRef dict_st;
_PyStackRef res;
@@ -1291,7 +1266,7 @@
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
int rc = PyDict_GetItemRef(dict, sub, &res_o);
@@ -1316,7 +1291,7 @@
break;
}
- case _BINARY_SUBSCR_CHECK_FUNC: {
+ case _BINARY_OP_SUBSCR_CHECK_FUNC: {
_PyStackRef container;
_PyStackRef getitem;
container = stack_pointer[-2];
@@ -1344,14 +1319,14 @@
JUMP_TO_JUMP_TARGET();
}
getitem = PyStackRef_FromPyObjectNew(getitem_o);
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
stack_pointer[0] = getitem;
stack_pointer += 1;
assert(WITHIN_STACK_BOUNDS());
break;
}
- case _BINARY_SUBSCR_INIT_CALL: {
+ case _BINARY_OP_SUBSCR_INIT_CALL: {
_PyStackRef getitem;
_PyStackRef sub;
_PyStackRef container;
@@ -1362,7 +1337,7 @@
new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
new_frame->localsplus[0] = container;
new_frame->localsplus[1] = sub;
- frame->return_offset = 2 ;
+ frame->return_offset = 6 ;
stack_pointer[-3].bits = (uintptr_t)new_frame;
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
diff --git a/Python/flowgraph.c b/Python/flowgraph.c
index 95ab53c..12eedc3 100644
--- a/Python/flowgraph.c
+++ b/Python/flowgraph.c
@@ -2,6 +2,7 @@
#include <stdbool.h>
#include "Python.h"
+#include "opcode.h"
#include "pycore_flowgraph.h"
#include "pycore_compile.h"
#include "pycore_intrinsics.h"
@@ -1492,10 +1493,14 @@ newop_from_folded(PyObject *newconst, PyObject *consts,
}
static int
-optimize_if_const_subscr(basicblock *bb, int n, PyObject *consts, PyObject *const_cache)
+optimize_if_const_op(basicblock *bb, int n, PyObject *consts, PyObject *const_cache)
{
cfg_instr *subscr = &bb->b_instr[n];
- assert(subscr->i_opcode == BINARY_SUBSCR);
+ assert(subscr->i_opcode == BINARY_OP);
+ if (subscr->i_oparg != NB_SUBSCR) {
+ /* TODO: support other binary ops */
+ return SUCCESS;
+ }
cfg_instr *arg, *idx;
if (!find_load_const_pair(bb, n-1, &arg, &idx)) {
return SUCCESS;
@@ -2033,8 +2038,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts)
INSTR_SET_OP0(inst, NOP);
}
break;
- case BINARY_SUBSCR:
- RETURN_IF_ERROR(optimize_if_const_subscr(bb, i, consts, const_cache));
+ case BINARY_OP:
+ RETURN_IF_ERROR(optimize_if_const_op(bb, i, consts, const_cache));
break;
}
}
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 914b069..f02e13f 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -50,7 +50,7 @@
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION_FT */
assert(NB_ADD <= oparg);
- assert(oparg <= NB_INPLACE_XOR);
+ assert(oparg <= NB_OPARG_LAST);
}
/* Skip 4 cache entries */
// _BINARY_OP
@@ -480,251 +480,31 @@
DISPATCH();
}
- TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+ TARGET(BINARY_OP_SUBSCR_DICT) {
#if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_OP_SUBTRACT_FLOAT;
+ int opcode = BINARY_OP_SUBSCR_DICT;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
next_instr += 6;
- INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
+ INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT);
static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
- _PyStackRef left;
- _PyStackRef right;
- _PyStackRef res;
- // _GUARD_BOTH_FLOAT
- {
- right = stack_pointer[-1];
- left = stack_pointer[-2];
- PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!PyFloat_CheckExact(left_o)) {
- UPDATE_MISS_STATS(BINARY_OP);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
- }
- if (!PyFloat_CheckExact(right_o)) {
- UPDATE_MISS_STATS(BINARY_OP);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
- }
- }
- /* Skip 5 cache entries */
- // _BINARY_OP_SUBTRACT_FLOAT
- {
- PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- assert(PyFloat_CheckExact(left_o));
- assert(PyFloat_CheckExact(right_o));
- STAT_INC(BINARY_OP, hit);
- double dres =
- ((PyFloatObject *)left_o)->ob_fval -
- ((PyFloatObject *)right_o)->ob_fval;
- PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
- }
-
- TARGET(BINARY_OP_SUBTRACT_INT) {
- #if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_OP_SUBTRACT_INT;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- frame->instr_ptr = next_instr;
- next_instr += 6;
- INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
- _PyStackRef left;
- _PyStackRef right;
- _PyStackRef res;
- // _GUARD_BOTH_INT
- {
- right = stack_pointer[-1];
- left = stack_pointer[-2];
- PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- if (!PyLong_CheckExact(left_o)) {
- UPDATE_MISS_STATS(BINARY_OP);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
- }
- if (!PyLong_CheckExact(right_o)) {
- UPDATE_MISS_STATS(BINARY_OP);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
- JUMP_TO_PREDICTED(BINARY_OP);
- }
- }
- /* Skip 5 cache entries */
- // _BINARY_OP_SUBTRACT_INT
- {
- PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
- PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
- assert(PyLong_CheckExact(left_o));
- assert(PyLong_CheckExact(right_o));
- STAT_INC(BINARY_OP, hit);
- PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
- PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
- PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
- }
-
- TARGET(BINARY_SLICE) {
- #if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SLICE;
- (void)(opcode);
- #endif
- frame->instr_ptr = next_instr;
- next_instr += 1;
- INSTRUCTION_STATS(BINARY_SLICE);
- _PyStackRef container;
- _PyStackRef start;
- _PyStackRef stop;
- _PyStackRef res;
- // _SPECIALIZE_BINARY_SLICE
- {
- // Placeholder until we implement BINARY_SLICE specialization
- #if ENABLE_SPECIALIZATION
- OPCODE_DEFERRED_INC(BINARY_SLICE);
- #endif /* ENABLE_SPECIALIZATION */
- }
- // _BINARY_SLICE
- {
- stop = stack_pointer[-1];
- start = stack_pointer[-2];
- container = stack_pointer[-3];
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
- PyStackRef_AsPyObjectSteal(stop));
- stack_pointer = _PyFrame_GetStackPointer(frame);
- PyObject *res_o;
- // Can't use ERROR_IF() here, because we haven't
- // DECREF'ed container yet, and we still own slice.
- if (slice == NULL) {
- res_o = NULL;
- }
- else {
- stack_pointer += -2;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
- Py_DECREF(slice);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- stack_pointer += 2;
- assert(WITHIN_STACK_BOUNDS());
- }
- stack_pointer += -3;
- assert(WITHIN_STACK_BOUNDS());
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyStackRef_CLOSE(container);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (res_o == NULL) {
- JUMP_TO_LABEL(error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
- stack_pointer[0] = res;
- stack_pointer += 1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
- }
-
- TARGET(BINARY_SUBSCR) {
- #if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR;
- (void)(opcode);
- #endif
- frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR);
- PREDICTED_BINARY_SUBSCR:;
- _Py_CODEUNIT* const this_instr = next_instr - 2;
- (void)this_instr;
- _PyStackRef container;
- _PyStackRef sub;
- _PyStackRef res;
- // _SPECIALIZE_BINARY_SUBSCR
- {
- sub = stack_pointer[-1];
- container = stack_pointer[-2];
- uint16_t counter = read_u16(&this_instr[1].cache);
- (void)counter;
- #if ENABLE_SPECIALIZATION_FT
- assert(frame->stackpointer == NULL);
- if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
- next_instr = this_instr;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- _Py_Specialize_BinarySubscr(container, sub, next_instr);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- DISPATCH_SAME_OPARG();
- }
- OPCODE_DEFERRED_INC(BINARY_SUBSCR);
- ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
- #endif /* ENABLE_SPECIALIZATION_FT */
- }
- // _BINARY_SUBSCR
- {
- PyObject *container_o = PyStackRef_AsPyObjectBorrow(container);
- PyObject *sub_o = PyStackRef_AsPyObjectBorrow(sub);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- PyObject *res_o = PyObject_GetItem(container_o, sub_o);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- PyStackRef_CLOSE(container);
- PyStackRef_CLOSE(sub);
- if (res_o == NULL) {
- JUMP_TO_LABEL(pop_2_error);
- }
- res = PyStackRef_FromPyObjectSteal(res_o);
- }
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- DISPATCH();
- }
-
- TARGET(BINARY_SUBSCR_DICT) {
- #if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR_DICT;
- (void)(opcode);
- #endif
- _Py_CODEUNIT* const this_instr = next_instr;
- (void)this_instr;
- frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR_DICT);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
_PyStackRef dict_st;
_PyStackRef sub_st;
_PyStackRef res;
- /* Skip 1 cache entry */
+ /* Skip 5 cache entries */
sub_st = stack_pointer[-1];
dict_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
if (!PyDict_CheckExact(dict)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o;
_PyFrame_SetStackPointer(frame, stack_pointer);
int rc = PyDict_GetItemRef(dict, sub, &res_o);
@@ -747,70 +527,70 @@
DISPATCH();
}
- TARGET(BINARY_SUBSCR_GETITEM) {
+ TARGET(BINARY_OP_SUBSCR_GETITEM) {
#if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR_GETITEM;
+ int opcode = BINARY_OP_SUBSCR_GETITEM;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef container;
_PyStackRef getitem;
_PyStackRef sub;
_PyInterpreterFrame *new_frame;
- /* Skip 1 cache entry */
+ /* Skip 5 cache entries */
// _CHECK_PEP_523
{
if (tstate->interp->eval_frame) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
}
- // _BINARY_SUBSCR_CHECK_FUNC
+ // _BINARY_OP_SUBSCR_CHECK_FUNC
{
container = stack_pointer[-2];
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem);
if (getitem_o == NULL) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
assert(PyFunction_Check(getitem_o));
uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version);
if (((PyFunctionObject *)getitem_o)->func_version != cached_version) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o);
assert(code->co_argcount == 2);
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
getitem = PyStackRef_FromPyObjectNew(getitem_o);
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
}
- // _BINARY_SUBSCR_INIT_CALL
+ // _BINARY_OP_SUBSCR_INIT_CALL
{
sub = stack_pointer[-1];
new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame);
new_frame->localsplus[0] = container;
new_frame->localsplus[1] = sub;
- frame->return_offset = 2 ;
+ frame->return_offset = 6 ;
}
// _PUSH_FRAME
{
@@ -832,40 +612,40 @@
DISPATCH();
}
- TARGET(BINARY_SUBSCR_LIST_INT) {
+ TARGET(BINARY_OP_SUBSCR_LIST_INT) {
#if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR_LIST_INT;
+ int opcode = BINARY_OP_SUBSCR_LIST_INT;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef list_st;
_PyStackRef sub_st;
_PyStackRef res;
- /* Skip 1 cache entry */
+ /* Skip 5 cache entries */
sub_st = stack_pointer[-1];
list_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);
if (!PyLong_CheckExact(sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyList_CheckExact(list)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyList_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
@@ -873,18 +653,18 @@
PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
stack_pointer = _PyFrame_GetStackPointer(frame);
if (res_o == NULL) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
#else
if (index >= PyList_GET_SIZE(list)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyList_GET_ITEM(list, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -902,54 +682,54 @@
DISPATCH();
}
- TARGET(BINARY_SUBSCR_STR_INT) {
+ TARGET(BINARY_OP_SUBSCR_STR_INT) {
#if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR_STR_INT;
+ int opcode = BINARY_OP_SUBSCR_STR_INT;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef str_st;
_PyStackRef sub_st;
_PyStackRef res;
- /* Skip 1 cache entry */
+ /* Skip 5 cache entries */
sub_st = stack_pointer[-1];
str_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);
if (!PyLong_CheckExact(sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyUnicode_CheckExact(str)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (PyUnicode_GET_LENGTH(str) <= index) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
// Specialize for reading an ASCII character from any string:
Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
stack_pointer += -2;
@@ -964,48 +744,48 @@
DISPATCH();
}
- TARGET(BINARY_SUBSCR_TUPLE_INT) {
+ TARGET(BINARY_OP_SUBSCR_TUPLE_INT) {
#if defined(Py_TAIL_CALL_INTERP)
- int opcode = BINARY_SUBSCR_TUPLE_INT;
+ int opcode = BINARY_OP_SUBSCR_TUPLE_INT;
(void)(opcode);
#endif
_Py_CODEUNIT* const this_instr = next_instr;
(void)this_instr;
frame->instr_ptr = next_instr;
- next_instr += 2;
- INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT);
- static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
_PyStackRef tuple_st;
_PyStackRef sub_st;
_PyStackRef res;
- /* Skip 1 cache entry */
+ /* Skip 5 cache entries */
sub_st = stack_pointer[-1];
tuple_st = stack_pointer[-2];
PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);
if (!PyLong_CheckExact(sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
if (!PyTuple_CheckExact(tuple)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
// Deopt unless 0 <= sub < PyTuple_Size(list)
if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
if (index >= PyTuple_GET_SIZE(tuple)) {
- UPDATE_MISS_STATS(BINARY_SUBSCR);
- assert(_PyOpcode_Deopt[opcode] == (BINARY_SUBSCR));
- JUMP_TO_PREDICTED(BINARY_SUBSCR);
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
}
- STAT_INC(BINARY_SUBSCR, hit);
+ STAT_INC(BINARY_OP, hit);
PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
assert(res_o != NULL);
Py_INCREF(res_o);
@@ -1022,6 +802,173 @@
DISPATCH();
}
+ TARGET(BINARY_OP_SUBTRACT_FLOAT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode = BINARY_OP_SUBTRACT_FLOAT;
+ (void)(opcode);
+ #endif
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
+ frame->instr_ptr = next_instr;
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
+ _PyStackRef left;
+ _PyStackRef right;
+ _PyStackRef res;
+ // _GUARD_BOTH_FLOAT
+ {
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyFloat_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyFloat_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ }
+ /* Skip 5 cache entries */
+ // _BINARY_OP_SUBTRACT_FLOAT
+ {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyFloat_CheckExact(left_o));
+ assert(PyFloat_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ double dres =
+ ((PyFloatObject *)left_o)->ob_fval -
+ ((PyFloatObject *)right_o)->ob_fval;
+ PyObject *res_o = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
+ if (res_o == NULL) {
+ JUMP_TO_LABEL(pop_2_error);
+ }
+ res = PyStackRef_FromPyObjectSteal(res_o);
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ DISPATCH();
+ }
+
+ TARGET(BINARY_OP_SUBTRACT_INT) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode = BINARY_OP_SUBTRACT_INT;
+ (void)(opcode);
+ #endif
+ _Py_CODEUNIT* const this_instr = next_instr;
+ (void)this_instr;
+ frame->instr_ptr = next_instr;
+ next_instr += 6;
+ INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT);
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size");
+ _PyStackRef left;
+ _PyStackRef right;
+ _PyStackRef res;
+ // _GUARD_BOTH_INT
+ {
+ right = stack_pointer[-1];
+ left = stack_pointer[-2];
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ if (!PyLong_CheckExact(left_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ if (!PyLong_CheckExact(right_o)) {
+ UPDATE_MISS_STATS(BINARY_OP);
+ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP));
+ JUMP_TO_PREDICTED(BINARY_OP);
+ }
+ }
+ /* Skip 5 cache entries */
+ // _BINARY_OP_SUBTRACT_INT
+ {
+ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
+ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
+ assert(PyLong_CheckExact(left_o));
+ assert(PyLong_CheckExact(right_o));
+ STAT_INC(BINARY_OP, hit);
+ PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
+ PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
+ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
+ if (res_o == NULL) {
+ JUMP_TO_LABEL(pop_2_error);
+ }
+ res = PyStackRef_FromPyObjectSteal(res_o);
+ }
+ stack_pointer[-2] = res;
+ stack_pointer += -1;
+ assert(WITHIN_STACK_BOUNDS());
+ DISPATCH();
+ }
+
+ TARGET(BINARY_SLICE) {
+ #if defined(Py_TAIL_CALL_INTERP)
+ int opcode = BINARY_SLICE;
+ (void)(opcode);
+ #endif
+ frame->instr_ptr = next_instr;
+ next_instr += 1;
+ INSTRUCTION_STATS(BINARY_SLICE);
+ _PyStackRef container;
+ _PyStackRef start;
+ _PyStackRef stop;
+ _PyStackRef res;
+ // _SPECIALIZE_BINARY_SLICE
+ {
+ // Placeholder until we implement BINARY_SLICE specialization
+ #if ENABLE_SPECIALIZATION
+ OPCODE_DEFERRED_INC(BINARY_SLICE);
+ #endif /* ENABLE_SPECIALIZATION */
+ }
+ // _BINARY_SLICE
+ {
+ stop = stack_pointer[-1];
+ start = stack_pointer[-2];
+ container = stack_pointer[-3];
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
+ PyStackRef_AsPyObjectSteal(stop));
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ PyObject *res_o;
+ // Can't use ERROR_IF() here, because we haven't
+ // DECREF'ed container yet, and we still own slice.
+ if (slice == NULL) {
+ res_o = NULL;
+ }
+ else {
+ stack_pointer += -2;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
+ Py_DECREF(slice);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ stack_pointer += 2;
+ assert(WITHIN_STACK_BOUNDS());
+ }
+ stack_pointer += -3;
+ assert(WITHIN_STACK_BOUNDS());
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ PyStackRef_CLOSE(container);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (res_o == NULL) {
+ JUMP_TO_LABEL(error);
+ }
+ res = PyStackRef_FromPyObjectSteal(res_o);
+ }
+ stack_pointer[0] = res;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ DISPATCH();
+ }
+
TARGET(BUILD_LIST) {
#if defined(Py_TAIL_CALL_INTERP)
int opcode = BUILD_LIST;
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 27c4d53..039a6ee 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -2,9 +2,8 @@
static void *opcode_targets[256] = {
&&TARGET_CACHE,
&&TARGET_BINARY_SLICE,
- &&TARGET_BINARY_SUBSCR,
- &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
&&TARGET_CALL_FUNCTION_EX,
+ &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
&&TARGET_CHECK_EG_MATCH,
&&TARGET_CHECK_EXC_MATCH,
&&TARGET_CLEANUP_THROW,
@@ -17,8 +16,8 @@ static void *opcode_targets[256] = {
&&TARGET_FORMAT_WITH_SPEC,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
- &&TARGET_RESERVED,
&&TARGET_GET_ITER,
+ &&TARGET_RESERVED,
&&TARGET_GET_LEN,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_INTERPRETER_EXIT,
@@ -149,6 +148,7 @@ static void *opcode_targets[256] = {
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,
+ &&_unknown_opcode,
&&TARGET_RESUME,
&&TARGET_BINARY_OP_ADD_FLOAT,
&&TARGET_BINARY_OP_ADD_INT,
@@ -156,13 +156,13 @@ static void *opcode_targets[256] = {
&&TARGET_BINARY_OP_EXTEND,
&&TARGET_BINARY_OP_MULTIPLY_FLOAT,
&&TARGET_BINARY_OP_MULTIPLY_INT,
+ &&TARGET_BINARY_OP_SUBSCR_DICT,
+ &&TARGET_BINARY_OP_SUBSCR_GETITEM,
+ &&TARGET_BINARY_OP_SUBSCR_LIST_INT,
+ &&TARGET_BINARY_OP_SUBSCR_STR_INT,
+ &&TARGET_BINARY_OP_SUBSCR_TUPLE_INT,
&&TARGET_BINARY_OP_SUBTRACT_FLOAT,
&&TARGET_BINARY_OP_SUBTRACT_INT,
- &&TARGET_BINARY_SUBSCR_DICT,
- &&TARGET_BINARY_SUBSCR_GETITEM,
- &&TARGET_BINARY_SUBSCR_LIST_INT,
- &&TARGET_BINARY_SUBSCR_STR_INT,
- &&TARGET_BINARY_SUBSCR_TUPLE_INT,
&&TARGET_CALL_ALLOC_AND_ENTER_INIT,
&&TARGET_CALL_BOUND_METHOD_EXACT_ARGS,
&&TARGET_CALL_BOUND_METHOD_GENERAL,
@@ -277,15 +277,14 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAM
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS);
+Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_DICT(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_GETITEM(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_LIST_INT(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_STR_INT(TAIL_CALL_PARAMS);
-Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS);
Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS);
@@ -511,15 +510,14 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {
[BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE,
[BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT,
[BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT,
+ [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_BINARY_OP_SUBSCR_DICT,
+ [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM,
+ [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT,
+ [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT,
+ [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT,
[BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT,
[BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT,
[BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE,
- [BINARY_SUBSCR] = _TAIL_CALL_BINARY_SUBSCR,
- [BINARY_SUBSCR_DICT] = _TAIL_CALL_BINARY_SUBSCR_DICT,
- [BINARY_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_SUBSCR_GETITEM,
- [BINARY_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_SUBSCR_LIST_INT,
- [BINARY_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_SUBSCR_STR_INT,
- [BINARY_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_SUBSCR_TUPLE_INT,
[BUILD_LIST] = _TAIL_CALL_BUILD_LIST,
[BUILD_MAP] = _TAIL_CALL_BUILD_MAP,
[BUILD_SET] = _TAIL_CALL_BUILD_SET,
@@ -725,6 +723,7 @@ static py_tail_call_funcptr INSTRUCTION_TABLE[256] = {
[UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE,
[WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START,
[YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE,
+ [117] = _TAIL_CALL_UNKNOWN_OPCODE,
[118] = _TAIL_CALL_UNKNOWN_OPCODE,
[119] = _TAIL_CALL_UNKNOWN_OPCODE,
[120] = _TAIL_CALL_UNKNOWN_OPCODE,
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 340770a..bef5728 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -753,7 +753,7 @@ translate_bytecode_to_trace(
assert(i + 1 == nuops);
if (opcode == FOR_ITER_GEN ||
opcode == LOAD_ATTR_PROPERTY ||
- opcode == BINARY_SUBSCR_GETITEM ||
+ opcode == BINARY_OP_SUBSCR_GETITEM ||
opcode == SEND_GEN)
{
DPRINTF(2, "Bailing due to dynamic target\n");
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 09f1915..41eb59c 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -384,7 +384,7 @@ dummy_func(void) {
GETLOCAL(this_instr->operand0) = res;
}
- op(_BINARY_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) {
+ op(_BINARY_OP_SUBSCR_INIT_CALL, (container, sub, getitem -- new_frame: _Py_UOpsAbstractFrame *)) {
new_frame = NULL;
ctx->done = true;
}
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 2383be8..fd84867 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -575,15 +575,6 @@
break;
}
- case _BINARY_SUBSCR: {
- JitOptSymbol *res;
- res = sym_new_not_null(ctx);
- stack_pointer[-2] = res;
- stack_pointer += -1;
- assert(WITHIN_STACK_BOUNDS());
- break;
- }
-
case _BINARY_SLICE: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
@@ -599,7 +590,7 @@
break;
}
- case _BINARY_SUBSCR_LIST_INT: {
+ case _BINARY_OP_SUBSCR_LIST_INT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
stack_pointer[-2] = res;
@@ -608,7 +599,7 @@
break;
}
- case _BINARY_SUBSCR_STR_INT: {
+ case _BINARY_OP_SUBSCR_STR_INT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
stack_pointer[-2] = res;
@@ -617,7 +608,7 @@
break;
}
- case _BINARY_SUBSCR_TUPLE_INT: {
+ case _BINARY_OP_SUBSCR_TUPLE_INT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
stack_pointer[-2] = res;
@@ -626,7 +617,7 @@
break;
}
- case _BINARY_SUBSCR_DICT: {
+ case _BINARY_OP_SUBSCR_DICT: {
JitOptSymbol *res;
res = sym_new_not_null(ctx);
stack_pointer[-2] = res;
@@ -635,7 +626,7 @@
break;
}
- case _BINARY_SUBSCR_CHECK_FUNC: {
+ case _BINARY_OP_SUBSCR_CHECK_FUNC: {
JitOptSymbol *getitem;
getitem = sym_new_not_null(ctx);
stack_pointer[0] = getitem;
@@ -644,7 +635,7 @@
break;
}
- case _BINARY_SUBSCR_INIT_CALL: {
+ case _BINARY_OP_SUBSCR_INIT_CALL: {
_Py_UOpsAbstractFrame *new_frame;
new_frame = NULL;
ctx->done = true;
diff --git a/Python/specialize.c b/Python/specialize.c
index 4f84b29..c741c4f 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -113,7 +113,6 @@ _Py_GetSpecializationStats(void) {
err += add_stat_dict(stats, LOAD_SUPER_ATTR, "load_super_attr");
err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
- err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
err += add_stat_dict(stats, CALL, "call");
@@ -553,11 +552,8 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters
#define SPEC_FAIL_SUBSCR_ARRAY_INT 9
#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10
#define SPEC_FAIL_SUBSCR_LIST_SLICE 11
-#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12
-#define SPEC_FAIL_SUBSCR_STRING_SLICE 14
-#define SPEC_FAIL_SUBSCR_BUFFER_INT 15
-#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
-#define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17
+#define SPEC_FAIL_SUBSCR_BUFFER_INT 12
+#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 13
/* Store subscr */
#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
@@ -593,6 +589,11 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters
#define SPEC_FAIL_BINARY_OP_OR_DIFFERENT_TYPES 30
#define SPEC_FAIL_BINARY_OP_XOR_INT 31
#define SPEC_FAIL_BINARY_OP_XOR_DIFFERENT_TYPES 32
+#define SPEC_FAIL_BINARY_OP_SUBSCR 33
+#define SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE 34
+#define SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE 35
+#define SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE 36
+#define SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE 37
/* Calls */
@@ -1761,37 +1762,6 @@ _Py_Specialize_LoadGlobal(
Py_END_CRITICAL_SECTION2();
}
-#ifdef Py_STATS
-static int
-binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
-{
- if (strcmp(container_type->tp_name, "array.array") == 0) {
- if (PyLong_CheckExact(sub)) {
- return SPEC_FAIL_SUBSCR_ARRAY_INT;
- }
- if (PySlice_Check(sub)) {
- return SPEC_FAIL_SUBSCR_ARRAY_SLICE;
- }
- return SPEC_FAIL_OTHER;
- }
- else if (container_type->tp_as_buffer) {
- if (PyLong_CheckExact(sub)) {
- return SPEC_FAIL_SUBSCR_BUFFER_INT;
- }
- if (PySlice_Check(sub)) {
- return SPEC_FAIL_SUBSCR_BUFFER_SLICE;
- }
- return SPEC_FAIL_OTHER;
- }
- else if (container_type->tp_as_sequence) {
- if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) {
- return SPEC_FAIL_SUBSCR_SEQUENCE_INT;
- }
- }
- return SPEC_FAIL_OTHER;
-}
-#endif // Py_STATS
-
static int
function_kind(PyCodeObject *code) {
int flags = code->co_flags;
@@ -1837,107 +1807,6 @@ function_get_version(PyObject *o, int opcode)
return version;
}
-void
-_Py_Specialize_BinarySubscr(
- _PyStackRef container_st, _PyStackRef sub_st, _Py_CODEUNIT *instr)
-{
- PyObject *container = PyStackRef_AsPyObjectBorrow(container_st);
- PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
-
- assert(ENABLE_SPECIALIZATION_FT);
- assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
- INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
- PyTypeObject *container_type = Py_TYPE(container);
- uint8_t specialized_op;
- if (container_type == &PyList_Type) {
- if (PyLong_CheckExact(sub)) {
- if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- specialized_op = BINARY_SUBSCR_LIST_INT;
- goto success;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
- goto fail;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR,
- PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER);
- goto fail;
- }
- if (container_type == &PyTuple_Type) {
- if (PyLong_CheckExact(sub)) {
- if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- specialized_op = BINARY_SUBSCR_TUPLE_INT;
- goto success;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
- goto fail;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR,
- PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER);
- goto fail;
- }
- if (container_type == &PyUnicode_Type) {
- if (PyLong_CheckExact(sub)) {
- if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) {
- specialized_op = BINARY_SUBSCR_STR_INT;
- goto success;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
- goto fail;
- }
- SPECIALIZATION_FAIL(BINARY_SUBSCR,
- PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_STRING_SLICE : SPEC_FAIL_OTHER);
- goto fail;
- }
- if (container_type == &PyDict_Type) {
- specialized_op = BINARY_SUBSCR_DICT;
- goto success;
- }
- unsigned int tp_version;
- PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version);
- if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
- if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE);
- Py_DECREF(descriptor);
- goto fail;
- }
- PyFunctionObject *func = (PyFunctionObject *)descriptor;
- PyCodeObject *fcode = (PyCodeObject *)func->func_code;
- int kind = function_kind(fcode);
- if (kind != SIMPLE_FUNCTION) {
- SPECIALIZATION_FAIL(BINARY_SUBSCR, kind);
- Py_DECREF(descriptor);
- goto fail;
- }
- if (fcode->co_argcount != 2) {
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
- Py_DECREF(descriptor);
- goto fail;
- }
-
- PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
- /* Don't specialize if PEP 523 is active */
- if (_PyInterpreterState_GET()->eval_frame) {
- SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER);
- Py_DECREF(descriptor);
- goto fail;
- }
- if (_PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version)) {
- specialized_op = BINARY_SUBSCR_GETITEM;
- Py_DECREF(descriptor);
- goto success;
- }
- }
- Py_XDECREF(descriptor);
- SPECIALIZATION_FAIL(BINARY_SUBSCR,
- binary_subscr_fail_kind(container_type, sub));
-fail:
- unspecialize(instr);
- return;
-success:
- specialize(instr, specialized_op);
-}
-
-
#ifdef Py_STATS
static int
store_subscr_fail_kind(PyObject *container, PyObject *sub)
@@ -2431,6 +2300,59 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
return SPEC_FAIL_BINARY_OP_XOR_INT;
}
return SPEC_FAIL_BINARY_OP_XOR;
+ case NB_SUBSCR:
+ if (PyList_CheckExact(lhs)) {
+ if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) {
+ return SPEC_FAIL_OUT_OF_RANGE;
+ }
+ if (PySlice_Check(rhs)) {
+ return SPEC_FAIL_BINARY_OP_SUBSCR_LIST_SLICE;
+ }
+ }
+ if (PyTuple_CheckExact(lhs)) {
+ if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) {
+ return SPEC_FAIL_OUT_OF_RANGE;
+ }
+ if (PySlice_Check(rhs)) {
+ return SPEC_FAIL_BINARY_OP_SUBSCR_TUPLE_SLICE;
+ }
+ }
+ if (PyUnicode_CheckExact(lhs)) {
+ if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) {
+ return SPEC_FAIL_OUT_OF_RANGE;
+ }
+ if (PySlice_Check(rhs)) {
+ return SPEC_FAIL_BINARY_OP_SUBSCR_STRING_SLICE;
+ }
+ }
+ unsigned int tp_version;
+ PyTypeObject *container_type = Py_TYPE(lhs);
+ PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version);
+ if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
+ if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+ Py_DECREF(descriptor);
+ return SPEC_FAIL_BINARY_OP_SUBSCR_NOT_HEAP_TYPE;
+ }
+ PyFunctionObject *func = (PyFunctionObject *)descriptor;
+ PyCodeObject *fcode = (PyCodeObject *)func->func_code;
+ int kind = function_kind(fcode);
+ if (kind != SIMPLE_FUNCTION) {
+ Py_DECREF(descriptor);
+ return kind;
+ }
+ if (fcode->co_argcount != 2) {
+ Py_DECREF(descriptor);
+ return SPEC_FAIL_WRONG_NUMBER_ARGUMENTS;
+ }
+
+ if (_PyInterpreterState_GET()->eval_frame) {
+ /* Don't specialize if PEP 523 is active */
+ Py_DECREF(descriptor);
+ return SPEC_FAIL_OTHER;
+ }
+ }
+ Py_XDECREF(descriptor);
+ return SPEC_FAIL_BINARY_OP_SUBSCR;
}
Py_UNREACHABLE();
}
@@ -2536,45 +2458,40 @@ LONG_FLOAT_ACTION(compactlong_float_multiply, *)
LONG_FLOAT_ACTION(compactlong_float_true_div, /)
#undef LONG_FLOAT_ACTION
-static _PyBinaryOpSpecializationDescr compactlongs_specs[NB_OPARG_LAST+1] = {
- [NB_OR] = {compactlongs_guard, compactlongs_or},
- [NB_AND] = {compactlongs_guard, compactlongs_and},
- [NB_XOR] = {compactlongs_guard, compactlongs_xor},
- [NB_INPLACE_OR] = {compactlongs_guard, compactlongs_or},
- [NB_INPLACE_AND] = {compactlongs_guard, compactlongs_and},
- [NB_INPLACE_XOR] = {compactlongs_guard, compactlongs_xor},
-};
-
-static _PyBinaryOpSpecializationDescr float_compactlong_specs[NB_OPARG_LAST+1] = {
- [NB_ADD] = {float_compactlong_guard, float_compactlong_add},
- [NB_SUBTRACT] = {float_compactlong_guard, float_compactlong_subtract},
- [NB_TRUE_DIVIDE] = {nonzero_float_compactlong_guard, float_compactlong_true_div},
- [NB_MULTIPLY] = {float_compactlong_guard, float_compactlong_multiply},
-};
-
-static _PyBinaryOpSpecializationDescr compactlong_float_specs[NB_OPARG_LAST+1] = {
- [NB_ADD] = {compactlong_float_guard, compactlong_float_add},
- [NB_SUBTRACT] = {compactlong_float_guard, compactlong_float_subtract},
- [NB_TRUE_DIVIDE] = {nonzero_compactlong_float_guard, compactlong_float_true_div},
- [NB_MULTIPLY] = {compactlong_float_guard, compactlong_float_multiply},
+static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = {
+ /* long-long arithmetic */
+ {NB_OR, compactlongs_guard, compactlongs_or},
+ {NB_AND, compactlongs_guard, compactlongs_and},
+ {NB_XOR, compactlongs_guard, compactlongs_xor},
+ {NB_INPLACE_OR, compactlongs_guard, compactlongs_or},
+ {NB_INPLACE_AND, compactlongs_guard, compactlongs_and},
+ {NB_INPLACE_XOR, compactlongs_guard, compactlongs_xor},
+
+ /* float-long arithemetic */
+ {NB_ADD, float_compactlong_guard, float_compactlong_add},
+ {NB_SUBTRACT, float_compactlong_guard, float_compactlong_subtract},
+ {NB_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div},
+ {NB_MULTIPLY, float_compactlong_guard, float_compactlong_multiply},
+
+ /* float-float arithmetic */
+ {NB_ADD, compactlong_float_guard, compactlong_float_add},
+ {NB_SUBTRACT, compactlong_float_guard, compactlong_float_subtract},
+ {NB_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div},
+ {NB_MULTIPLY, compactlong_float_guard, compactlong_float_multiply},
};
static int
binary_op_extended_specialization(PyObject *lhs, PyObject *rhs, int oparg,
_PyBinaryOpSpecializationDescr **descr)
{
-#define LOOKUP_SPEC(TABLE, OPARG) \
- if ((TABLE)[(OPARG)].action) { \
- if ((TABLE)[(OPARG)].guard(lhs, rhs)) { \
- *descr = &((TABLE)[OPARG]); \
- return 1; \
- } \
+ size_t n = sizeof(binaryop_extend_descrs)/sizeof(_PyBinaryOpSpecializationDescr);
+ for (size_t i = 0; i < n; i++) {
+ _PyBinaryOpSpecializationDescr *d = &binaryop_extend_descrs[i];
+ if (d->oparg == oparg && d->guard(lhs, rhs)) {
+ *descr = d;
+ return 1;
+ }
}
-
- LOOKUP_SPEC(compactlong_float_specs, oparg);
- LOOKUP_SPEC(float_compactlong_specs, oparg);
- LOOKUP_SPEC(compactlongs_specs, oparg);
-#undef LOOKUP_SPEC
return 0;
}
@@ -2645,6 +2562,47 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in
return;
}
break;
+ case NB_SUBSCR:
+ if (PyLong_CheckExact(rhs) && _PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) {
+ if (PyList_CheckExact(lhs)) {
+ specialize(instr, BINARY_OP_SUBSCR_LIST_INT);
+ return;
+ }
+ if (PyTuple_CheckExact(lhs)) {
+ specialize(instr, BINARY_OP_SUBSCR_TUPLE_INT);
+ return;
+ }
+ if (PyUnicode_CheckExact(lhs)) {
+ specialize(instr, BINARY_OP_SUBSCR_STR_INT);
+ return;
+ }
+ }
+ if (PyDict_CheckExact(lhs)) {
+ specialize(instr, BINARY_OP_SUBSCR_DICT);
+ return;
+ }
+ unsigned int tp_version;
+ PyTypeObject *container_type = Py_TYPE(lhs);
+ PyObject *descriptor = _PyType_LookupRefAndVersion(container_type, &_Py_ID(__getitem__), &tp_version);
+ if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type &&
+ container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)
+ {
+ PyFunctionObject *func = (PyFunctionObject *)descriptor;
+ PyCodeObject *fcode = (PyCodeObject *)func->func_code;
+ int kind = function_kind(fcode);
+ PyHeapTypeObject *ht = (PyHeapTypeObject *)container_type;
+ if (kind == SIMPLE_FUNCTION &&
+ fcode->co_argcount == 2 &&
+ !_PyInterpreterState_GET()->eval_frame && /* Don't specialize if PEP 523 is active */
+ _PyType_CacheGetItemForSpecialization(ht, descriptor, (uint32_t)tp_version))
+ {
+ specialize(instr, BINARY_OP_SUBSCR_GETITEM);
+ Py_DECREF(descriptor);
+ return;
+ }
+ }
+ Py_XDECREF(descriptor);
+ break;
}
_PyBinaryOpSpecializationDescr *descr;
generated by cgit v0.12 at 2025-09-02 23:25:17 (GMT)