diff options
| author | Mark Shannon <mark@hotpy.org> | 2023-12-07 12:49:40 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-07 12:49:40 (GMT) |
| commit | b449415b2f1b41e1c44cb453428657fdf6ff1d36 (patch) | |
| tree | a31422e8d6a348baf55421d40dce503d1265b8d5 /Python/generated_cases.c.h | |
| parent | 3d712a9f4c9f366edbe16b804ec4f6ae50b0a59f (diff) | |
| download | cpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.zip cpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.tar.gz cpython-b449415b2f1b41e1c44cb453428657fdf6ff1d36.tar.bz2 | |
GH-111485: Separate out parsing, analysis and code-gen phases of tier 1 code generator (GH-112299)
Diffstat (limited to 'Python/generated_cases.c.h')
| -rw-r--r-- | Python/generated_cases.c.h | 1047 |
1 files changed, 536 insertions, 511 deletions
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0ac99e7..24243ecf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1,6 +1,6 @@ -// This file is generated by Tools/cases_generator/generate_cases.py +// This file is generated by Tools/cases_generator/tier1_generator.py // from: -// Python/bytecodes.c +// ['./Python/bytecodes.c'] // Do not edit! #ifdef TIER_TWO @@ -8,6 +8,7 @@ #endif #define TIER_ONE 1 + TARGET(BEFORE_ASYNC_WITH) { frame->instr_ptr = next_instr; next_instr += 1; @@ -45,9 +46,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -91,9 +92,9 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - STACK_GROW(1); - stack_pointer[-2] = exit; - stack_pointer[-1] = res; + stack_pointer[-1] = exit; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -103,7 +104,6 @@ INSTRUCTION_STATS(BINARY_OP); PREDICTED(BINARY_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -133,8 +133,8 @@ Py_DECREF(rhs); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -142,6 +142,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -156,12 +157,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval + - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval + + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -169,6 +170,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -187,8 +189,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -196,6 +198,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -214,8 +217,8 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -223,6 +226,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -258,7 +262,7 @@ assert(next_instr->op.code == STORE_FAST); SKIP_OVER(1); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -266,6 +270,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -280,12 +285,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval * - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval * + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -293,6 +298,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -311,8 +317,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -320,6 +326,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -334,12 +341,12 @@ { STAT_INC(BINARY_OP, hit); double dres = - ((PyFloatObject *)left)->ob_fval - - ((PyFloatObject *)right)->ob_fval; + ((PyFloatObject *)left)->ob_fval - + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -347,6 +354,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -365,8 +373,8 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -393,8 +401,8 @@ } Py_DECREF(container); if (res == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = res; + stack_pointer[-3] = res; + stack_pointer += -2; DISPATCH(); } @@ -404,7 +412,6 @@ INSTRUCTION_STATS(BINARY_SUBSCR); PREDICTED(BINARY_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -431,8 +438,8 @@ Py_DECREF(sub); if (res == NULL) goto pop_2_error; } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -440,6 +447,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *res; @@ -454,8 +462,9 @@ Py_DECREF(dict); Py_DECREF(sub); if (rc <= 0) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -463,6 +472,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -494,6 +504,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *res; @@ -501,7 +512,6 @@ list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -512,8 +522,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -521,6 +531,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *str; PyObject *res; @@ -538,8 +549,8 @@ res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(str); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -547,6 +558,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(BINARY_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *tuple; PyObject *res; @@ -554,7 +566,6 @@ tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); - // Deopt unless 0 <= sub < PyTuple_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -565,8 +576,8 @@ Py_INCREF(res); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -578,7 +589,7 @@ PyObject **values; PyObject *map; keys = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -586,15 +597,15 @@ GOTO_ERROR(error); // Pop the keys and values. } map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(values[_i]); } Py_DECREF(keys); - if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } - STACK_SHRINK(oparg); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -1 - oparg; goto error; } + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; DISPATCH(); } @@ -604,12 +615,11 @@ INSTRUCTION_STATS(BUILD_LIST); PyObject **values; PyObject *list; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; list = _PyList_FromArraySteal(values, oparg); - if (list == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = list; + if (list == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -619,18 +629,17 @@ INSTRUCTION_STATS(BUILD_MAP); PyObject **values; PyObject *map; - values = stack_pointer - oparg*2; + values = &stack_pointer[-oparg*2]; map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); + values, 2, + values+1, 2, + oparg); for (int _i = oparg*2; --_i >= 0;) { Py_DECREF(values[_i]); } - if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } - STACK_SHRINK(oparg*2); - STACK_GROW(1); - stack_pointer[-1] = map; + if (map == NULL) { stack_pointer += -oparg*2; goto error; } + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; DISPATCH(); } @@ -640,24 +649,23 @@ INSTRUCTION_STATS(BUILD_SET); PyObject **values; PyObject *set; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; set = PySet_New(NULL); if (set == NULL) - GOTO_ERROR(error); + GOTO_ERROR(error); int err = 0; for (int i = 0; i < oparg; i++) { PyObject *item = values[i]; if (err == 0) - err = PySet_Add(set, item); + err = PySet_Add(set, item); Py_DECREF(item); } if (err != 0) { Py_DECREF(set); - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = set; + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -669,17 +677,16 @@ PyObject *stop; PyObject *start; PyObject *slice; - if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } - stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; - start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; + if (oparg == 3) { step = stack_pointer[-(((oparg == 3) ? 1 : 0))]; } + stop = stack_pointer[-1 - (((oparg == 3) ? 1 : 0))]; + start = stack_pointer[-2 - (((oparg == 3) ? 1 : 0))]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - STACK_SHRINK(((oparg == 3) ? 1 : 0)); - STACK_SHRINK(1); - stack_pointer[-1] = slice; + if (slice == NULL) { stack_pointer += -2 - (((oparg == 3) ? 1 : 0)); goto error; } + stack_pointer[-2 - (((oparg == 3) ? 1 : 0))] = slice; + stack_pointer += -1 - (((oparg == 3) ? 1 : 0)); DISPATCH(); } @@ -689,15 +696,14 @@ INSTRUCTION_STATS(BUILD_STRING); PyObject **pieces; PyObject *str; - pieces = stack_pointer - oparg; + pieces = &stack_pointer[-oparg]; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); } - if (str == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = str; + if (str == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -707,12 +713,11 @@ INSTRUCTION_STATS(BUILD_TUPLE); PyObject **values; PyObject *tup; - values = stack_pointer - oparg; + values = &stack_pointer[-oparg]; tup = _PyTuple_FromArraySteal(values, oparg); - if (tup == NULL) { STACK_SHRINK(oparg); goto error; } - STACK_SHRINK(oparg); - STACK_GROW(1); - stack_pointer[-1] = tup; + if (tup == NULL) { stack_pointer += -oparg; goto error; } + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; DISPATCH(); } @@ -730,13 +735,12 @@ INSTRUCTION_STATS(CALL); PREDICTED(CALL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; // _SPECIALIZE_CALL - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { @@ -793,12 +797,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -818,11 +822,10 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -831,10 +834,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* This instruction does the following: @@ -890,13 +894,15 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *null; PyObject *callable; + PyObject *func; PyObject *self; PyObject *self_or_null; - PyObject *func; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -936,7 +942,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -960,26 +967,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -987,11 +994,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1009,10 +1017,9 @@ Py_DECREF(args[i]); } Py_DECREF(tp); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1021,11 +1028,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ @@ -1044,21 +1052,19 @@ args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - /* Not deopting because this doesn't mean our optimization was - wrong. `res` can be NULL for valid reasons. Eg. getattr(x, - 'invalid'). In those cases an exception is set, so we must - handle it. - */ - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + /* Not deopting because this doesn't mean our optimization was + wrong. `res` can be NULL for valid reasons. Eg. getattr(x, + 'invalid'). In those cases an exception is set, so we must + handle it. + */ + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1067,11 +1073,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -1085,20 +1092,18 @@ STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1107,11 +1112,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ @@ -1134,13 +1140,11 @@ res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1155,9 +1159,9 @@ PyObject *callargs; PyObject *func; PyObject *result; - if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } - callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; + if (oparg & 1) { kwargs = stack_pointer[-((oparg & 1))]; } + callargs = stack_pointer[-1 - ((oparg & 1))]; + func = stack_pointer[-3 - ((oparg & 1))]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -1177,7 +1181,7 @@ !PyFunction_Check(func) && !PyMethod_Check(func) ) { PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : Py_None; + PyTuple_GET_ITEM(callargs, 0) : Py_None; int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); @@ -1205,10 +1209,9 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)func, locals, - nargs, callargs, kwargs); + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -1224,10 +1227,9 @@ Py_DECREF(callargs); Py_XDECREF(kwargs); assert(PEEK(2 + (oparg & 1)) == NULL); - if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - STACK_SHRINK(((oparg & 1) ? 1 : 0)); - STACK_SHRINK(2); - stack_pointer[-1] = result; + if (result == NULL) { stack_pointer += -3 - ((oparg & 1)); goto error; } + stack_pointer[-3 - ((oparg & 1))] = result; + stack_pointer += -2 - ((oparg & 1)); CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1261,8 +1263,8 @@ Py_DECREF(value2); Py_DECREF(value1); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1270,11 +1272,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* isinstance(o, o2) */ @@ -1295,14 +1298,12 @@ } res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1318,7 +1319,7 @@ PyObject *callable; PyObject *res; kwnames = stack_pointer[-1]; - args = stack_pointer - 1 - oparg; + args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; // oparg counts all of the args, but *not* self: @@ -1363,12 +1364,12 @@ } /* Callable is not a normal Python function */ res = PyObject_Vectorcall( - callable, args, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames); + callable, args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); if (opcode == INSTRUMENTED_CALL_KW) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -1389,10 +1390,9 @@ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_3_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(2); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -3 - oparg; goto error; } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1401,11 +1401,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; /* len(o) */ @@ -1425,13 +1426,11 @@ } res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1439,10 +1438,11 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1467,11 +1467,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1488,7 +1489,7 @@ DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = - (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -1497,10 +1498,9 @@ Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1509,11 +1509,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1531,19 +1532,17 @@ STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = - (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; res = cfunc(self, args + 1, nargs, NULL); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ for (int i = 0; i < total_args; i++) { Py_DECREF(args[i]); } Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1552,11 +1551,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 0 || oparg == 1); @@ -1584,10 +1584,9 @@ assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1596,11 +1595,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int total_args = oparg; @@ -1629,10 +1629,9 @@ Py_DECREF(self); Py_DECREF(arg); Py_DECREF(callable); - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1641,10 +1640,12 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject *self_or_null; PyObject *callable; PyObject **args; _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ // _CHECK_PEP_523 { DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -1668,7 +1669,8 @@ DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; { int argcount = oparg; if (self_or_null != NULL) { @@ -1692,26 +1694,26 @@ #endif } // _PUSH_FRAME - STACK_SHRINK(oparg); - STACK_SHRINK(2); { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); - STORE_SP(); + stack_pointer += -2 - oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = new_frame; tstate->py_recursion_remaining--; LOAD_SP(); LOAD_IP(0); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -1719,10 +1721,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); @@ -1763,11 +1766,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1778,10 +1782,9 @@ res = PyObject_Str(arg); Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1790,11 +1793,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1805,10 +1809,9 @@ res = PySequence_Tuple(arg); Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -1817,11 +1820,12 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; PyObject *null; PyObject *callable; PyObject *res; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; assert(oparg == 1); @@ -1832,9 +1836,8 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - STACK_SHRINK(oparg); - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; DISPATCH(); } @@ -1853,18 +1856,15 @@ Py_DECREF(match_type); if (true) goto pop_2_error; } - match = NULL; rest = NULL; int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); + &match, &rest); Py_DECREF(exc_value); Py_DECREF(match_type); if (res < 0) goto pop_2_error; - assert((match == NULL) == (rest == NULL)); if (match == NULL) goto pop_2_error; - if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } @@ -1884,10 +1884,9 @@ left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { - Py_DECREF(right); - if (true) goto pop_1_error; + Py_DECREF(right); + if (true) goto pop_1_error; } - int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = res ? Py_True : Py_False; @@ -1922,9 +1921,9 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(1); - stack_pointer[-2] = none; - stack_pointer[-1] = value; + stack_pointer[-3] = none; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -1934,7 +1933,6 @@ INSTRUCTION_STATS(COMPARE_OP); PREDICTED(COMPARE_OP); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1968,8 +1966,8 @@ res = res_bool ? Py_True : Py_False; } } - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -1977,6 +1975,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -1993,8 +1992,8 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2002,6 +2001,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2022,8 +2022,8 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2031,6 +2031,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2048,8 +2049,8 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; // It's always a bool, so we don't care about oparg & 16. - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2067,8 +2068,8 @@ Py_DECREF(right); if (res < 0) goto pop_2_error; b = (res ^ oparg) ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -2098,8 +2099,8 @@ bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); - STACK_GROW(1); - stack_pointer[-1] = top; + stack_pointer[0] = top; + stack_pointer += 1; DISPATCH(); } @@ -2130,7 +2131,7 @@ int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2172,7 +2173,7 @@ if (err != 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } GOTO_ERROR(error); } @@ -2195,8 +2196,8 @@ // Can't use ERROR_IF here. if (err != 0) { _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); + NAME_ERROR_MSG, + name); GOTO_ERROR(error); } DISPATCH(); @@ -2215,7 +2216,7 @@ Py_DECREF(container); Py_DECREF(sub); if (err) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2235,7 +2236,7 @@ if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2250,14 +2251,14 @@ if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); } Py_DECREF(update); if (true) goto pop_1_error; } Py_DECREF(update); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2281,7 +2282,7 @@ monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2290,17 +2291,17 @@ next_instr += 1; INSTRUCTION_STATS(END_FOR); PyObject *value; - // POP_TOP + // _POP_TOP value = stack_pointer[-1]; { Py_DECREF(value); } - // POP_TOP + // _POP_TOP value = stack_pointer[-2]; { Py_DECREF(value); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2313,8 +2314,8 @@ value = stack_pointer[-1]; receiver = stack_pointer[-2]; Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -2324,7 +2325,6 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); TIER_ONE_ONLY CHECK_EVAL_BREAKER(); - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = (_PyExecutorObject *)code->co_executors->executors[oparg&255]; int original_oparg = executor->vm_data.oparg | (oparg & 0xfffff00); @@ -2352,11 +2352,11 @@ assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(should_be_none)->tp_name); + "__init__() should return None, not '%.200s'", + Py_TYPE(should_be_none)->tp_name); GOTO_ERROR(error); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -2405,8 +2405,8 @@ Py_DECREF(value); Py_DECREF(fmt_spec); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2416,7 +2416,6 @@ INSTRUCTION_STATS(FOR_ITER); PREDICTED(FOR_ITER); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; // _SPECIALIZE_FOR_ITER @@ -2448,7 +2447,7 @@ } /* iterator ended normally */ assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); Py_DECREF(iter); STACK_SHRINK(1); /* Jump forward oparg, then skip following END_FOR instruction */ @@ -2457,8 +2456,8 @@ } // Common case: no jump, leave it to the code generator } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2466,6 +2465,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -2489,8 +2489,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_LIST iter = stack_pointer[-1]; { @@ -2523,8 +2525,8 @@ assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2532,8 +2534,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { @@ -2564,8 +2568,8 @@ next = PyLong_FromLong(value); if (next == NULL) goto error; } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2573,8 +2577,10 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { @@ -2607,8 +2613,8 @@ assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); } - STACK_GROW(1); - stack_pointer[-1] = next; + stack_pointer[0] = next; + stack_pointer += 1; DISPATCH(); } @@ -2621,11 +2627,9 @@ obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } - if (getter == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " @@ -2634,14 +2638,11 @@ Py_DECREF(obj); if (true) goto pop_1_error; } - iter = (*getter)(obj); Py_DECREF(obj); if (iter == NULL) goto pop_1_error; - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", @@ -2663,7 +2664,6 @@ unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { awaitable = type->tp_as_async->am_anext(aiter); if (awaitable == NULL) { @@ -2673,7 +2673,6 @@ if (type->tp_as_async != NULL){ getter = type->tp_as_async->am_anext; } - if (getter != NULL) { next_iter = (*getter)(aiter); if (next_iter == NULL) { @@ -2687,7 +2686,6 @@ type->tp_name); GOTO_ERROR(error); } - awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( @@ -2695,15 +2693,14 @@ "'async for' received an invalid object " "from __anext__: %.100s", Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); GOTO_ERROR(error); } else { Py_DECREF(next_iter); } } - STACK_GROW(1); - stack_pointer[-1] = awaitable; + stack_pointer[0] = awaitable; + stack_pointer += 1; DISPATCH(); } @@ -2715,13 +2712,10 @@ PyObject *iter; iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { PyObject *yf = _PyGen_yf((PyGenObject*)iter); if (yf != NULL) { @@ -2735,7 +2729,6 @@ /* The code below jumps to `error` if `iter` is NULL. */ } } - if (iter == NULL) goto pop_1_error; stack_pointer[-1] = iter; DISPATCH(); @@ -2768,8 +2761,8 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = len_o; + stack_pointer[0] = len_o; + stack_pointer += 1; DISPATCH(); } @@ -2819,8 +2812,8 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -2839,8 +2832,8 @@ Py_DECREF(level); Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; - STACK_SHRINK(1); - stack_pointer[-1] = res; + stack_pointer[-2] = res; + stack_pointer += -1; DISPATCH(); } @@ -2852,10 +2845,10 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); GO_TO_INSTRUCTION(CALL); @@ -2876,10 +2869,10 @@ int total_args = oparg + is_meth; PyObject *function = PEEK(oparg + 3); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING - : PEEK(total_args + 1); + : PEEK(total_args + 1); int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); if (err) goto error; GO_TO_INSTRUCTION(CALL_KW); } @@ -2904,7 +2897,7 @@ } Py_DECREF(receiver); Py_DECREF(value); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -2925,8 +2918,8 @@ PyErr_SetRaisedException(NULL); } Py_DECREF(receiver); - STACK_SHRINK(1); - stack_pointer[-1] = value; + stack_pointer[-2] = value; + stack_pointer += -1; DISPATCH(); } @@ -3094,7 +3087,7 @@ } _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); + tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) goto error; if (frame->instr_ptr != this_instr) { @@ -3112,8 +3105,8 @@ INSTRUCTION_STATS(INSTRUMENTED_RETURN_CONST); PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); Py_INCREF(retval); assert(EMPTY()); @@ -3136,8 +3129,8 @@ PyObject *retval; retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, retval); if (err) GOTO_ERROR(error); STACK_SHRINK(1); assert(EMPTY()); @@ -3167,8 +3160,8 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyFrame_SetStackPointer(frame, stack_pointer - 1); int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, retval); + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, retval); if (err) GOTO_ERROR(error); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -3211,8 +3204,8 @@ Py_DECREF(left); Py_DECREF(right); b = res ? Py_True : Py_False; - STACK_SHRINK(1); - stack_pointer[-1] = b; + stack_pointer[-2] = b; + stack_pointer += -1; DISPATCH(); } @@ -3286,7 +3279,7 @@ v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3301,19 +3294,19 @@ PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { _PyErr_Clear(tstate); _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); } Py_DECREF(iterable); if (true) goto pop_1_error; } assert(Py_IsNone(none_val)); Py_DECREF(iterable); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -3323,8 +3316,8 @@ INSTRUCTION_STATS(LOAD_ASSERTION_ERROR); PyObject *value; value = Py_NewRef(PyExc_AssertionError); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3334,7 +3327,6 @@ INSTRUCTION_STATS(LOAD_ATTR); PREDICTED(LOAD_ATTR); _Py_CODEUNIT *this_instr = next_instr - 10; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -3374,7 +3366,7 @@ the second element of the stack to NULL, to signal CALL that it's not a method call. NULL | meth | arg1 | ... | argN - */ + */ Py_DECREF(owner); if (attr == NULL) goto pop_1_error; self_or_null = NULL; @@ -3387,9 +3379,9 @@ if (attr == NULL) goto pop_1_error; } } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3397,9 +3389,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS owner = stack_pointer[-1]; { @@ -3408,6 +3402,7 @@ assert(type_version != 0); DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3417,9 +3412,9 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3427,6 +3422,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -3445,7 +3441,6 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); @@ -3461,9 +3456,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3490,9 +3487,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3500,9 +3498,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3519,6 +3519,7 @@ /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3529,9 +3530,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3539,9 +3540,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3550,6 +3553,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3561,9 +3565,9 @@ attr = Py_NewRef(descr); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3571,9 +3575,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; - PyObject *self; + PyObject *self = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3606,9 +3612,9 @@ assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); self = owner; } - STACK_GROW(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self; + stack_pointer[-1] = attr; + if (1) stack_pointer[0] = self; + stack_pointer += (((1) ? 1 : 0)); DISPATCH(); } @@ -3616,9 +3622,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _CHECK_ATTR_MODULE owner = stack_pointer[-1]; { @@ -3642,9 +3650,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3652,8 +3661,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3662,6 +3673,7 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); @@ -3673,6 +3685,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -3680,8 +3693,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3713,6 +3728,7 @@ attr = Py_NewRef(descr); } stack_pointer[-1] = attr; + stack_pointer += (((0) ? 1 : 0)); DISPATCH(); } @@ -3720,6 +3736,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); @@ -3727,7 +3744,6 @@ PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); - PyTypeObject *cls = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -3752,9 +3768,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3774,9 +3792,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3784,9 +3803,11 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -3827,9 +3848,10 @@ null = NULL; Py_DECREF(owner); } - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + /* Skip 5 cache entries */ + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = null; + stack_pointer += ((oparg & 1)); DISPATCH(); } @@ -3844,8 +3866,8 @@ "__build_class__ not found"); if (true) goto error; } - STACK_GROW(1); - stack_pointer[-1] = bc; + stack_pointer[0] = bc; + stack_pointer += 1; DISPATCH(); } @@ -3856,8 +3878,8 @@ PyObject *value; value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3873,8 +3895,8 @@ if (true) goto error; } Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3886,8 +3908,8 @@ value = GETLOCAL(oparg); assert(value != NULL); Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3899,8 +3921,8 @@ value = GETLOCAL(oparg); // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3912,8 +3934,8 @@ value = GETLOCAL(oparg); if (value == NULL) goto unbound_local_error; Py_INCREF(value); - STACK_GROW(1); - stack_pointer[-1] = value; + stack_pointer[0] = value; + stack_pointer += 1; DISPATCH(); } @@ -3929,9 +3951,9 @@ value2 = GETLOCAL(oparg2); Py_INCREF(value1); Py_INCREF(value2); - STACK_GROW(2); - stack_pointer[-2] = value1; - stack_pointer[-1] = value2; + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; DISPATCH(); } @@ -3984,8 +4006,8 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } @@ -4001,7 +4023,6 @@ INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED(LOAD_GLOBAL); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; // _SPECIALIZE_LOAD_GLOBAL @@ -4026,14 +4047,14 @@ && PyDict_CheckExact(BUILTINS())) { res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + NAME_ERROR_MSG, name); } if (true) goto error; } @@ -4048,18 +4069,17 @@ if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; if (res == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); if (true) goto error; } } } null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4067,8 +4087,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4096,10 +4118,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4107,8 +4128,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; + /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); @@ -4117,6 +4140,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + /* Skip 1 cache entry */ // _LOAD_GLOBAL_MODULE { uint16_t index = read_u16(&this_instr[4].cache); @@ -4128,10 +4152,9 @@ STAT_INC(LOAD_GLOBAL, hit); null = NULL; } - STACK_GROW(1); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + ((oparg & 1)); DISPATCH(); } @@ -4147,8 +4170,8 @@ if (true) goto error; } Py_INCREF(locals); - STACK_GROW(1); - stack_pointer[-1] = locals; + stack_pointer[0] = locals; + stack_pointer += 1; DISPATCH(); } @@ -4177,14 +4200,14 @@ } if (v == NULL) { _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); GOTO_ERROR(error); } } } - STACK_GROW(1); - stack_pointer[-1] = v; + stack_pointer[0] = v; + stack_pointer += 1; DISPATCH(); } @@ -4194,7 +4217,6 @@ INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *class; PyObject *global_super; PyObject *self; @@ -4224,8 +4246,8 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); if (err) goto pop_3_error; } // we make no attempt to optimize here; specializations should @@ -4258,10 +4280,9 @@ if (attr == NULL) goto pop_3_error; null = NULL; } - STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; - if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-3] = attr; + if (oparg & 1) stack_pointer[-2] = null; + stack_pointer += -2 + ((oparg & 1)); DISPATCH(); } @@ -4269,6 +4290,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -4286,8 +4308,8 @@ Py_DECREF(class); Py_DECREF(self); if (attr == NULL) goto pop_3_error; - STACK_SHRINK(2); - stack_pointer[-1] = attr; + stack_pointer[-3] = attr; + stack_pointer += -2 + (((0) ? 1 : 0)); DISPATCH(); } @@ -4295,6 +4317,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); PyObject *self; PyObject *class; PyObject *global_super; @@ -4324,9 +4347,9 @@ Py_DECREF(self); self_or_null = NULL; } - STACK_SHRINK(1); - stack_pointer[-2] = attr; - stack_pointer[-1] = self_or_null; + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; DISPATCH(); } @@ -4352,17 +4375,14 @@ PyObject *codeobj; PyObject *func; codeobj = stack_pointer[-1]; - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - + PyFunction_New(codeobj, GLOBALS()); Py_DECREF(codeobj); if (func_obj == NULL) { GOTO_ERROR(error); } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); @@ -4382,7 +4402,7 @@ /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error; - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -4409,10 +4429,11 @@ } else { if (_PyErr_Occurred(tstate)) goto pop_3_error; + // Error! attrs = Py_None; // Failure! } - STACK_SHRINK(2); - stack_pointer[-1] = attrs; + stack_pointer[-3] = attrs; + stack_pointer += -2; DISPATCH(); } @@ -4428,8 +4449,8 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = values_or_none; + stack_pointer[0] = values_or_none; + stack_pointer += 1; DISPATCH(); } @@ -4442,8 +4463,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4456,8 +4477,8 @@ subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4476,7 +4497,7 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4492,7 +4513,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4524,7 +4545,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4556,7 +4577,7 @@ #endif JUMPBY(oparg * flag); } - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4572,7 +4593,7 @@ this_instr[1].cache = (this_instr[1].cache << 1) | flag; #endif JUMPBY(oparg * flag); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4583,7 +4604,7 @@ PyObject *value; value = stack_pointer[-1]; Py_DECREF(value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4603,9 +4624,9 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - STACK_GROW(1); - stack_pointer[-2] = prev_exc; - stack_pointer[-1] = new_exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; DISPATCH(); } @@ -4615,8 +4636,8 @@ INSTRUCTION_STATS(PUSH_NULL); PyObject *res; res = NULL; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -4625,29 +4646,29 @@ next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); PyObject **args; - args = stack_pointer - oparg; + args = &stack_pointer[-oparg]; TIER_ONE_ONLY PyObject *cause = NULL, *exc = NULL; switch (oparg) { - case 2: + case 2: cause = args[1]; /* fall through */ - case 1: + case 1: exc = args[0]; /* fall through */ - case 0: + case 0: if (do_raise(tstate, exc, cause)) { assert(oparg == 0); monitor_reraise(tstate, frame, this_instr); goto exception_unwind; } break; - default: + default: _PyErr_SetString(tstate, PyExc_SystemError, "bad RAISE_VARARGS oparg"); break; } - if (true) { STACK_SHRINK(oparg); goto error; } + if (true) { stack_pointer += -oparg; goto error; } } TARGET(RERAISE) { @@ -4657,7 +4678,7 @@ PyObject *exc; PyObject **values; exc = stack_pointer[-1]; - values = stack_pointer - 1 - oparg; + values = &stack_pointer[-1 - oparg]; TIER_ONE_ONLY assert(oparg >= 0 && oparg <= 2); if (oparg) { @@ -4693,12 +4714,11 @@ INSTRUCTION_STATS(RESUME); PREDICTED(RESUME); _Py_CODEUNIT *this_instr = next_instr - 1; - static_assert(0 == 0, "incorrect cache size"); TIER_ONE_ONLY assert(frame == tstate->current_frame); uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; + _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((code_version & 255) == 0); if (code_version != global_version) { @@ -4719,10 +4739,11 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); -#if defined(__EMSCRIPTEN__) + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; -#endif + #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->interp->ceval.eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); @@ -4736,7 +4757,7 @@ INSTRUCTION_STATS(RETURN_CONST); PyObject *value; PyObject *retval; - // LOAD_CONST + // _LOAD_CONST { value = GETITEM(FRAME_CO_CONSTS, oparg); Py_INCREF(value); @@ -4744,11 +4765,11 @@ // _POP_FRAME retval = value; { - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4757,12 +4778,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); - #if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } - #endif + #endif } DISPATCH(); } @@ -4801,12 +4822,12 @@ INSTRUCTION_STATS(RETURN_VALUE); PyObject *retval; retval = stack_pointer[-1]; - STACK_SHRINK(1); - assert(EMPTY()); #if TIER_ONE assert(frame != &entry_frame); #endif - STORE_SP(); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(EMPTY()); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -4815,12 +4836,12 @@ _PyFrame_StackPush(frame, retval); LOAD_SP(); LOAD_IP(frame->return_offset); -#if LLTRACE && TIER_ONE + #if LLTRACE && TIER_ONE lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); if (lltrace < 0) { goto exit_unwind; } -#endif + #endif DISPATCH(); } @@ -4830,7 +4851,6 @@ INSTRUCTION_STATS(SEND); PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *receiver; PyObject *v; PyObject *retval; @@ -4897,6 +4917,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -4955,7 +4976,7 @@ int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -4971,28 +4992,28 @@ PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { case MAKE_FUNCTION_CLOSURE: - assert(func_obj->func_closure == NULL); - func_obj->func_closure = attr; - break; + assert(func_obj->func_closure == NULL); + func_obj->func_closure = attr; + break; case MAKE_FUNCTION_ANNOTATIONS: - assert(func_obj->func_annotations == NULL); - func_obj->func_annotations = attr; - break; + assert(func_obj->func_annotations == NULL); + func_obj->func_annotations = attr; + break; case MAKE_FUNCTION_KWDEFAULTS: - assert(PyDict_CheckExact(attr)); - assert(func_obj->func_kwdefaults == NULL); - func_obj->func_kwdefaults = attr; - break; + assert(PyDict_CheckExact(attr)); + assert(func_obj->func_kwdefaults == NULL); + func_obj->func_kwdefaults = attr; + break; case MAKE_FUNCTION_DEFAULTS: - assert(PyTuple_CheckExact(attr)); - assert(func_obj->func_defaults == NULL); - func_obj->func_defaults = attr; - break; + assert(PyTuple_CheckExact(attr)); + assert(func_obj->func_defaults == NULL); + func_obj->func_defaults = attr; + break; default: - Py_UNREACHABLE(); + Py_UNREACHABLE(); } - STACK_SHRINK(1); - stack_pointer[-1] = func; + stack_pointer[-2] = func; + stack_pointer += -1; DISPATCH(); } @@ -5007,7 +5028,7 @@ int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5017,7 +5038,6 @@ INSTRUCTION_STATS(STORE_ATTR); PREDICTED(STORE_ATTR); _Py_CODEUNIT *this_instr = next_instr - 5; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; // _SPECIALIZE_STORE_ATTR @@ -5045,7 +5065,7 @@ Py_DECREF(owner); if (err) goto pop_2_error; } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5053,8 +5073,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5086,7 +5108,7 @@ } Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5094,8 +5116,10 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; + /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -5115,7 +5139,7 @@ Py_XDECREF(old_value); Py_DECREF(owner); } - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5123,6 +5147,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; @@ -5167,7 +5192,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5181,7 +5206,7 @@ PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); Py_XDECREF(oldobj); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5192,7 +5217,7 @@ PyObject *value; value = stack_pointer[-1]; SETLOCAL(oparg, value); - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5224,7 +5249,7 @@ uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); SETLOCAL(oparg2, value2); - STACK_SHRINK(2); + stack_pointer += -2; DISPATCH(); } @@ -5238,7 +5263,7 @@ int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5258,12 +5283,12 @@ if (true) goto pop_1_error; } if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); + err = PyDict_SetItem(ns, name, v); else - err = PyObject_SetItem(ns, name, v); + err = PyObject_SetItem(ns, name, v); Py_DECREF(v); if (err) goto pop_1_error; - STACK_SHRINK(1); + stack_pointer += -1; DISPATCH(); } @@ -5291,7 +5316,7 @@ Py_DECREF(v); Py_DECREF(container); if (err) goto pop_4_error; - STACK_SHRINK(4); + stack_pointer += -4; DISPATCH(); } @@ -5301,7 +5326,6 @@ INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED(STORE_SUBSCR); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -5331,7 +5355,7 @@ Py_DECREF(sub); if (err) goto pop_3_error; } - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5339,6 +5363,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *dict; PyObject *value; @@ -5350,7 +5375,7 @@ int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); if (err) goto pop_3_error; - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5358,6 +5383,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *list; PyObject *value; @@ -5366,21 +5392,19 @@ value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); - // Ensure nonnegative, zero-or-one-digit ints. DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); assert(old_value != NULL); Py_DECREF(old_value); _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); - STACK_SHRINK(3); + stack_pointer += -3; DISPATCH(); } @@ -5404,7 +5428,6 @@ INSTRUCTION_STATS(TO_BOOL); PREDICTED(TO_BOOL); _Py_CODEUNIT *this_instr = next_instr - 4; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; // _SPECIALIZE_TO_BOOL @@ -5437,6 +5460,7 @@ _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5455,6 +5479,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); @@ -5466,6 +5491,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5487,6 +5513,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5502,6 +5529,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5517,6 +5545,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -5587,7 +5616,7 @@ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq); if (res == 0) goto pop_1_error; - STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + stack_pointer += (oparg >> 8) + (oparg & 0xFF); DISPATCH(); } @@ -5597,7 +5626,6 @@ INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED(UNPACK_SEQUENCE); _Py_CODEUNIT *this_instr = next_instr - 2; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; @@ -5623,8 +5651,7 @@ Py_DECREF(seq); if (res == 0) goto pop_1_error; } - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5632,10 +5659,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5644,8 +5672,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5653,10 +5680,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -5665,8 +5693,7 @@ *values++ = Py_NewRef(items[i]); } Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5674,10 +5701,11 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; - values = stack_pointer - 1; + values = &stack_pointer[-1]; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -5685,8 +5713,7 @@ values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); Py_DECREF(seq); - STACK_SHRINK(1); - STACK_GROW(oparg); + stack_pointer += -1 + oparg; DISPATCH(); } @@ -5708,9 +5735,8 @@ - exit_func: FOURTH = the context.__exit__ bound method We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). Then we push the __exit__ return value. - */ + */ PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); exc = PyExceptionInstance_Class(val); tb = PyException_GetTraceback(val); @@ -5724,10 +5750,10 @@ (void)lasti; // Shut up compiler warning if asserts are off PyObject *stack[4] = {NULL, exc, val, tb}; res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - STACK_GROW(1); - stack_pointer[-1] = res; + stack_pointer[0] = res; + stack_pointer += 1; DISPATCH(); } @@ -5759,5 +5785,4 @@ LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); goto resume_frame; } - #undef TIER_ONE |