diff options
Diffstat (limited to 'Python/ceval.c')
| -rw-r--r-- | Python/ceval.c | 515 |
1 files changed, 325 insertions, 190 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index 106e408..cd05780 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1636,6 +1636,17 @@ pop_frame(PyThreadState *tstate, InterpreterFrame *frame) return prev_frame; } +/* It is only between a PRECALL_METHOD/FUNCTION instruction and the following CALL, + * that these values have any meaning. + */ +typedef struct { + PyObject *callable; + PyObject *kwnames; + int total_args; + int postcall_shrink; +} CallShape; + + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag) { @@ -1654,22 +1665,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker; CFrame cframe; - - /* Variables used for making calls */ - PyObject *kwnames; - int nargs; - /* - * It is only between a PRECALL_METHOD instruction and the following instruction, - * that these two values can be anything other than their defaults. */ - int postcall_shrink = 1; - int extra_args = 0; -#define RESET_STACK_ADJUST_FOR_CALLS \ - do { \ - postcall_shrink = 1; \ - extra_args = 0; \ - } while (0) -#define STACK_ADJUST_IS_RESET \ - (postcall_shrink == 1 && extra_args == 0) + CallShape call_shape; + call_shape.kwnames = NULL; // Borrowed reference + call_shape.postcall_shrink = 0; + call_shape.total_args = 0; + call_shape.callable = NULL; // Strong reference /* WARNING: Because the CFrame lives on the C stack, * but can be accessed from a heap allocated object (tstate) @@ -2532,12 +2532,12 @@ handle_eval_breaker: PyObject *iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { - int opcode_at_minus_3 = 0; - if ((next_instr - first_instr) > 2) { - opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]); + int opcode_at_minus_4 = 0; + if ((next_instr - first_instr) > 4) { + opcode_at_minus_4 = _Py_OPCODE(next_instr[-4]); } format_awaitable_error(tstate, Py_TYPE(iterable), - opcode_at_minus_3, + opcode_at_minus_4, _Py_OPCODE(next_instr[-2])); } @@ -4173,7 +4173,6 @@ handle_eval_breaker: if (iter == NULL) goto error; PREDICT(FOR_ITER); - PREDICT(CALL_NO_KW); DISPATCH(); } @@ -4494,6 +4493,16 @@ handle_eval_breaker: NOTRACE_DISPATCH(); } + TARGET(PRECALL_FUNCTION) { + /* Move ownership of reference from stack to call_shape */ + call_shape.callable = PEEK(oparg + 1); + call_shape.postcall_shrink = 1; + + call_shape.total_args = oparg; + call_shape.kwnames = NULL; + DISPATCH(); + } + TARGET(PRECALL_METHOD) { /* Designed to work in tamdem with LOAD_METHOD. */ /* `meth` is NULL when LOAD_METHOD thinks that it's not @@ -4522,54 +4531,57 @@ handle_eval_breaker: make it accept the `self` as a first argument. */ int is_method = (PEEK(oparg + 2) != NULL); - extra_args = is_method; - postcall_shrink = 2-is_method; + int nargs = oparg + is_method; + /* Move ownership of reference from stack to call_shape + * and make sure that NULL is cleared from stack */ + call_shape.callable = PEEK(nargs + 1); + call_shape.postcall_shrink = 2-is_method; + + call_shape.total_args = nargs; + call_shape.kwnames = NULL; DISPATCH(); } - TARGET(CALL_KW) { - kwnames = POP(); - oparg += extra_args; - extra_args = 0; - nargs = oparg - (int)PyTuple_GET_SIZE(kwnames); - goto call_function; + TARGET(KW_NAMES) { + assert(call_shape.kwnames == NULL); + assert(oparg < PyTuple_GET_SIZE(consts)); + call_shape.kwnames = GETITEM(consts, oparg); + DISPATCH(); } - TARGET(CALL_NO_KW) { + TARGET(CALL) { + PREDICTED(CALL); PyObject *function; - PREDICTED(CALL_NO_KW); - kwnames = NULL; - oparg += extra_args; - nargs = oparg; + assert((oparg == 0 && call_shape.kwnames == NULL) + || (oparg != 0 && oparg == PyTuple_GET_SIZE(call_shape.kwnames))); call_function: - function = PEEK(oparg + 1); + function = call_shape.callable; if (Py_TYPE(function) == &PyMethod_Type) { PyObject *meth = ((PyMethodObject *)function)->im_func; PyObject *self = ((PyMethodObject *)function)->im_self; Py_INCREF(meth); Py_INCREF(self); - PEEK(oparg + 1) = self; + PEEK(call_shape.total_args + 1) = self; Py_DECREF(function); function = meth; - oparg++; - nargs++; - assert(postcall_shrink >= 1); - postcall_shrink--; + call_shape.total_args++; + assert(call_shape.postcall_shrink >= 1); + call_shape.postcall_shrink--; } + int total_args = call_shape.total_args; + int positional_args = total_args - oparg; // Check if the call can be inlined or not if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function); - STACK_SHRINK(oparg); + STACK_SHRINK(total_args); InterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, (PyFunctionObject *)function, locals, - stack_pointer, nargs, kwnames + stack_pointer, positional_args, call_shape.kwnames ); - STACK_SHRINK(postcall_shrink); - RESET_STACK_ADJUST_FOR_CALLS; + STACK_SHRINK(call_shape.postcall_shrink); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - Py_XDECREF(kwnames); Py_DECREF(function); if (new_frame == NULL) { goto error; @@ -4582,22 +4594,24 @@ handle_eval_breaker: /* Callable is not a normal Python function */ PyObject *res; if (cframe.use_tracing) { - res = trace_call_function(tstate, function, stack_pointer-oparg, nargs, kwnames); + res = trace_call_function( + tstate, function, stack_pointer-total_args, + positional_args, call_shape.kwnames); } else { - res = PyObject_Vectorcall(function, stack_pointer-oparg, - nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); + res = PyObject_Vectorcall( + function, stack_pointer-total_args, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + call_shape.kwnames); } assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(function); - Py_XDECREF(kwnames); /* Clear the stack */ - STACK_SHRINK(oparg); - for (int i = 0; i < oparg; i++) { + STACK_SHRINK(total_args); + for (int i = 0; i < total_args; i++) { Py_DECREF(stack_pointer[i]); } - STACK_SHRINK(postcall_shrink); - RESET_STACK_ADJUST_FOR_CALLS; + STACK_SHRINK(call_shape.postcall_shrink); PUSH(res); if (res == NULL) { goto error; @@ -4606,65 +4620,87 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_ADAPTIVE) { + TARGET(CALL_ADAPTIVE) { SpecializedCacheEntry *cache = GET_CACHE(); - oparg = cache->adaptive.original_oparg; + int named_args = cache->adaptive.original_oparg; + assert((named_args == 0 && call_shape.kwnames == NULL) + || (named_args != 0 && named_args == PyTuple_GET_SIZE(call_shape.kwnames))); if (cache->adaptive.counter == 0) { next_instr--; - int nargs = oparg+extra_args; - if (_Py_Specialize_CallNoKw( - PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) { + int nargs = call_shape.total_args; + int err = _Py_Specialize_CallNoKw( + call_shape.callable, next_instr, nargs, + call_shape.kwnames, cache, BUILTINS()); + if (err < 0) { goto error; } DISPATCH(); } else { - STAT_INC(CALL_NO_KW, deferred); + STAT_INC(CALL, deferred); cache->adaptive.counter--; - kwnames = NULL; - oparg += extra_args; - nargs = oparg; + oparg = named_args; goto call_function; } } - TARGET(CALL_NO_KW_PY_SIMPLE) { + TARGET(CALL_PY_EXACT_ARGS) { SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; - int argcount = cache0->original_oparg + extra_args; - DEOPT_IF(argcount != cache0->index, CALL_NO_KW); + int argcount = call_shape.total_args; + DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL); _PyCallCache *cache1 = &caches[-1].call; - PyObject *callable = PEEK(argcount+1); - DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW); - /* PEP 523 */ - DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); + PyFunctionObject *func = (PyFunctionObject *)call_shape.callable; + DEOPT_IF(func->func_version != cache1->func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE; - InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size); + DEOPT_IF(code->co_argcount != argcount, CALL); + InterpreterFrame *new_frame = _PyFrame_Push(tstate, func); if (new_frame == NULL) { - RESET_STACK_ADJUST_FOR_CALLS; goto error; } - _PyFrame_InitializeSpecials(new_frame, func, - NULL, code->co_nlocalsplus); STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = stack_pointer[i]; } - int deflen = cache1->defaults_len; - for (int i = 0; i < deflen; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, cache1->defaults_start+i); + for (int i = argcount; i < code->co_nlocalsplus; i++) { + new_frame->localsplus[i] = NULL; + } + STACK_SHRINK(call_shape.postcall_shrink); + Py_DECREF(func); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + frame = cframe.current_frame = new_frame; + goto start_frame; + } + + TARGET(CALL_PY_WITH_DEFAULTS) { + SpecializedCacheEntry *caches = GET_CACHE(); + int argcount = call_shape.total_args; + DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL); + _PyCallCache *cache1 = &caches[-1].call; + PyFunctionObject *func = (PyFunctionObject *)call_shape.callable; + DEOPT_IF(func->func_version != cache1->func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(argcount > code->co_argcount, CALL); + int minargs = cache1->min_args; + DEOPT_IF(argcount < minargs, CALL); + InterpreterFrame *new_frame = _PyFrame_Push(tstate, func); + if (new_frame == NULL) { + goto error; + } + STACK_SHRINK(argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = stack_pointer[i]; + } + int def_offset = cache1->defaults_len - code->co_argcount; + for (int i = argcount; i < code->co_argcount; i++) { + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i + def_offset); Py_INCREF(def); - new_frame->localsplus[argcount+i] = def; + new_frame->localsplus[i] = def; } - for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) { + for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - STACK_SHRINK(postcall_shrink); - RESET_STACK_ADJUST_FOR_CALLS; + STACK_SHRINK(call_shape.postcall_shrink); Py_DECREF(func); _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; @@ -4674,35 +4710,75 @@ handle_eval_breaker: TARGET(CALL_NO_KW_TYPE_1) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); - assert(GET_CACHE()->adaptive.original_oparg == 1); + DEOPT_IF(call_shape.total_args != 1, CALL); + assert(call_shape.kwnames == NULL); PyObject *obj = TOP(); PyObject *callable = SECOND(); - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL_NO_KW); + DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); PyObject *res = Py_NewRef(Py_TYPE(obj)); - STACK_SHRINK(1); Py_DECREF(callable); Py_DECREF(obj); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); NOTRACE_DISPATCH(); } - TARGET(CALL_NO_KW_BUILTIN_CLASS_1) { + TARGET(CALL_NO_KW_STR_1) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); - SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; - assert(cache0->original_oparg == 1); - PyObject *callable = SECOND(); + DEOPT_IF(!PyType_Check(call_shape.callable), CALL); + PyTypeObject *tp = (PyTypeObject *)call_shape.callable; + DEOPT_IF(call_shape.total_args != 1, CALL); + DEOPT_IF(tp != &PyUnicode_Type, CALL); + STAT_INC(CALL, hit); + assert(call_shape.kwnames == NULL); PyObject *arg = TOP(); - DEOPT_IF(!PyType_Check(callable), CALL_NO_KW); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != cache0->version, CALL_NO_KW); - STACK_SHRINK(1); - PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, 1, NULL); + PyObject *res = PyObject_Str(arg); + Py_DECREF(arg); + Py_DECREF(&PyUnicode_Type); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); - Py_DECREF(tp); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(CALL_NO_KW_TUPLE_1) { + DEOPT_IF(!PyType_Check(call_shape.callable), CALL); + PyTypeObject *tp = (PyTypeObject *)call_shape.callable; + DEOPT_IF(call_shape.total_args != 1, CALL); + DEOPT_IF(tp != &PyTuple_Type, CALL); + STAT_INC(CALL, hit); + assert(call_shape.kwnames == NULL); + PyObject *arg = TOP(); + PyObject *res = PySequence_Tuple(arg); Py_DECREF(arg); + Py_DECREF(&PyTuple_Type); + STACK_SHRINK(call_shape.postcall_shrink); + SET_TOP(res); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_CLASS) { + DEOPT_IF(!PyType_Check(call_shape.callable), CALL); + PyTypeObject *tp = (PyTypeObject *)call_shape.callable; + DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + STAT_INC(CALL, hit); + int kwnames_len = GET_CACHE()->adaptive.original_oparg; + + int nargs = call_shape.total_args - kwnames_len; + STACK_SHRINK(call_shape.total_args); + PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, nargs, call_shape.kwnames); + /* Free the arguments. */ + for (int i = 0; i < call_shape.total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + Py_DECREF(tp); + STACK_SHRINK(call_shape.postcall_shrink-1); + SET_TOP(res); if (res == NULL) { goto error; } @@ -4711,13 +4787,13 @@ handle_eval_breaker: TARGET(CALL_NO_KW_BUILTIN_O) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); /* Builtin METH_O functions */ - - PyObject *callable = SECOND(); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); + assert(call_shape.kwnames == NULL); + DEOPT_IF(call_shape.total_args != 1, CALL); + PyObject *callable = call_shape.callable; + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -4725,14 +4801,14 @@ handle_eval_breaker: if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = POP(); + PyObject *arg = TOP(); PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCall(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the function object. */ Py_DECREF(arg); Py_DECREF(callable); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); if (res == NULL) { goto error; @@ -4742,32 +4818,31 @@ handle_eval_breaker: TARGET(CALL_NO_KW_BUILTIN_FAST) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); /* Builtin METH_FASTCALL functions, without keywords */ - SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; - int nargs = cache0->original_oparg; - PyObject **pfunc = &PEEK(nargs + 1); - PyObject *callable = *pfunc; - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW); + assert(call_shape.kwnames == NULL); + PyObject *callable = call_shape.callable; + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); + CALL); + STAT_INC(CALL, hit); + int nargs = call_shape.total_args; PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + STACK_SHRINK(nargs); /* res = func(self, args, nargs) */ PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - &PEEK(nargs), + stack_pointer, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the function object. */ - while (stack_pointer > pfunc) { - PyObject *x = POP(); - Py_DECREF(x); + /* Free the arguments. */ + for (int i = 0; i < nargs; i++) { + Py_DECREF(stack_pointer[i]); } + STACK_SHRINK(call_shape.postcall_shrink); PUSH(res); + Py_DECREF(callable); if (res == NULL) { /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, @@ -4779,29 +4854,72 @@ handle_eval_breaker: DISPATCH(); } + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(cframe.use_tracing == 0); + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + PyObject *callable = call_shape.callable; + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != + (METH_FASTCALL | METH_KEYWORDS), CALL); + STAT_INC(CALL, hit); + int kwnames_len = GET_CACHE()->adaptive.original_oparg; + assert( + (call_shape.kwnames == NULL && kwnames_len == 0) || + (call_shape.kwnames != NULL && + PyTuple_GET_SIZE(call_shape.kwnames) == kwnames_len) + ); + int nargs = call_shape.total_args - kwnames_len; + STACK_SHRINK(call_shape.total_args); + /* res = func(self, args, nargs, kwnames) */ + _PyCFunctionFastWithKeywords cfunc = + (_PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + PyObject *res = cfunc( + PyCFunction_GET_SELF(callable), + stack_pointer, + nargs, + call_shape.kwnames + ); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + + /* Free the arguments. */ + for (int i = 0; i < call_shape.total_args; i++) { + Py_DECREF(stack_pointer[i]); + } + STACK_SHRINK(call_shape.postcall_shrink); + PUSH(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + TARGET(CALL_NO_KW_LEN) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); + assert(call_shape.kwnames == NULL); /* len(o) */ SpecializedCacheEntry *caches = GET_CACHE(); - assert(caches[0].adaptive.original_oparg == 1); + DEOPT_IF(call_shape.total_args != 1, CALL); + assert(caches[0].adaptive.original_oparg == 0); _PyObjectCache *cache1 = &caches[-1].obj; - PyObject *callable = SECOND(); - DEOPT_IF(callable != cache1->obj, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); + PyObject *callable = call_shape.callable; + DEOPT_IF(callable != cache1->obj, CALL); + STAT_INC(CALL, hit); - Py_ssize_t len_i = PyObject_Length(TOP()); + PyObject *arg = TOP(); + Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } PyObject *res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the function object. */ - Py_DECREF(POP()); - Py_DECREF(callable); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); + Py_DECREF(callable); + Py_DECREF(arg); if (res == NULL) { goto error; } @@ -4810,28 +4928,30 @@ handle_eval_breaker: TARGET(CALL_NO_KW_ISINSTANCE) { assert(cframe.use_tracing == 0); - assert(STACK_ADJUST_IS_RESET); + assert(call_shape.kwnames == NULL); /* isinstance(o, o2) */ SpecializedCacheEntry *caches = GET_CACHE(); - assert(caches[0].adaptive.original_oparg == 2); + assert(caches[0].adaptive.original_oparg == 0); + DEOPT_IF(call_shape.total_args != 2, CALL); _PyObjectCache *cache1 = &caches[-1].obj; - PyObject *callable = THIRD(); - DEOPT_IF(callable != cache1->obj, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); + DEOPT_IF(call_shape.callable != cache1->obj, CALL); + STAT_INC(CALL, hit); - int retval = PyObject_IsInstance(SECOND(), TOP()); + PyObject *cls = POP(); + PyObject *inst = TOP(); + int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { goto error; } PyObject *res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the function object. */ - Py_DECREF(POP()); - Py_DECREF(POP()); - Py_DECREF(callable); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); + Py_DECREF(inst); + Py_DECREF(cls); + Py_DECREF(call_shape.callable); if (res == NULL) { goto error; } @@ -4840,57 +4960,78 @@ handle_eval_breaker: TARGET(CALL_NO_KW_LIST_APPEND) { assert(cframe.use_tracing == 0); - assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); - assert(GET_CACHE()->adaptive.original_oparg == 1); - DEOPT_IF(extra_args == 0, CALL_NO_KW); + assert(call_shape.kwnames == NULL); + SpecializedCacheEntry *caches = GET_CACHE(); + _PyObjectCache *cache1 = &caches[-1].obj; + DEOPT_IF(call_shape.total_args != 2, CALL); + DEOPT_IF(call_shape.callable != cache1->obj, CALL); PyObject *list = SECOND(); - DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); - assert(extra_args == 1); - extra_args = 0; - assert(STACK_ADJUST_IS_RESET); + DEOPT_IF(!PyList_Check(list), CALL); + STAT_INC(CALL, hit); PyObject *arg = TOP(); int err = PyList_Append(list, arg); if (err) { goto error; } - PyObject *callable = THIRD(); Py_DECREF(arg); Py_DECREF(list); + STACK_SHRINK(call_shape.postcall_shrink+1); Py_INCREF(Py_None); - STACK_SHRINK(2); SET_TOP(Py_None); - Py_DECREF(callable); + Py_DECREF(call_shape.callable); NOTRACE_DISPATCH(); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); - assert(GET_CACHE()->adaptive.original_oparg == 1); - DEOPT_IF(extra_args == 0, CALL_NO_KW); - assert(extra_args == 1); - PyObject *callable = THIRD(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW); - DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); - assert(extra_args == 1); - extra_args = 0; - assert(STACK_ADJUST_IS_RESET); - PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth; + assert(call_shape.kwnames == NULL); + DEOPT_IF(call_shape.total_args != 2, CALL); + DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method; + DEOPT_IF(meth->ml_flags != METH_O, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall // function calls. if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = POP(); - PyObject *self = POP(); + PyObject *arg = TOP(); + PyObject *self = SECOND(); PyObject *res = cfunc(self, arg); _Py_LeaveRecursiveCall(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); + STACK_SHRINK(call_shape.postcall_shrink+1); SET_TOP(res); - Py_DECREF(callable); + Py_DECREF(call_shape.callable); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + assert(call_shape.kwnames == NULL); + DEOPT_IF(call_shape.total_args != 1, CALL); + DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method; + DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { + goto error; + } + PyObject *self = TOP(); + PyObject *res = cfunc(self, NULL); + _Py_LeaveRecursiveCall(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + STACK_SHRINK(call_shape.postcall_shrink); + SET_TOP(res); + Py_DECREF(call_shape.callable); if (res == NULL) { goto error; } @@ -4898,32 +5039,26 @@ handle_eval_breaker: } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); + assert(call_shape.kwnames == NULL); /* Builtin METH_FASTCALL methods, without keywords */ - SpecializedCacheEntry *caches = GET_CACHE(); - _PyAdaptiveEntry *cache0 = &caches[0].adaptive; - DEOPT_IF(extra_args == 0, CALL_NO_KW); - assert(extra_args == 1); - int nargs = cache0->original_oparg; - PyObject *callable = PEEK(nargs + 2); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW); - PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW); - STAT_INC(CALL_NO_KW, hit); - assert(extra_args == 1); - extra_args = 0; - assert(STACK_ADJUST_IS_RESET); + DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - PyObject *self = PEEK(nargs+1); - PyObject *res = cfunc(self, &PEEK(nargs), nargs); + int nargs = call_shape.total_args-1; + STACK_SHRINK(nargs); + PyObject *self = TOP(); + PyObject *res = cfunc(self, stack_pointer, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ - STACK_SHRINK(nargs+1); - for (int i = 0; i <= nargs; i++) { + for (int i = 0; i < nargs; i++) { Py_DECREF(stack_pointer[i]); } + Py_DECREF(self); + STACK_SHRINK(call_shape.postcall_shrink); SET_TOP(res); - Py_DECREF(callable); + Py_DECREF(call_shape.callable); if (res == NULL) { goto error; } @@ -5283,7 +5418,7 @@ MISS_WITH_CACHE(LOAD_ATTR) MISS_WITH_CACHE(STORE_ATTR) MISS_WITH_CACHE(LOAD_GLOBAL) MISS_WITH_CACHE(LOAD_METHOD) -MISS_WITH_CACHE(CALL_NO_KW) +MISS_WITH_CACHE(CALL) MISS_WITH_CACHE(BINARY_OP) MISS_WITH_CACHE(COMPARE_OP) MISS_WITH_CACHE(BINARY_SUBSCR) @@ -7321,7 +7456,7 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg) } static void -format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevopcode, int prevopcode) +format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevprevopcode, int prevopcode) { if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) { if (prevopcode == BEFORE_ASYNC_WITH) { @@ -7330,7 +7465,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop "that does not implement __await__: %.100s", type->tp_name); } - else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == LOAD_CONST)) { + else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL && prevprevprevopcode == LOAD_CONST)) { _PyErr_Format(tstate, PyExc_TypeError, "'async with' received an object from __aexit__ " "that does not implement __await__: %.100s", |