From 616aec1ff148ba4570aa2d4b8ea420c715c206e4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 8 Feb 2023 11:40:10 -0800 Subject: gh-98831: Modernize CALL and family (#101508) Includes a slight improvement to `DECREF_INPUTS()`. --- Python/bytecodes.c | 532 ++++++++++++++---------------- Python/ceval.c | 6 - Python/ceval_macros.h | 3 + Python/generated_cases.c.h | 560 ++++++++++++++++++-------------- Python/opcode_metadata.h | 112 +++---- Tools/cases_generator/generate_cases.py | 21 +- 6 files changed, 646 insertions(+), 588 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 1169d8d..2b9f12f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2360,70 +2360,87 @@ dummy_func( assert(oparg & 1); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - } - inst(KW_NAMES, (--)) { assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); kwnames = GETITEM(consts, oparg); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL) { + // Cache layout: counter/1, func_version/2, min_args/1 + // Neither CALL_INTRINSIC_1 nor CALL_FUNCTION_EX are members! + family(call, INLINE_CACHE_ENTRIES_CALL) = { + CALL, + CALL_BOUND_METHOD_EXACT_ARGS, + CALL_PY_EXACT_ARGS, + CALL_PY_WITH_DEFAULTS, + CALL_NO_KW_TYPE_1, + CALL_NO_KW_STR_1, + CALL_NO_KW_TUPLE_1, + CALL_BUILTIN_CLASS, + CALL_NO_KW_BUILTIN_O, + CALL_NO_KW_BUILTIN_FAST, + CALL_BUILTIN_FAST_WITH_KEYWORDS, + CALL_NO_KW_LEN, + CALL_NO_KW_ISINSTANCE, + CALL_NO_KW_LIST_APPEND, + CALL_NO_KW_METHOD_DESCRIPTOR_O, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + }; + + // On entry, the stack is either + // [NULL, callable, arg1, arg2, ...] + // or + // [method, self, arg1, arg2, ...] + // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) + // On exit, the stack is [result]. + // When calling Python, inline the call using DISPATCH_INLINED(). + inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - int total_args, is_meth; - is_meth = is_method(stack_pointer, oparg); - PyObject *function = PEEK(oparg + 1); - if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = Py_NewRef(meth); - Py_DECREF(function); - is_meth = 1; - } - total_args = oparg + is_meth; - function = PEEK(total_args + 1); + if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { + is_meth = 1; // For consistenct; it's dead, though + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not - if (Py_TYPE(function) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); - STACK_SHRINK(total_args); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, kwnames + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2-is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -2433,190 +2450,180 @@ dummy_func( DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer-total_args, + tstate, callable, args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer-total_args, + callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(function); - /* Clear the stack */ - STACK_SHRINK(total_args); + Py_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_PY_EXACT_ARGS) { + // Start out with [NULL, bound_method, arg1, arg2, ...] + // Transform to [callable, self, arg1, arg2, ...] + // Then fall through to CALL_PY_EXACT_ARGS + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) { + DEOPT_IF(method != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + STAT_INC(CALL, hit); + PyObject *self = ((PyMethodObject *)callable)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // callable + PyObject *meth = ((PyMethodObject *)callable)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // method + Py_DECREF(callable); + GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + } + + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); - int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(2-is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_PY_WITH_DEFAULTS) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); - int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(argcount > code->co_argcount, CALL); - int minargs = cache->min_args; - DEOPT_IF(argcount < minargs, CALL); + DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, - i - minargs); + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2-is_meth); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_TYPE_1) { + inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *obj = TOP(); - PyObject *callable = SECOND(); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - PyObject *res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(callable); + res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - STACK_SHRINK(2); - SET_TOP(res); + Py_DECREF(&PyType_Type); // I.e., callable } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_STR_1) { + inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PyObject_Str(arg); + PyObject *arg = args[0]; + res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyUnicode_Type); // I.e., callable + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_TUPLE_1) { + inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PySequence_Tuple(arg); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyTuple_Type); // I.e., tuple + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BUILTIN_CLASS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } int kwnames_len = KWNAMES_LEN(); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); - PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, kwnames); + res = tp->tp_vectorcall((PyObject *)tp, args, + total_args - kwnames_len, kwnames); kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } Py_DECREF(tp); - STACK_SHRINK(1-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_BUILTIN_O) { + inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -2626,81 +2633,74 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = TOP(); - PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - STACK_SHRINK(2-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_BUILTIN_FAST) { + inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - STACK_SHRINK(total_args); /* res = func(self, args, nargs) */ - PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - stack_pointer, + args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { + ERROR_IF(res == NULL, error); /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); - PyObject *res = cfunc( + res = cfunc( PyCFunction_GET_SELF(callable), - stack_pointer, + args, total_args - KWNAMES_LEN(), kwnames ); @@ -2709,117 +2709,109 @@ dummy_func( /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_LEN) { + inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); + PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } - PyObject *res = PyLong_FromSsize_t(len_i); + res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_ISINSTANCE) { + inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = POP(); - PyObject *inst = TOP(); + PyObject *cls = args[1]; + PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - Py_DECREF(cls); goto error; } - PyObject *res = PyBool_FromLong(retval); + res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_LIST_APPEND) { + // This is secretly a super-instruction + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); - PyObject *callable = PEEK(3); + assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { - goto error; + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already } - STACK_SHRINK(2); - Py_DECREF(list); - Py_DECREF(callable); + Py_DECREF(self); + Py_DECREF(method); + STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); + DISPATCH(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(total_args != 2, CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = TOP(); - PyObject *self = SECOND(); + PyObject *arg = args[1]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -2828,69 +2820,62 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); - int nargs = total_args-1; - STACK_SHRINK(nargs); + int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - kwnames); + res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; /* Free the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = TOP(); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -2900,52 +2885,43 @@ dummy_func( if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } - // stack effect: (__0, __array[oparg] -- ) - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args-1; - STACK_SHRINK(nargs); - PyObject *res = cfunc(self, stack_pointer, nargs); + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + ERROR_IF(res == NULL, error); CHECK_EVAL_BREAKER(); } @@ -3153,12 +3129,4 @@ dummy_func( // Future families go below this point // -family(call, INLINE_CACHE_ENTRIES_CALL) = { - CALL, CALL_PY_EXACT_ARGS, - CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, - CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST, - CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN, - CALL_NO_KW_LIST_APPEND, CALL_NO_KW_METHOD_DESCRIPTOR_FAST, CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, - CALL_NO_KW_TYPE_1 }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; diff --git a/Python/ceval.c b/Python/ceval.c index a91f5ba..611d62b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -688,12 +688,6 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { } -// GH-89279: Must be a macro to be sure it's inlined by MSVC. -#define is_method(stack_pointer, args) (PEEK((args)+2) != NULL) - -#define KWNAMES_LEN() \ - (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) - /* Disable unused label warnings. They are handy for debugging, even if computed gotos aren't used. */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d7a8f0b..691bf8e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -347,3 +347,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { } while (0); #define NAME_ERROR_MSG "name '%.200s' is not defined" + +#define KWNAMES_LEN() \ + (kwnames == NULL ? 0 : ((int)PyTuple_GET_SIZE(kwnames))) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 09eb689..a224d4e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2994,19 +2994,6 @@ DISPATCH(); } - TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - DEOPT_IF(is_method(stack_pointer, oparg), CALL); - PyObject *function = PEEK(oparg + 1); - DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, CALL); - STAT_INC(CALL, hit); - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); - Py_DECREF(function); - GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - } - TARGET(KW_NAMES) { assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(consts)); @@ -3016,48 +3003,55 @@ TARGET(CALL) { PREDICTED(CALL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size"); + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } #if ENABLE_SPECIALIZATION _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); - int is_meth = is_method(stack_pointer, oparg); - int nargs = oparg + is_meth; - PyObject *callable = PEEK(nargs + 1); next_instr--; - _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _Py_Specialize_Call(callable, next_instr, total_args, kwnames); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - int total_args, is_meth; - is_meth = is_method(stack_pointer, oparg); - PyObject *function = PEEK(oparg + 1); - if (!is_meth && Py_TYPE(function) == &PyMethod_Type) { - PyObject *self = ((PyMethodObject *)function)->im_self; - PEEK(oparg+1) = Py_NewRef(self); - PyObject *meth = ((PyMethodObject *)function)->im_func; - PEEK(oparg+2) = Py_NewRef(meth); - Py_DECREF(function); - is_meth = 1; - } - total_args = oparg + is_meth; - function = PEEK(total_args + 1); + if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { + is_meth = 1; // For consistenct; it's dead, though + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } int positional_args = total_args - KWNAMES_LEN(); // Check if the call can be inlined or not - if (Py_TYPE(function) == &PyFunction_Type && + if (Py_TYPE(callable) == &PyFunction_Type && tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)function)->vectorcall == _PyFunction_Vectorcall) + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(function)); - STACK_SHRINK(total_args); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)function, locals, - stack_pointer, positional_args, kwnames + tstate, (PyFunctionObject *)callable, locals, + args, positional_args, kwnames ); kwnames = NULL; - STACK_SHRINK(2-is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. if (new_frame == NULL) { @@ -3067,189 +3061,234 @@ DISPATCH_INLINED(new_frame); } /* Callable is not a normal Python function */ - PyObject *res; if (cframe.use_tracing) { res = trace_call_function( - tstate, function, stack_pointer-total_args, + tstate, callable, args, positional_args, kwnames); } else { res = PyObject_Vectorcall( - function, stack_pointer-total_args, + callable, args, positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); } kwnames = NULL; assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(function); - /* Clear the stack */ - STACK_SHRINK(total_args); + Py_DECREF(callable); for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + DEOPT_IF(method != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + STAT_INC(CALL, hit); + PyObject *self = ((PyMethodObject *)callable)->im_self; + PEEK(oparg + 1) = Py_NewRef(self); // callable + PyObject *meth = ((PyMethodObject *)callable)->im_func; + PEEK(oparg + 2) = Py_NewRef(meth); // method + Py_DECREF(callable); + GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + } + TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + uint32_t func_version = read_u32(&next_instr[1].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); - int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(code->co_argcount != argcount, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } - STACK_SHRINK(2-is_meth); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } TARGET(CALL_PY_WITH_DEFAULTS) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + uint32_t func_version = read_u32(&next_instr[1].cache); + uint16_t min_args = read_u16(&next_instr[3].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); - _PyCallCache *cache = (_PyCallCache *)next_instr; - int is_meth = is_method(stack_pointer, oparg); - int argcount = oparg + is_meth; - PyObject *callable = PEEK(argcount + 1); + int is_meth = method != NULL; + int argcount = oparg; + if (is_meth) { + callable = method; + args--; + argcount++; + } DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != read_u32(cache->func_version), CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(argcount > code->co_argcount, CALL); - int minargs = cache->min_args; - DEOPT_IF(argcount < minargs, CALL); + DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - STACK_SHRINK(argcount); for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = stack_pointer[i]; + new_frame->localsplus[i] = args[i]; } for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, - i - minargs); + PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); new_frame->localsplus[i] = Py_NewRef(def); } - STACK_SHRINK(2-is_meth); + // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); } TARGET(CALL_NO_KW_TYPE_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *obj = TOP(); - PyObject *callable = SECOND(); + DEOPT_IF(null != NULL, CALL); + PyObject *obj = args[0]; DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); STAT_INC(CALL, hit); - JUMPBY(INLINE_CACHE_ENTRIES_CALL); - PyObject *res = Py_NewRef(Py_TYPE(obj)); - Py_DECREF(callable); + res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); - STACK_SHRINK(2); - SET_TOP(res); + Py_DECREF(&PyType_Type); // I.e., callable + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } TARGET(CALL_NO_KW_STR_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PyObject_Str(arg); + PyObject *arg = args[0]; + res = PyObject_Str(arg); Py_DECREF(arg); - Py_DECREF(&PyUnicode_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyUnicode_Type); // I.e., callable + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_TUPLE_1) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *null = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); - PyObject *callable = PEEK(2); + DEOPT_IF(null != NULL, CALL); DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); - PyObject *res = PySequence_Tuple(arg); + PyObject *arg = args[0]; + res = PySequence_Tuple(arg); Py_DECREF(arg); - Py_DECREF(&PyTuple_Type); - STACK_SHRINK(2); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + Py_DECREF(&PyTuple_Type); // I.e., tuple + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_CLASS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } int kwnames_len = KWNAMES_LEN(); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyType_Check(callable), CALL); PyTypeObject *tp = (PyTypeObject *)callable; DEOPT_IF(tp->tp_vectorcall == NULL, CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); - PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, - total_args-kwnames_len, kwnames); + res = tp->tp_vectorcall((PyObject *)tp, args, + total_args - kwnames_len, kwnames); kwnames = NULL; /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } Py_DECREF(tp); - STACK_SHRINK(1-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_BUILTIN_O) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); STAT_INC(CALL, hit); @@ -3259,81 +3298,92 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *arg = TOP(); - PyObject *res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + PyObject *arg = args[0]; + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(arg); Py_DECREF(callable); - STACK_SHRINK(2-is_meth); - SET_TOP(res); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_BUILTIN_FAST) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - STACK_SHRINK(total_args); /* res = func(self, args, nargs) */ - PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)( + res = ((_PyCFunctionFast)(void(*)(void))cfunc)( PyCFunction_GET_SELF(callable), - stack_pointer, + args, total_args); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } /* Not deopting because this doesn't mean our optimization was wrong. `res` can be NULL for valid reasons. Eg. getattr(x, 'invalid'). In those cases an exception is set, so we must handle it. */ - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); - STACK_SHRINK(total_args); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void)) PyCFunction_GET_FUNCTION(callable); - PyObject *res = cfunc( + res = cfunc( PyCFunction_GET_SELF(callable), - stack_pointer, + args, total_args - KWNAMES_LEN(), kwnames ); @@ -3342,99 +3392,112 @@ /* Free the arguments. */ for (int i = 0; i < total_args; i++) { - Py_DECREF(stack_pointer[i]); + Py_DECREF(args[i]); } - STACK_SHRINK(2-is_meth); - PUSH(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_LEN) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); - PyObject *callable = PEEK(total_args + 1); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.len, CALL); STAT_INC(CALL, hit); - PyObject *arg = TOP(); + PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { goto error; } - PyObject *res = PyLong_FromSsize_t(len_i); + res = PyLong_FromSsize_t(len_i); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); Py_DECREF(arg); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } TARGET(CALL_NO_KW_ISINSTANCE) { + PyObject **args = &PEEK(oparg); + PyObject *callable = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + callable = method; + args--; + total_args++; + } DEOPT_IF(total_args != 2, CALL); PyInterpreterState *interp = _PyInterpreterState_GET(); DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); STAT_INC(CALL, hit); - PyObject *cls = POP(); - PyObject *inst = TOP(); + PyObject *cls = args[1]; + PyObject *inst = args[0]; int retval = PyObject_IsInstance(inst, cls); if (retval < 0) { - Py_DECREF(cls); goto error; } - PyObject *res = PyBool_FromLong(retval); + res = PyBool_FromLong(retval); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(inst); Py_DECREF(cls); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); DISPATCH(); } TARGET(CALL_NO_KW_LIST_APPEND) { + PyObject **args = &PEEK(oparg); + PyObject *self = PEEK(1 + oparg); + PyObject *method = PEEK(2 + oparg); assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); - PyObject *callable = PEEK(3); + assert(method != NULL); PyInterpreterState *interp = _PyInterpreterState_GET(); - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); - PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); + DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); - PyObject *arg = POP(); - if (_PyList_AppendTakeRef((PyListObject *)list, arg) < 0) { - goto error; + if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { + goto pop_1_error; // Since arg is DECREF'ed already } - STACK_SHRINK(2); - Py_DECREF(list); - Py_DECREF(callable); + Py_DECREF(self); + Py_DECREF(method); + STACK_SHRINK(3); // CALL + POP_TOP JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(_Py_OPCODE(next_instr[-1]) == POP_TOP); @@ -3442,17 +3505,24 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(total_args != 2, CALL); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); - PyObject *arg = TOP(); - PyObject *self = SECOND(); + PyObject *arg = args[1]; + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3461,69 +3531,78 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, arg); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); PyTypeObject *d_type = callable->d_common.d_type; - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); - int nargs = total_args-1; - STACK_SHRINK(nargs); + int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), - kwnames); + res = cfunc(self, args + 1, nargs - KWNAMES_LEN(), kwnames); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); kwnames = NULL; /* Free the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } DEOPT_IF(total_args != 1, CALL); PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; - PyObject *self = TOP(); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); @@ -3533,52 +3612,55 @@ if (_Py_EnterRecursiveCallTstate(tstate, " while calling a Python object")) { goto error; } - PyObject *res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); _Py_LeaveRecursiveCallTstate(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); - STACK_SHRINK(oparg + 1); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + PyObject **args = &PEEK(oparg); + PyObject *method = PEEK(2 + oparg); + PyObject *res; assert(kwnames == NULL); - int is_meth = is_method(stack_pointer, oparg); - int total_args = oparg + is_meth; + int is_meth = method != NULL; + int total_args = oparg; + if (is_meth) { + args--; + total_args++; + } PyMethodDescrObject *callable = (PyMethodDescrObject *)PEEK(total_args + 1); /* Builtin METH_FASTCALL methods, without keywords */ DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); PyMethodDef *meth = callable->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - PyObject *self = PEEK(total_args); + PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args-1; - STACK_SHRINK(nargs); - PyObject *res = cfunc(self, stack_pointer, nargs); + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); /* Clear the stack of the arguments. */ - for (int i = 0; i < nargs; i++) { - Py_DECREF(stack_pointer[i]); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); } - Py_DECREF(self); - STACK_SHRINK(2-is_meth); - SET_TOP(res); Py_DECREF(callable); - if (res == NULL) { - goto error; - } - JUMPBY(INLINE_CACHE_ENTRIES_CALL); + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + POKE(1, res); + JUMPBY(4); CHECK_EVAL_BREAKER(); DISPATCH(); } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 054ef6c..9879104 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -286,44 +286,44 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_ATTR_METHOD_LAZY_DICT: return 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; case KW_NAMES: return 0; case CALL: - return -1; + return oparg + 2; + case CALL_BOUND_METHOD_EXACT_ARGS: + return oparg + 2; case CALL_PY_EXACT_ARGS: - return -1; + return oparg + 2; case CALL_PY_WITH_DEFAULTS: - return -1; + return oparg + 2; case CALL_NO_KW_TYPE_1: - return -1; + return oparg + 2; case CALL_NO_KW_STR_1: - return -1; + return oparg + 2; case CALL_NO_KW_TUPLE_1: - return -1; + return oparg + 2; case CALL_BUILTIN_CLASS: - return -1; + return oparg + 2; case CALL_NO_KW_BUILTIN_O: - return -1; + return oparg + 2; case CALL_NO_KW_BUILTIN_FAST: - return -1; + return oparg + 2; case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return -1; + return oparg + 2; case CALL_NO_KW_LEN: - return -1; + return oparg + 2; case CALL_NO_KW_ISINSTANCE: - return -1; + return oparg + 2; case CALL_NO_KW_LIST_APPEND: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_O: - return -1; + return oparg + 2; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: - return -1; + return oparg + 2; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: - return -1; + return oparg + 2; case CALL_FUNCTION_EX: return ((oparg & 1) ? 1 : 0) + 3; case MAKE_FUNCTION: @@ -634,44 +634,44 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_METHOD_LAZY_DICT: return ((oparg & 1) ? 1 : 0) + 1; - case CALL_BOUND_METHOD_EXACT_ARGS: - return -1; case KW_NAMES: return 0; case CALL: - return -1; + return 1; + case CALL_BOUND_METHOD_EXACT_ARGS: + return 1; case CALL_PY_EXACT_ARGS: - return -1; + return 1; case CALL_PY_WITH_DEFAULTS: - return -1; + return 1; case CALL_NO_KW_TYPE_1: - return -1; + return 1; case CALL_NO_KW_STR_1: - return -1; + return 1; case CALL_NO_KW_TUPLE_1: - return -1; + return 1; case CALL_BUILTIN_CLASS: - return -1; + return 1; case CALL_NO_KW_BUILTIN_O: - return -1; + return 1; case CALL_NO_KW_BUILTIN_FAST: - return -1; + return 1; case CALL_BUILTIN_FAST_WITH_KEYWORDS: - return -1; + return 1; case CALL_NO_KW_LEN: - return -1; + return 1; case CALL_NO_KW_ISINSTANCE: - return -1; + return 1; case CALL_NO_KW_LIST_APPEND: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_O: - return -1; + return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: - return -1; + return 1; case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: - return -1; + return 1; case CALL_FUNCTION_EX: return 1; case MAKE_FUNCTION: @@ -846,25 +846,25 @@ struct opcode_metadata { [LOAD_ATTR_METHOD_WITH_VALUES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_NO_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC00000000 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [KW_NAMES] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_PY_EXACT_ARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_PY_WITH_DEFAULTS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_TYPE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_STR_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_TUPLE_1] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_BUILTIN_CLASS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_BUILTIN_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_BUILTIN_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_LEN] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_ISINSTANCE] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_LIST_APPEND] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [CALL_FUNCTION_EX] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [RETURN_GENERATOR] = { DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 9b5aa91..1fcfbb6 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -391,9 +391,11 @@ class Instruction: # Write the body, substituting a goto for ERROR_IF() and other stuff assert dedent <= 0 extra = " " * -dedent + names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) for line in self.block_text: if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): space, cond, label = m.groups() + space = extra + space # ERROR_IF() must pop the inputs from the stack. # The code block is responsible for DECREF()ing them. # NOTE: If the label doesn't exist, just add it to ceval.c. @@ -412,16 +414,25 @@ class Instruction: symbolic = "" if symbolic: out.write_raw( - f"{extra}{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" + f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" ) else: - out.write_raw(f"{extra}{space}if ({cond}) goto {label};\n") + out.write_raw(f"{space}if ({cond}) goto {label};\n") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): if not self.register: - space = m.group(1) + space = extra + m.group(1) for ieff in self.input_effects: - if ieff.name not in self.unmoved_names: - out.write_raw(f"{extra}{space}Py_DECREF({ieff.name});\n") + if ieff.name in names_to_skip: + continue + if ieff.size: + out.write_raw( + f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" + ) + out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") + out.write_raw(f"{space}}}\n") + else: + decref = "XDECREF" if ieff.cond else "DECREF" + out.write_raw(f"{space}Py_{decref}({ieff.name});\n") else: out.write_raw(extra + line) -- cgit v0.12