diff options
author | Mark Shannon <mark@hotpy.org> | 2022-02-22 14:57:01 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-22 14:57:01 (GMT) |
commit | 9058a35558422810061989f41571fdfea7ea8cbe (patch) | |
tree | 28e96f82e115b8447c12dbb4834eb11229241332 | |
parent | 1e344684d8d42206858c4eca8ec7950e644f4220 (diff) | |
download | cpython-9058a35558422810061989f41571fdfea7ea8cbe.zip cpython-9058a35558422810061989f41571fdfea7ea8cbe.tar.gz cpython-9058a35558422810061989f41571fdfea7ea8cbe.tar.bz2 |
Move call specializations from CALL to PRECALL. (GH-31496)
-rw-r--r-- | Include/internal/pycore_code.h | 4 | ||||
-rw-r--r-- | Include/opcode.h | 89 | ||||
-rw-r--r-- | Lib/opcode.py | 29 | ||||
-rw-r--r-- | Python/ceval.c | 207 | ||||
-rw-r--r-- | Python/opcode_targets.h | 84 | ||||
-rw-r--r-- | Python/specialize.c | 104 |
6 files changed, 311 insertions, 206 deletions
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index ead9541..d7a514d 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -271,7 +271,9 @@ int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNI int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); -int _Py_Specialize_CallNoKw(PyObject *callable, _Py_CODEUNIT *instr, int nargs, +int _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, + PyObject *kwnames, SpecializedCacheEntry *cache); +int _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins); void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); diff --git a/Include/opcode.h b/Include/opcode.h index df93a93..7820af6 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -135,49 +135,52 @@ extern "C" { #define STORE_SUBSCR_LIST_INT 28 #define STORE_SUBSCR_DICT 29 #define CALL_ADAPTIVE 34 -#define CALL_BUILTIN_CLASS 36 -#define CALL_NO_KW_BUILTIN_O 37 -#define CALL_NO_KW_BUILTIN_FAST 38 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_ISINSTANCE 41 -#define CALL_PY_EXACT_ARGS 42 -#define CALL_PY_WITH_DEFAULTS 43 -#define CALL_NO_KW_LIST_APPEND 44 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 45 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46 -#define CALL_NO_KW_STR_1 47 -#define CALL_NO_KW_TUPLE_1 48 -#define CALL_NO_KW_TYPE_1 55 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 56 -#define JUMP_ABSOLUTE_QUICK 57 -#define LOAD_ATTR_ADAPTIVE 58 -#define LOAD_ATTR_INSTANCE_VALUE 59 -#define LOAD_ATTR_WITH_HINT 62 -#define LOAD_ATTR_SLOT 63 -#define LOAD_ATTR_MODULE 64 -#define LOAD_GLOBAL_ADAPTIVE 65 -#define LOAD_GLOBAL_MODULE 66 -#define LOAD_GLOBAL_BUILTIN 67 -#define LOAD_METHOD_ADAPTIVE 72 -#define LOAD_METHOD_CACHED 76 -#define LOAD_METHOD_CLASS 77 -#define LOAD_METHOD_MODULE 78 -#define LOAD_METHOD_NO_DICT 79 -#define RESUME_QUICK 80 -#define STORE_ATTR_ADAPTIVE 81 -#define STORE_ATTR_INSTANCE_VALUE 131 -#define STORE_ATTR_SLOT 140 -#define STORE_ATTR_WITH_HINT 141 -#define UNPACK_SEQUENCE_ADAPTIVE 143 -#define UNPACK_SEQUENCE_LIST 150 -#define UNPACK_SEQUENCE_TUPLE 153 -#define UNPACK_SEQUENCE_TWO_TUPLE 154 -#define LOAD_FAST__LOAD_FAST 158 -#define STORE_FAST__LOAD_FAST 159 -#define LOAD_FAST__LOAD_CONST 161 -#define LOAD_CONST__LOAD_FAST 167 -#define STORE_FAST__STORE_FAST 168 +#define CALL_PY_EXACT_ARGS 36 +#define CALL_PY_WITH_DEFAULTS 37 +#define JUMP_ABSOLUTE_QUICK 38 +#define LOAD_ATTR_ADAPTIVE 39 +#define LOAD_ATTR_INSTANCE_VALUE 40 +#define LOAD_ATTR_WITH_HINT 41 +#define LOAD_ATTR_SLOT 42 +#define LOAD_ATTR_MODULE 43 +#define LOAD_GLOBAL_ADAPTIVE 44 +#define LOAD_GLOBAL_MODULE 45 +#define LOAD_GLOBAL_BUILTIN 46 +#define LOAD_METHOD_ADAPTIVE 47 +#define LOAD_METHOD_CACHED 48 +#define LOAD_METHOD_CLASS 55 +#define LOAD_METHOD_MODULE 56 +#define LOAD_METHOD_NO_DICT 57 +#define PRECALL_ADAPTIVE 58 +#define PRECALL_BUILTIN_CLASS 59 +#define PRECALL_NO_KW_BUILTIN_O 62 +#define PRECALL_NO_KW_BUILTIN_FAST 63 +#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64 +#define PRECALL_NO_KW_LEN 65 +#define PRECALL_NO_KW_ISINSTANCE 66 +#define PRECALL_NO_KW_LIST_APPEND 67 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 72 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76 +#define PRECALL_NO_KW_STR_1 77 +#define PRECALL_NO_KW_TUPLE_1 78 +#define PRECALL_NO_KW_TYPE_1 79 +#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80 +#define PRECALL_BOUND_METHOD 81 +#define PRECALL_PYFUNC 131 +#define RESUME_QUICK 140 +#define STORE_ATTR_ADAPTIVE 141 +#define STORE_ATTR_INSTANCE_VALUE 143 +#define STORE_ATTR_SLOT 150 +#define STORE_ATTR_WITH_HINT 153 +#define UNPACK_SEQUENCE_ADAPTIVE 154 +#define UNPACK_SEQUENCE_LIST 158 +#define UNPACK_SEQUENCE_TUPLE 159 +#define UNPACK_SEQUENCE_TWO_TUPLE 161 +#define LOAD_FAST__LOAD_FAST 167 +#define STORE_FAST__LOAD_FAST 168 +#define LOAD_FAST__LOAD_CONST 169 +#define LOAD_CONST__LOAD_FAST 170 +#define STORE_FAST__STORE_FAST 173 #define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { diff --git a/Lib/opcode.py b/Lib/opcode.py index 9579245..0c859c1 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -248,21 +248,8 @@ _specialized_instructions = [ "STORE_SUBSCR_LIST_INT", "STORE_SUBSCR_DICT", "CALL_ADAPTIVE", - "CALL_BUILTIN_CLASS", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_NO_KW_LEN", - "CALL_NO_KW_ISINSTANCE", "CALL_PY_EXACT_ARGS", "CALL_PY_WITH_DEFAULTS", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", "JUMP_ABSOLUTE_QUICK", "LOAD_ATTR_ADAPTIVE", "LOAD_ATTR_INSTANCE_VALUE", @@ -277,6 +264,22 @@ _specialized_instructions = [ "LOAD_METHOD_CLASS", "LOAD_METHOD_MODULE", "LOAD_METHOD_NO_DICT", + "PRECALL_ADAPTIVE", + "PRECALL_BUILTIN_CLASS", + "PRECALL_NO_KW_BUILTIN_O", + "PRECALL_NO_KW_BUILTIN_FAST", + "PRECALL_BUILTIN_FAST_WITH_KEYWORDS", + "PRECALL_NO_KW_LEN", + "PRECALL_NO_KW_ISINSTANCE", + "PRECALL_NO_KW_LIST_APPEND", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_O", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "PRECALL_NO_KW_STR_1", + "PRECALL_NO_KW_TUPLE_1", + "PRECALL_NO_KW_TYPE_1", + "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST", + "PRECALL_BOUND_METHOD", + "PRECALL_PYFUNC", "RESUME_QUICK", "STORE_ATTR_ADAPTIVE", "STORE_ATTR_INSTANCE_VALUE", diff --git a/Python/ceval.c b/Python/ceval.c index d3ab1da..4d7c0d0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4482,6 +4482,7 @@ handle_eval_breaker: } TARGET(PRECALL) { + PREDICTED(PRECALL); /* Designed to work in tamdem with LOAD_METHOD. */ /* `meth` is NULL when LOAD_METHOD thinks that it's not a method call. @@ -4529,9 +4530,36 @@ handle_eval_breaker: PEEK(oparg+1) = self; PEEK(oparg+2) = meth; Py_DECREF(function); - function = meth; } + DISPATCH(); + } + + TARGET(PRECALL_BOUND_METHOD) { + SpecializedCacheEntry *cache = GET_CACHE(); + int original_oparg = cache->adaptive.original_oparg; + int is_method = (PEEK(original_oparg + 2) != NULL); + DEOPT_IF(is_method, PRECALL); + PyObject *function = PEEK(original_oparg + 1); + DEOPT_IF(Py_TYPE(function) != &PyMethod_Type, PRECALL); + STAT_INC(PRECALL, hit); + PyObject *meth = ((PyMethodObject *)function)->im_func; + PyObject *self = ((PyMethodObject *)function)->im_self; + Py_INCREF(meth); + Py_INCREF(self); + PEEK(original_oparg+1) = self; + PEEK(original_oparg+2) = meth; + Py_DECREF(function); + DISPATCH(); + } + TARGET(PRECALL_PYFUNC) { + SpecializedCacheEntry *cache = GET_CACHE(); + int original_oparg = cache->adaptive.original_oparg; + int is_method = (PEEK(original_oparg + 2) != NULL); + int nargs = original_oparg + is_method; + PyObject *function = PEEK(nargs + 1); + DEOPT_IF(Py_TYPE(function) != &PyFunction_Type, PRECALL); + STAT_INC(PRECALL, hit); DISPATCH(); } @@ -4602,7 +4630,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_ADAPTIVE) { + TARGET(PRECALL_ADAPTIVE) { SpecializedCacheEntry *cache = GET_CACHE(); int original_oparg = cache->adaptive.original_oparg; if (cache->adaptive.counter == 0) { @@ -4610,7 +4638,7 @@ handle_eval_breaker: int is_meth = is_method(stack_pointer, original_oparg); int nargs = original_oparg + is_meth; PyObject *callable = PEEK(nargs + 1); - int err = _Py_Specialize_CallNoKw( + int err = _Py_Specialize_Precall( callable, next_instr, nargs, call_shape.kwnames, cache, BUILTINS()); if (err < 0) { @@ -4619,6 +4647,30 @@ handle_eval_breaker: DISPATCH(); } else { + STAT_INC(PRECALL, deferred); + cache->adaptive.counter--; + oparg = original_oparg; + JUMP_TO_INSTRUCTION(PRECALL); + } + } + + TARGET(CALL_ADAPTIVE) { + SpecializedCacheEntry *cache = GET_CACHE(); + int original_oparg = cache->adaptive.original_oparg; + if (cache->adaptive.counter == 0) { + next_instr--; + int is_meth = is_method(stack_pointer, original_oparg); + int nargs = original_oparg + is_meth; + PyObject *callable = PEEK(nargs + 1); + int err = _Py_Specialize_Call( + callable, next_instr, nargs, + call_shape.kwnames, cache); + if (err < 0) { + goto error; + } + DISPATCH(); + } + else { STAT_INC(CALL, deferred); cache->adaptive.counter--; oparg = original_oparg; @@ -4698,14 +4750,16 @@ handle_eval_breaker: goto start_frame; } - TARGET(CALL_NO_KW_TYPE_1) { + TARGET(PRECALL_NO_KW_TYPE_1) { assert(call_shape.kwnames == NULL); assert(cframe.use_tracing == 0); assert(GET_CACHE()->adaptive.original_oparg == 1); - DEOPT_IF(is_method(stack_pointer, 1), CALL); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); PyObject *obj = TOP(); PyObject *callable = SECOND(); - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(callable != (PyObject *)&PyType_Type, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyObject *res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(callable); Py_DECREF(obj); @@ -4714,16 +4768,15 @@ handle_eval_breaker: NOTRACE_DISPATCH(); } - TARGET(CALL_NO_KW_STR_1) { + TARGET(PRECALL_NO_KW_STR_1) { assert(call_shape.kwnames == NULL); assert(cframe.use_tracing == 0); assert(GET_CACHE()->adaptive.original_oparg == 1); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); PyObject *callable = PEEK(2); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(is_method(stack_pointer, 1), CALL); - DEOPT_IF(tp != &PyUnicode_Type, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyObject *arg = TOP(); PyObject *res = PyObject_Str(arg); Py_DECREF(arg); @@ -4737,16 +4790,14 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_TUPLE_1) { + TARGET(PRECALL_NO_KW_TUPLE_1) { assert(call_shape.kwnames == NULL); assert(GET_CACHE()->adaptive.original_oparg == 1); - int is_meth = is_method(stack_pointer, 1); + DEOPT_IF(is_method(stack_pointer, 1), PRECALL); PyObject *callable = PEEK(2); - DEOPT_IF(!PyType_Check(callable), CALL); - PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(is_meth, CALL); - DEOPT_IF(tp != &PyTuple_Type, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyObject *arg = TOP(); PyObject *res = PySequence_Tuple(arg); Py_DECREF(arg); @@ -4760,16 +4811,17 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_BUILTIN_CLASS) { + TARGET(PRECALL_BUILTIN_CLASS) { int original_oparg = GET_CACHE()->adaptive.original_oparg; int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; int kwnames_len = KWNAMES_LEN(); PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(!PyType_Check(callable), PRECALL); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(tp->tp_vectorcall == NULL, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); STACK_SHRINK(total_args); PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, total_args-kwnames_len, call_shape.kwnames); @@ -4788,7 +4840,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_BUILTIN_O) { + TARGET(PRECALL_NO_KW_BUILTIN_O) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(call_shape.kwnames == NULL); @@ -4796,12 +4848,12 @@ handle_eval_breaker: int original_oparg = caches->adaptive.original_oparg; int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, PRECALL); PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); - STAT_INC(CALL, hit); - + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall // function calls. @@ -4824,7 +4876,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_BUILTIN_FAST) { + TARGET(PRECALL_NO_KW_BUILTIN_FAST) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(call_shape.kwnames == NULL); @@ -4833,11 +4885,11 @@ handle_eval_breaker: int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL); - STAT_INC(CALL, hit); - + PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); STACK_SHRINK(total_args); /* res = func(self, args, nargs) */ @@ -4866,7 +4918,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + TARGET(PRECALL_BUILTIN_FAST_WITH_KEYWORDS) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ SpecializedCacheEntry *caches = GET_CACHE(); @@ -4874,10 +4926,11 @@ handle_eval_breaker: int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable), PRECALL); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); - STAT_INC(CALL, hit); + (METH_FASTCALL | METH_KEYWORDS), PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); STACK_SHRINK(total_args); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -4906,7 +4959,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_LEN) { + TARGET(PRECALL_NO_KW_LEN) { assert(cframe.use_tracing == 0); assert(call_shape.kwnames == NULL); /* len(o) */ @@ -4914,12 +4967,12 @@ handle_eval_breaker: int original_oparg = caches->adaptive.original_oparg; int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1, PRECALL); _PyObjectCache *cache1 = &caches[-1].obj; PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(callable != cache1->obj, CALL); - STAT_INC(CALL, hit); - + DEOPT_IF(callable != cache1->obj, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyObject *arg = TOP(); Py_ssize_t len_i = PyObject_Length(arg); if (len_i < 0) { @@ -4938,7 +4991,7 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_ISINSTANCE) { + TARGET(PRECALL_NO_KW_ISINSTANCE) { assert(cframe.use_tracing == 0); assert(call_shape.kwnames == NULL); /* isinstance(o, o2) */ @@ -4947,12 +5000,12 @@ handle_eval_breaker: int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2, PRECALL); _PyObjectCache *cache1 = &caches[-1].obj; - DEOPT_IF(callable != cache1->obj, CALL); - STAT_INC(CALL, hit); - + DEOPT_IF(callable != cache1->obj, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyObject *cls = POP(); PyObject *inst = TOP(); int retval = PyObject_IsInstance(inst, cls); @@ -4974,20 +5027,19 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_LIST_APPEND) { + TARGET(PRECALL_NO_KW_LIST_APPEND) { assert(cframe.use_tracing == 0); assert(call_shape.kwnames == NULL); + assert(GET_CACHE()->adaptive.original_oparg == 1); SpecializedCacheEntry *caches = GET_CACHE(); - int original_oparg = caches->adaptive.original_oparg; _PyObjectCache *cache1 = &caches[-1].obj; - int is_meth = is_method(stack_pointer, original_oparg); - int total_args = original_oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(callable != cache1->obj, CALL); + assert(cache1->obj != NULL); + PyObject *callable = PEEK(3); + DEOPT_IF(callable != cache1->obj, PRECALL); PyObject *list = SECOND(); - DEOPT_IF(!PyList_Check(list), CALL); - STAT_INC(CALL, hit); + DEOPT_IF(!PyList_Check(list), PRECALL); + STAT_INC(PRECALL, hit); + next_instr++; // Skip following call PyObject *arg = TOP(); int err = PyList_Append(list, arg); if (err) { @@ -4995,24 +5047,25 @@ handle_eval_breaker: } Py_DECREF(arg); Py_DECREF(list); - STACK_SHRINK(3-is_meth); + STACK_SHRINK(2); Py_INCREF(Py_None); SET_TOP(Py_None); Py_DECREF(callable); NOTRACE_DISPATCH(); } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_O) { assert(call_shape.kwnames == NULL); int original_oparg = GET_CACHE()->adaptive.original_oparg; int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2, PRECALL); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(meth->ml_flags != METH_O, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall // function calls. @@ -5026,7 +5079,7 @@ handle_eval_breaker: assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); Py_DECREF(arg); - STACK_SHRINK(3-is_meth); + STACK_SHRINK(original_oparg+1); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { @@ -5036,17 +5089,19 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { assert(call_shape.kwnames == NULL); int original_oparg = GET_CACHE()->adaptive.original_oparg; + assert(original_oparg == 0 || original_oparg == 1); int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; - PyObject *callable = PEEK(total_args + 1); - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 1, PRECALL); + PyObject *callable = SECOND(); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method; - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(meth->ml_flags != METH_NOARGS, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall // function calls. @@ -5058,7 +5113,7 @@ handle_eval_breaker: _Py_LeaveRecursiveCall(tstate); assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); Py_DECREF(self); - STACK_SHRINK(2-is_meth); + STACK_SHRINK(original_oparg+1); SET_TOP(res); Py_DECREF(callable); if (res == NULL) { @@ -5068,17 +5123,18 @@ handle_eval_breaker: DISPATCH(); } - TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST) { assert(call_shape.kwnames == NULL); int original_oparg = GET_CACHE()->adaptive.original_oparg; int is_meth = is_method(stack_pointer, original_oparg); int total_args = original_oparg + is_meth; PyObject *callable = PEEK(total_args + 1); /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL); PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); - STAT_INC(CALL, hit); + DEOPT_IF(meth->ml_flags != METH_FASTCALL, PRECALL); + next_instr++; // Skip following call + STAT_INC(PRECALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args-1; STACK_SHRINK(nargs); @@ -5456,6 +5512,7 @@ MISS_WITH_CACHE(LOAD_ATTR) MISS_WITH_CACHE(STORE_ATTR) MISS_WITH_CACHE(LOAD_GLOBAL) MISS_WITH_CACHE(LOAD_METHOD) +MISS_WITH_CACHE(PRECALL) MISS_WITH_CACHE(CALL) MISS_WITH_CACHE(BINARY_OP) MISS_WITH_CACHE(COMPARE_OP) diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index f6cbec7..cf5bdc7 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -35,52 +35,52 @@ static void *opcode_targets[256] = { &&TARGET_MATCH_KEYS, &&TARGET_CALL_ADAPTIVE, &&TARGET_PUSH_EXC_INFO, - &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_NO_KW_BUILTIN_O, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_LEN, - &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_NO_KW_LIST_APPEND, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_WITH_EXCEPT_START, - &&TARGET_GET_AITER, - &&TARGET_GET_ANEXT, - &&TARGET_BEFORE_ASYNC_WITH, - &&TARGET_BEFORE_WITH, - &&TARGET_END_ASYNC_FOR, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_JUMP_ABSOLUTE_QUICK, &&TARGET_LOAD_ATTR_ADAPTIVE, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_LOAD_METHOD_CACHED, + &&TARGET_WITH_EXCEPT_START, + &&TARGET_GET_AITER, + &&TARGET_GET_ANEXT, + &&TARGET_BEFORE_ASYNC_WITH, + &&TARGET_BEFORE_WITH, + &&TARGET_END_ASYNC_FOR, + &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_METHOD_MODULE, + &&TARGET_LOAD_METHOD_NO_DICT, + &&TARGET_PRECALL_ADAPTIVE, + &&TARGET_PRECALL_BUILTIN_CLASS, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, + &&TARGET_PRECALL_NO_KW_BUILTIN_O, + &&TARGET_PRECALL_NO_KW_BUILTIN_FAST, + &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_PRECALL_NO_KW_LEN, + &&TARGET_PRECALL_NO_KW_ISINSTANCE, + &&TARGET_PRECALL_NO_KW_LIST_APPEND, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_METHOD_ADAPTIVE, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_GET_AWAITABLE, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_METHOD_CACHED, - &&TARGET_LOAD_METHOD_CLASS, - &&TARGET_LOAD_METHOD_MODULE, - &&TARGET_LOAD_METHOD_NO_DICT, - &&TARGET_RESUME_QUICK, - &&TARGET_STORE_ATTR_ADAPTIVE, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_PRECALL_NO_KW_STR_1, + &&TARGET_PRECALL_NO_KW_TUPLE_1, + &&TARGET_PRECALL_NO_KW_TYPE_1, + &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, + &&TARGET_PRECALL_BOUND_METHOD, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, @@ -130,7 +130,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_PRECALL_PYFUNC, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_NO_INTERRUPT, @@ -139,40 +139,40 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_RESUME_QUICK, + &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, &&TARGET_LOAD_CLASSDEREF, &&TARGET_COPY_FREE_VARS, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_STORE_ATTR_SLOT, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_UNPACK_SEQUENCE_ADAPTIVE, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_LOAD_METHOD, - &&TARGET_LOAD_FAST__LOAD_CONST, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, &&TARGET_PRECALL, + &&TARGET_LOAD_FAST__LOAD_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_STORE_FAST__STORE_FAST, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_CALL, &&TARGET_KW_NAMES, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 7dd12c7..b46f701 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -48,6 +48,7 @@ static uint8_t adaptive_opcodes[256] = { [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, [CALL] = CALL_ADAPTIVE, + [PRECALL] = PRECALL_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -62,6 +63,7 @@ static uint8_t cache_requirements[256] = { [BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */ [STORE_SUBSCR] = 0, [CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ + [PRECALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */ [BINARY_OP] = 1, // _PyAdaptiveEntry [COMPARE_OP] = 1, /* _PyAdaptiveEntry */ @@ -1454,35 +1456,36 @@ specialize_class_call( PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache) { + assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE); PyTypeObject *tp = _PyType_CAST(callable); if (tp->tp_new == PyBaseObject_Type.tp_new) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PYTHON_CLASS); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS); return -1; } if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { - if (nargs == 1 && kwnames == NULL) { + if (nargs == 1 && kwnames == NULL && cache->adaptive.original_oparg == 1) { if (tp == &PyUnicode_Type) { - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_STR_1, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_STR_1, _Py_OPARG(*instr)); return 0; } else if (tp == &PyType_Type) { - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_TYPE_1, _Py_OPARG(*instr)); return 0; } else if (tp == &PyTuple_Type) { - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TUPLE_1, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_TUPLE_1, _Py_OPARG(*instr)); return 0; } } if (tp->tp_vectorcall != NULL) { - *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_CLASS, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(PRECALL_BUILTIN_CLASS, _Py_OPARG(*instr)); return 0; } - SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ? + SPECIALIZATION_FAIL(PRECALL, tp == &PyUnicode_Type ? SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL); return -1; } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_CLASS_MUTABLE); return -1; } @@ -1517,8 +1520,9 @@ specialize_method_descriptor( PyMethodDescrObject *descr, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache) { + assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE); if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES); return -1; } if (_list_append == NULL) { @@ -1526,10 +1530,9 @@ specialize_method_descriptor( &_Py_ID(append)); } assert(_list_append != NULL); - if (nargs == 2 && descr == _list_append) { - assert(_Py_OPCODE(instr[-1]) == PRECALL); + if (nargs == 2 && descr == _list_append && cache->adaptive.original_oparg == 1) { cache[-1].obj.obj = (PyObject *)_list_append; - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr)); return 0; } @@ -1538,29 +1541,29 @@ specialize_method_descriptor( METH_KEYWORDS | METH_METHOD)) { case METH_NOARGS: { if (nargs != 1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, _Py_OPARG(*instr)); return 0; } case METH_O: { if (nargs != 2) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_OUT_OF_RANGE); return -1; } - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_O, _Py_OPARG(*instr)); return 0; } case METH_FASTCALL: { - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST, _Py_OPARG(*instr)); return 0; } } - SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags)); + SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(descr->d_method->ml_flags)); return -1; } @@ -1569,6 +1572,7 @@ specialize_py_call( PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache) { + assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE); _PyCallCache *cache1 = &cache[-1].call; PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); @@ -1621,6 +1625,7 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins) { + assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE); _PyObjectCache *cache1 = &cache[-1].obj; if (PyCFunction_GET_FUNCTION(callable) == NULL) { return 1; @@ -1630,28 +1635,28 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, METH_KEYWORDS | METH_METHOD)) { case METH_O: { if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES); return -1; } if (nargs != 1) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return 1; } /* len(o) */ PyObject *builtin_len = PyDict_GetItemString(builtins, "len"); if (callable == builtin_len) { cache1->obj = builtin_len; // borrowed - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LEN, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_LEN, _Py_OPARG(*instr)); return 0; } - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_O, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_BUILTIN_O, _Py_OPARG(*instr)); return 0; } case METH_FASTCALL: { if (kwnames) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES); + SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES); return -1; } if (nargs == 2) { @@ -1660,22 +1665,22 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, builtins, "isinstance"); if (callable == builtin_isinstance) { cache1->obj = builtin_isinstance; // borrowed - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_ISINSTANCE, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_ISINSTANCE, _Py_OPARG(*instr)); return 0; } } - *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_FAST, + *instr = _Py_MAKECODEUNIT(PRECALL_NO_KW_BUILTIN_FAST, _Py_OPARG(*instr)); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_FAST_WITH_KEYWORDS, + *instr = _Py_MAKECODEUNIT(PRECALL_BUILTIN_FAST_WITH_KEYWORDS, _Py_OPARG(*instr)); return 0; } default: - SPECIALIZATION_FAIL(CALL, + SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); return 1; } @@ -1722,11 +1727,9 @@ call_fail_kind(PyObject *callable) } #endif -/* TODO: - - Specialize calling classes. -*/ + int -_Py_Specialize_CallNoKw( +_Py_Specialize_Precall( PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins) @@ -1737,7 +1740,8 @@ _Py_Specialize_CallNoKw( fail = specialize_c_call(callable, instr, nargs, kwnames, cache, builtins); } else if (PyFunction_Check(callable)) { - fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache); + *instr = _Py_MAKECODEUNIT(PRECALL_PYFUNC, _Py_OPARG(*instr)); + fail = 0; } else if (PyType_Check(callable)) { fail = specialize_class_call(callable, instr, nargs, kwnames, cache); @@ -1746,6 +1750,42 @@ _Py_Specialize_CallNoKw( fail = specialize_method_descriptor( (PyMethodDescrObject *)callable, instr, nargs, kwnames, cache); } + else if (Py_TYPE(callable) == &PyMethod_Type) { + *instr = _Py_MAKECODEUNIT(PRECALL_BOUND_METHOD, _Py_OPARG(*instr)); + fail = 0; + } + else { + SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); + fail = -1; + } + if (fail) { + STAT_INC(CALL, failure); + assert(!PyErr_Occurred()); + cache_backoff(cache0); + } + else { + STAT_INC(CALL, success); + assert(!PyErr_Occurred()); + cache0->counter = initial_counter_value(); + } + return 0; +} + + +/* TODO: + - Specialize calling classes. +*/ +int +_Py_Specialize_Call( + PyObject *callable, _Py_CODEUNIT *instr, + int nargs, PyObject *kwnames, + SpecializedCacheEntry *cache) +{ + _PyAdaptiveEntry *cache0 = &cache->adaptive; + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache); + } else { SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); fail = -1; |