summaryrefslogtreecommitdiffstats
path: root/Python
diff options
context:
space:
mode:
authorMark Shannon <mark@hotpy.org>2024-08-20 15:52:58 (GMT)
committerGitHub <noreply@github.com>2024-08-20 15:52:58 (GMT)
commitbb1d30336e83837d4191a016107fd501cd230328 (patch)
treebe948ce7bf459bbd95195bd57262d775dc7efa76 /Python
parentbffed80230f2617de2ee02bd4bdded1024234dab (diff)
downloadcpython-bb1d30336e83837d4191a016107fd501cd230328.zip
cpython-bb1d30336e83837d4191a016107fd501cd230328.tar.gz
cpython-bb1d30336e83837d4191a016107fd501cd230328.tar.bz2
GH-118093: Make `CALL_ALLOC_AND_ENTER_INIT` suitable for tier 2. (GH-123140)
* Convert CALL_ALLOC_AND_ENTER_INIT to micro-ops such that tier 2 supports it * Allow inexact arguments for CALL_ALLOC_AND_ENTER_INIT.
Diffstat (limited to 'Python')
-rw-r--r--Python/bytecodes.c81
-rw-r--r--Python/ceval.c20
-rw-r--r--Python/ceval_macros.h2
-rw-r--r--Python/executor_cases.c.h100
-rw-r--r--Python/generated_cases.c.h163
-rw-r--r--Python/optimizer_bytecodes.c19
-rw-r--r--Python/optimizer_cases.c.h41
-rw-r--r--Python/specialize.c6
8 files changed, 287 insertions, 145 deletions
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index ec57c07..ec0f6ab 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -797,7 +797,7 @@ dummy_func(
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem = ht->_spec_cache.getitem;
- new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2);
+ new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame);
SYNC_SP();
new_frame->localsplus[0] = container;
new_frame->localsplus[1] = sub;
@@ -1071,6 +1071,8 @@ dummy_func(
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
+ assert(gen_frame->previous == NULL);
+ gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
}
if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
@@ -1113,6 +1115,7 @@ dummy_func(
tstate->exc_info = &gen->gi_exc_state;
assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
+ gen_frame->previous = frame;
}
macro(SEND_GEN) =
@@ -2143,7 +2146,7 @@ dummy_func(
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
- new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
+ new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame);
new_frame->localsplus[0] = owner;
}
@@ -2175,7 +2178,7 @@ dummy_func(
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
- _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
+ _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
@@ -2956,6 +2959,7 @@ dummy_func(
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
+ gen_frame->previous = frame;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
}
@@ -3225,7 +3229,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
@@ -3315,7 +3319,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
@@ -3454,7 +3458,7 @@ dummy_func(
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3468,7 +3472,7 @@ dummy_func(
assert(tstate->interp->eval_frame == NULL);
SYNC_SP();
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -3550,58 +3554,51 @@ dummy_func(
_CALL_TUPLE_1 +
_CHECK_PERIODIC;
- inst(CALL_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) {
+ op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) {
PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
- /* This instruction does the following:
- * 1. Creates the object (by calling ``object.__new__``)
- * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
- * 3. Pushes the frame for ``__init__`` to the frame stack
- * */
- _PyCallCache *cache = (_PyCallCache *)&this_instr[1];
DEOPT_IF(!PyStackRef_IsNull(null));
DEOPT_IF(!PyType_Check(callable_o));
PyTypeObject *tp = (PyTypeObject *)callable_o;
- DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
+ DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
- PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
- PyCodeObject *code = (PyCodeObject *)init->func_code;
- DEOPT_IF(code->co_argcount != oparg+1);
+ PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init;
+ PyCodeObject *code = (PyCodeObject *)init_func->func_code;
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize));
STAT_INC(CALL, hit);
- PyObject *self = _PyType_NewManagedObject(tp);
- if (self == NULL) {
+ self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp));
+ if (PyStackRef_IsNull(self)) {
ERROR_NO_POP();
}
PyStackRef_CLOSE(callable);
+ init = PyStackRef_FromPyObjectNew(init_func);
+ }
+
+ op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _PyInterpreterFrame *)) {
_PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
- tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
+ tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
/* Push self onto stack of shim */
- Py_INCREF(self);
- shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
- Py_INCREF(init);
- _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
- /* Copy self followed by args to __init__ frame */
- init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
- for (int i = 0; i < oparg; i++) {
- init_frame->localsplus[i+1] = args[i];
- }
- frame->return_offset = (uint16_t)(next_instr - this_instr);
- STACK_SHRINK(oparg+2);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- /* Link frames */
- init_frame->previous = shim;
- shim->previous = frame;
- frame = tstate->current_frame = init_frame;
- CALL_STAT_INC(inlined_py_calls);
+ shim->localsplus[0] = PyStackRef_DUP(self);
+ PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init);
+ args[-1] = self;
+ init_frame = _PyEvalFramePushAndInit(
+ tstate, init_func, NULL, args-1, oparg+1, NULL, shim);
+ frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
+ SYNC_SP();
/* Account for pushing the extra frame.
* We don't check recursion depth here,
* as it will be checked after start_frame */
tstate->py_recursion_remaining--;
- goto start_frame;
}
+ macro(CALL_ALLOC_AND_ENTER_INIT) =
+ unused/1 +
+ _CHECK_PEP_523 +
+ _CHECK_AND_ALLOCATE_OBJECT +
+ _CREATE_INIT_FRAME +
+ _PUSH_FRAME;
+
inst(EXIT_INIT_CHECK, (should_be_none -- )) {
assert(STACK_LEVEL() == 2);
if (!PyStackRef_Is(should_be_none, PyStackRef_None)) {
@@ -4060,7 +4057,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
@@ -4129,7 +4126,7 @@ dummy_func(
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// The frame has stolen all the arguments from the stack,
@@ -4315,7 +4312,7 @@ dummy_func(
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
- nargs, callargs, kwargs);
+ nargs, callargs, kwargs, frame);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
diff --git a/Python/ceval.c b/Python/ceval.c
index c685a95..b615dee 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -246,7 +246,7 @@ static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
- PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs);
+ PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous);
#ifdef HAVE_ERRNO_H
#include <errno.h>
@@ -671,8 +671,6 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = {
{ .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START }
};
-extern const struct _PyCode_DEF(8) _Py_InitCleanup;
-
#ifdef Py_DEBUG
extern void _PyUOpPrint(const _PyUOpInstruction *uop);
#endif
@@ -1691,7 +1689,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame)
_PyInterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, _PyStackRef const* args,
- size_t argcount, PyObject *kwnames)
+ size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{
PyCodeObject * code = (PyCodeObject *)func->func_code;
CALL_STAT_INC(frames_pushed);
@@ -1699,7 +1697,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
if (frame == NULL) {
goto fail;
}
- _PyFrame_Initialize(frame, func, locals, code, 0);
+ _PyFrame_Initialize(frame, func, locals, code, 0, previous);
if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) {
assert(frame->owner == FRAME_OWNED_BY_THREAD);
clear_thread_frame(tstate, frame);
@@ -1726,7 +1724,7 @@ fail:
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
PyObject *locals, PyObject *const* args,
- size_t argcount, PyObject *kwnames)
+ size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous)
{
#if defined(Py_GIL_DISABLED)
size_t kw_count = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames);
@@ -1742,11 +1740,11 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
for (size_t i = 0; i < kw_count; i++) {
tagged_args_buffer[argcount + i] = PyStackRef_FromPyObjectSteal(args[argcount + i]);
}
- _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames);
+ _PyInterpreterFrame *res = _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)tagged_args_buffer, argcount, kwnames, previous);
PyMem_Free(tagged_args_buffer);
return res;
#else
- return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames);
+ return _PyEvalFramePushAndInit(tstate, func, locals, (_PyStackRef const *)args, argcount, kwnames, previous);
#endif
}
@@ -1755,7 +1753,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func,
*/
static _PyInterpreterFrame *
_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
- PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs)
+ PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous)
{
bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0);
PyObject *kwnames = NULL;
@@ -1776,7 +1774,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
}
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged(
tstate, (PyFunctionObject *)func, locals,
- newargs, nargs, kwnames
+ newargs, nargs, kwnames, previous
);
if (has_dict) {
_PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames);
@@ -1813,7 +1811,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
}
}
_PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged(
- tstate, func, locals, args, argcount, kwnames);
+ tstate, func, locals, args, argcount, kwnames, NULL);
if (frame == NULL) {
return NULL;
}
diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index ed146a1..9e15406 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -124,7 +124,7 @@ do { \
do { \
assert(tstate->interp->eval_frame == NULL); \
_PyFrame_SetStackPointer(frame, stack_pointer); \
- (NEW_FRAME)->previous = frame; \
+ assert((NEW_FRAME)->previous == frame); \
frame = tstate->current_frame = (NEW_FRAME); \
CALL_STAT_INC(inlined_py_calls); \
goto start_frame; \
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index d699e8d..9226b7a 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -1031,7 +1031,7 @@
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem = ht->_spec_cache.getitem;
- new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2);
+ new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
new_frame->localsplus[0] = container;
@@ -1319,6 +1319,7 @@
tstate->exc_info = &gen->gi_exc_state;
assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
+ gen_frame->previous = frame;
stack_pointer[-1].bits = (uintptr_t)gen_frame;
break;
}
@@ -2552,7 +2553,7 @@
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
- new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
+ new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame);
new_frame->localsplus[0] = owner;
stack_pointer[-1].bits = (uintptr_t)new_frame;
break;
@@ -3316,6 +3317,7 @@
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
+ gen_frame->previous = frame;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
stack_pointer[0].bits = (uintptr_t)gen_frame;
@@ -3604,7 +3606,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
@@ -3838,7 +3840,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3864,7 +3866,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3890,7 +3892,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3916,7 +3918,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3942,7 +3944,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3967,7 +3969,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -3988,7 +3990,7 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -4087,7 +4089,81 @@
break;
}
- /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
+ case _CHECK_AND_ALLOCATE_OBJECT: {
+ _PyStackRef *args;
+ _PyStackRef null;
+ _PyStackRef callable;
+ _PyStackRef self;
+ _PyStackRef init;
+ oparg = CURRENT_OPARG();
+ args = &stack_pointer[-oparg];
+ null = stack_pointer[-1 - oparg];
+ callable = stack_pointer[-2 - oparg];
+ uint32_t type_version = (uint32_t)CURRENT_OPERAND();
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ if (!PyStackRef_IsNull(null)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ if (!PyType_Check(callable_o)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
+ if (tp->tp_version_tag != type_version) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
+ PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init;
+ PyCodeObject *code = (PyCodeObject *)init_func->func_code;
+ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) {
+ UOP_STAT_INC(uopcode, miss);
+ JUMP_TO_JUMP_TARGET();
+ }
+ STAT_INC(CALL, hit);
+ self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp));
+ if (PyStackRef_IsNull(self)) {
+ JUMP_TO_ERROR();
+ }
+ PyStackRef_CLOSE(callable);
+ init = PyStackRef_FromPyObjectNew(init_func);
+ stack_pointer[-1 - oparg] = init;
+ stack_pointer[-2 - oparg] = self;
+ break;
+ }
+
+ case _CREATE_INIT_FRAME: {
+ _PyStackRef *args;
+ _PyStackRef init;
+ _PyStackRef self;
+ _PyInterpreterFrame *init_frame;
+ oparg = CURRENT_OPARG();
+ args = &stack_pointer[-oparg];
+ init = stack_pointer[-1 - oparg];
+ self = stack_pointer[-2 - oparg];
+ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
+ tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
+ assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
+ /* Push self onto stack of shim */
+ shim->localsplus[0] = PyStackRef_DUP(self);
+ PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init);
+ args[-1] = self;
+ init_frame = _PyEvalFramePushAndInit(
+ tstate, init_func, NULL, args-1, oparg+1, NULL, shim);
+ frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
+ stack_pointer += -2 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ /* Account for pushing the extra frame.
+ * We don't check recursion depth here,
+ * as it will be checked after start_frame */
+ tstate->py_recursion_remaining--;
+ stack_pointer[0].bits = (uintptr_t)init_frame;
+ stack_pointer += 1;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
case _EXIT_INIT_CHECK: {
_PyStackRef should_be_none;
@@ -4705,7 +4781,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// The frame has stolen all the arguments from the stack,
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 486d356..27971ce 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -504,7 +504,7 @@
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
PyObject *getitem = ht->_spec_cache.getitem;
- new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2);
+ new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame);
stack_pointer += -2;
assert(WITHIN_STACK_BOUNDS());
new_frame->localsplus[0] = container;
@@ -517,7 +517,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -888,7 +888,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
@@ -975,60 +975,75 @@
_PyStackRef callable;
_PyStackRef null;
_PyStackRef *args;
+ _PyStackRef self;
+ _PyStackRef init;
+ _PyInterpreterFrame *init_frame;
+ _PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
- /* Skip 2 cache entries */
+ // _CHECK_PEP_523
+ {
+ DEOPT_IF(tstate->interp->eval_frame, CALL);
+ }
+ // _CHECK_AND_ALLOCATE_OBJECT
args = &stack_pointer[-oparg];
null = stack_pointer[-1 - oparg];
callable = stack_pointer[-2 - oparg];
- PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
- /* This instruction does the following:
- * 1. Creates the object (by calling ``object.__new__``)
- * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``)
- * 3. Pushes the frame for ``__init__`` to the frame stack
- * */
- _PyCallCache *cache = (_PyCallCache *)&this_instr[1];
- DEOPT_IF(!PyStackRef_IsNull(null), CALL);
- DEOPT_IF(!PyType_Check(callable_o), CALL);
- PyTypeObject *tp = (PyTypeObject *)callable_o;
- DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
- assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
- PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
- PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
- PyCodeObject *code = (PyCodeObject *)init->func_code;
- DEOPT_IF(code->co_argcount != oparg+1, CALL);
- DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
- STAT_INC(CALL, hit);
- PyObject *self = _PyType_NewManagedObject(tp);
- if (self == NULL) {
- goto error;
+ {
+ uint32_t type_version = read_u32(&this_instr[2].cache);
+ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
+ DEOPT_IF(!PyStackRef_IsNull(null), CALL);
+ DEOPT_IF(!PyType_Check(callable_o), CALL);
+ PyTypeObject *tp = (PyTypeObject *)callable_o;
+ DEOPT_IF(tp->tp_version_tag != type_version, CALL);
+ assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+ PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o;
+ PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init;
+ PyCodeObject *code = (PyCodeObject *)init_func->func_code;
+ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL);
+ STAT_INC(CALL, hit);
+ self = PyStackRef_FromPyObjectSteal(_PyType_NewManagedObject(tp));
+ if (PyStackRef_IsNull(self)) {
+ goto error;
+ }
+ PyStackRef_CLOSE(callable);
+ init = PyStackRef_FromPyObjectNew(init_func);
+ stack_pointer[-1 - oparg] = init;
+ }
+ // _CREATE_INIT_FRAME
+ {
+ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
+ tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
+ assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
+ /* Push self onto stack of shim */
+ shim->localsplus[0] = PyStackRef_DUP(self);
+ PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init);
+ args[-1] = self;
+ init_frame = _PyEvalFramePushAndInit(
+ tstate, init_func, NULL, args-1, oparg+1, NULL, shim);
+ frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL;
+ stack_pointer += -2 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ /* Account for pushing the extra frame.
+ * We don't check recursion depth here,
+ * as it will be checked after start_frame */
+ tstate->py_recursion_remaining--;
}
- PyStackRef_CLOSE(callable);
- _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
- tstate, (PyCodeObject *)&_Py_InitCleanup, 1);
- assert(_PyCode_CODE((PyCodeObject *)shim->f_executable)[0].op.code == EXIT_INIT_CHECK);
- /* Push self onto stack of shim */
- Py_INCREF(self);
- shim->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
- Py_INCREF(init);
- _PyInterpreterFrame *init_frame = _PyFrame_PushUnchecked(tstate, init, oparg+1);
- /* Copy self followed by args to __init__ frame */
- init_frame->localsplus[0] = PyStackRef_FromPyObjectSteal(self);
- for (int i = 0; i < oparg; i++) {
- init_frame->localsplus[i+1] = args[i];
+ // _PUSH_FRAME
+ new_frame = init_frame;
+ {
+ // Write it out explicitly because it's subtly different.
+ // Eventually this should be the only occurrence of this code.
+ assert(tstate->interp->eval_frame == NULL);
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
+ CALL_STAT_INC(inlined_py_calls);
+ frame = tstate->current_frame = new_frame;
+ tstate->py_recursion_remaining--;
+ LOAD_SP();
+ LOAD_IP(0);
+ LLTRACE_RESUME_FRAME();
}
- frame->return_offset = (uint16_t)(next_instr - this_instr);
- STACK_SHRINK(oparg+2);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- /* Link frames */
- init_frame->previous = shim;
- shim->previous = frame;
- frame = tstate->current_frame = init_frame;
- CALL_STAT_INC(inlined_py_calls);
- /* Account for pushing the extra frame.
- * We don't check recursion depth here,
- * as it will be checked after start_frame */
- tstate->py_recursion_remaining--;
- goto start_frame;
+ DISPATCH();
}
TARGET(CALL_BOUND_METHOD_EXACT_ARGS) {
@@ -1099,7 +1114,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -1123,7 +1138,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -1194,7 +1209,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
@@ -1219,7 +1234,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -1595,7 +1610,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
- nargs, callargs, kwargs);
+ nargs, callargs, kwargs, frame);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
@@ -1781,7 +1796,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
@@ -1916,7 +1931,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// The frame has stolen all the arguments from the stack,
@@ -1942,7 +1957,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -2086,7 +2101,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, positional_args, kwnames_o
+ args, positional_args, kwnames_o, frame
);
PyStackRef_CLOSE(kwnames);
// The frame has stolen all the arguments from the stack,
@@ -2112,7 +2127,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -2635,7 +2650,7 @@
int has_self = !PyStackRef_IsNull(self_or_null);
STAT_INC(CALL, hit);
PyFunctionObject *func = (PyFunctionObject *)callable_o;
- new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self);
+ new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame);
_PyStackRef *first_non_self_local = new_frame->localsplus + has_self;
new_frame->localsplus[0] = self_or_null;
for (int i = 0; i < oparg; i++) {
@@ -2659,7 +2674,7 @@
stack_pointer += -2 - oparg;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -2710,7 +2725,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// The frame has stolen all the arguments from the stack,
// so there is no need to clean them up.
@@ -2735,7 +2750,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -3667,6 +3682,7 @@
gen->gi_frame_state = FRAME_EXECUTING;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
+ gen_frame->previous = frame;
// oparg is the return offset from the next instruction.
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg);
}
@@ -3677,7 +3693,7 @@
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -4090,7 +4106,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit(
tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals,
- args, total_args, NULL
+ args, total_args, NULL, frame
);
// Manipulate stack directly since we leave using DISPATCH_INLINED().
STACK_SHRINK(oparg + 2);
@@ -4944,7 +4960,7 @@
STAT_INC(LOAD_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
Py_INCREF(f);
- _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2);
+ _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
@@ -5274,7 +5290,7 @@
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
- new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
+ new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame);
new_frame->localsplus[0] = owner;
}
// _SAVE_RETURN_OFFSET
@@ -5294,7 +5310,7 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -6551,6 +6567,8 @@
tstate->exc_info = &gen->gi_exc_state;
assert(next_instr - this_instr + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(next_instr - this_instr + oparg);
+ assert(gen_frame->previous == NULL);
+ gen_frame->previous = frame;
DISPATCH_INLINED(gen_frame);
}
if (PyStackRef_Is(v, PyStackRef_None) && PyIter_Check(receiver_o)) {
@@ -6611,6 +6629,7 @@
tstate->exc_info = &gen->gi_exc_state;
assert(1 + INLINE_CACHE_ENTRIES_SEND + oparg <= UINT16_MAX);
frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg);
+ gen_frame->previous = frame;
}
// _PUSH_FRAME
new_frame = gen_frame;
@@ -6621,7 +6640,7 @@
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
- new_frame->previous = frame;
+ assert(new_frame->previous == frame || new_frame->previous->previous == frame);
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
@@ -7650,7 +7669,7 @@
PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func));
_PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate,
(PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals,
- nargs, callargs, kwargs);
+ nargs, callargs, kwargs, frame);
// Need to manually shrink the stack since we exit with DISPATCH_INLINED.
STACK_SHRINK(oparg + 3);
if (new_frame == NULL) {
diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c
index 97e4c64..6e46d9b 100644
--- a/Python/optimizer_bytecodes.c
+++ b/Python/optimizer_bytecodes.c
@@ -627,7 +627,7 @@ dummy_func(void) {
ctx->done = true;
}
- op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame*)) {
+ op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame: _Py_UOpsAbstractFrame *)) {
(void)callable;
(void)self_or_null;
(void)args;
@@ -636,6 +636,23 @@ dummy_func(void) {
ctx->done = true;
}
+ op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, null, args[oparg] -- self, init, args[oparg])) {
+ (void)type_version;
+ (void)callable;
+ (void)null;
+ (void)args;
+ self = sym_new_not_null(ctx);
+ init = sym_new_not_null(ctx);
+ }
+
+ op(_CREATE_INIT_FRAME, (self, init, args[oparg] -- init_frame: _Py_UOpsAbstractFrame *)) {
+ (void)self;
+ (void)init;
+ (void)args;
+ init_frame = NULL;
+ ctx->done = true;
+ }
+
op(_RETURN_VALUE, (retval -- res)) {
SYNC_SP();
ctx->frame->stack_pointer = stack_pointer;
diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h
index 3ec2e69..e5be9d0 100644
--- a/Python/optimizer_cases.c.h
+++ b/Python/optimizer_cases.c.h
@@ -1866,7 +1866,46 @@
break;
}
- /* _CALL_ALLOC_AND_ENTER_INIT is not a viable micro-op for tier 2 */
+ case _CHECK_AND_ALLOCATE_OBJECT: {
+ _Py_UopsSymbol **args;
+ _Py_UopsSymbol *null;
+ _Py_UopsSymbol *callable;
+ _Py_UopsSymbol *self;
+ _Py_UopsSymbol *init;
+ args = &stack_pointer[-oparg];
+ null = stack_pointer[-1 - oparg];
+ callable = stack_pointer[-2 - oparg];
+ args = &stack_pointer[-oparg];
+ uint32_t type_version = (uint32_t)this_instr->operand;
+ (void)type_version;
+ (void)callable;
+ (void)null;
+ (void)args;
+ self = sym_new_not_null(ctx);
+ init = sym_new_not_null(ctx);
+ stack_pointer[-2 - oparg] = self;
+ stack_pointer[-1 - oparg] = init;
+ break;
+ }
+
+ case _CREATE_INIT_FRAME: {
+ _Py_UopsSymbol **args;
+ _Py_UopsSymbol *init;
+ _Py_UopsSymbol *self;
+ _Py_UOpsAbstractFrame *init_frame;
+ args = &stack_pointer[-oparg];
+ init = stack_pointer[-1 - oparg];
+ self = stack_pointer[-2 - oparg];
+ (void)self;
+ (void)init;
+ (void)args;
+ init_frame = NULL;
+ ctx->done = true;
+ stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)init_frame;
+ stack_pointer += -1 - oparg;
+ assert(WITHIN_STACK_BOUNDS());
+ break;
+ }
case _EXIT_INIT_CHECK: {
stack_pointer += -1;
diff --git a/Python/specialize.c b/Python/specialize.c
index 4fec5a3..b3a2e07 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -1809,10 +1809,6 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs)
return -1;
}
if (init != NULL) {
- if (((PyCodeObject *)init->func_code)->co_argcount != nargs+1) {
- SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
- return -1;
- }
_PyCallCache *cache = (_PyCallCache *)(instr + 1);
write_u32(cache->func_version, tp->tp_version_tag);
_Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT);
@@ -2654,7 +2650,7 @@ static const PyBytesObject no_location = {
.ob_sval = { NO_LOC_4 }
};
-const struct _PyCode_DEF(8) _Py_InitCleanup = {
+const struct _PyCode8 _Py_InitCleanup = {
_PyVarObject_HEAD_INIT(&PyCode_Type, 3),
.co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
.co_names = (PyObject *)&_Py_SINGLETON(tuple_empty),